libcm is a C development framework with an emphasis on audio signal processing applications.
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

cmLex.c 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. typedef struct
  10. {
  11. unsigned code;
  12. const cmChar_t* msg;
  13. } cmLexErrorRecd;
  14. cmLexErrorRecd cmLexErrorArray[] =
  15. {
  16. { kOkLexRC, "No error. The operation completed successfully."},
  17. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  18. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  19. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  20. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  21. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  22. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  23. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  24. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  25. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  26. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  27. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  28. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  29. { kInvalidLexRC, "Unknown lexer error code." }
  30. };
  31. struct cmLex_str;
  32. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  33. // token match function record
  34. typedef struct
  35. {
  36. unsigned typeId; // token type this matcher recognizes
  37. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  38. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  39. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  40. bool enableFl; // true if this matcher is enabled
  41. } cmLexMatcher;
  42. typedef struct cmLex_str
  43. {
  44. cmErr_t err;
  45. const cmChar_t* cp; // character buffer
  46. unsigned cn; // count of characters in buffer
  47. unsigned ci; // current buffer index position
  48. unsigned flags; // lexer control flags
  49. unsigned curTokenId; // type id of the current token
  50. unsigned curTokenCharIdx; // index into cp[] of the current token
  51. unsigned curTokenCharCnt; // count of characters in the current token
  52. unsigned curLine; // line number of the current token
  53. unsigned curCol; // column number of the current token
  54. unsigned nextLine;
  55. unsigned nextCol;
  56. cmChar_t* blockBegCmtStr;
  57. cmChar_t* blockEndCmtStr;
  58. cmChar_t* lineCmtStr;
  59. cmLexMatcher* mfp; // base of matcher array
  60. unsigned mfi; // next available matcher array slot
  61. unsigned mfn; // count of elementes in mfp[]
  62. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  63. } cmLex;
  64. cmLexH cmLexNullH = { NULL };
  65. bool _cmLexIsNewline( cmChar_t c )
  66. { return c == '\n'; }
  67. bool _cmLexIsCommentTypeId( unsigned typeId )
  68. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  69. cmLex* _cmLexHandleToPtr( cmLexH h )
  70. {
  71. cmLex* p = (cmLex*)h.h;
  72. assert(p != NULL);
  73. return p;
  74. };
  75. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  76. {
  77. va_list vl;
  78. va_start(vl,fmt);
  79. unsigned bufCharCnt = 512;
  80. char buf[ bufCharCnt+1 ];
  81. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  82. unsigned sn = strlen(buf);
  83. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  84. buf[bufCharCnt]=0;
  85. cmErrMsg(&p->err,rc,"%s",buf);
  86. va_end(vl);
  87. return rc;
  88. }
  89. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  90. {
  91. unsigned i = 0;
  92. unsigned n = strlen(keyStr);
  93. if( n <= cn )
  94. for(; i<=cn-n; ++i)
  95. if( strncmp(cp + i, keyStr, n ) == 0 )
  96. return i+n;
  97. return cmInvalidIdx;
  98. }
  99. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  100. {
  101. unsigned n = strlen(keyStr);
  102. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  103. }
  104. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  105. {
  106. unsigned i=0;
  107. for(; i<cn; ++i)
  108. if( !isspace(cp[i]) )
  109. break;
  110. return i;
  111. }
  112. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  113. {
  114. unsigned i = 0;
  115. unsigned n = 0; // decimal point counter
  116. unsigned d = 0; // digit counter
  117. bool fl = false; // true if this real includes an exponent
  118. for(; i<cn && n<=1; ++i)
  119. {
  120. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  121. continue;
  122. if( isdigit(cp[i]) ) // allow digits
  123. {
  124. ++d;
  125. continue;
  126. }
  127. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  128. ++n;
  129. else
  130. break;
  131. }
  132. // if there was at least one digit and the next char is an 'e'
  133. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  134. {
  135. d=0;
  136. ++i;
  137. unsigned j = i;
  138. for(; i<cn; ++i)
  139. {
  140. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  141. continue;
  142. if( isdigit(cp[i]) )
  143. {
  144. ++d;
  145. continue;
  146. }
  147. // stop at the first non-digit
  148. break;
  149. }
  150. // an exp exists if digits follwed the 'e'
  151. fl = d > 0;
  152. }
  153. return i>1 && (n==1 || fl) ? i : 0;
  154. }
  155. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  156. {
  157. unsigned i = 0;
  158. bool signFl = false;
  159. for(; i<cn; ++i)
  160. {
  161. if( i==0 && cp[i]=='-' )
  162. {
  163. signFl = true;
  164. continue;
  165. }
  166. if( !isdigit(cp[i]) )
  167. break;
  168. }
  169. // BUG BUG BUG
  170. // If an integer is specified using 'e' notiation
  171. // (see _cmLexRealMatcher()) and the number of exponent places
  172. // specified following the 'e' is positive and >= number of
  173. // digits following the decimal point (in effect zeros are
  174. // padded on the right side) then the value is an integer.
  175. //
  176. // The current implementation recognizes all numeric strings
  177. // containing a decimal point as reals.
  178. return signFl && i==1 ? 0 : i;
  179. }
  180. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  181. {
  182. unsigned i = 0;
  183. if( cn < 3 )
  184. return 0;
  185. if( cp[0]=='0' && cp[1]=='x')
  186. for(i=2; i<cn; ++i)
  187. if( !isxdigit(cp[i]) )
  188. break;
  189. return i;
  190. }
  191. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  192. {
  193. unsigned i = 0;
  194. if( isalpha(cp[0]) || (cp[0]== '_'))
  195. {
  196. i = 1;
  197. for(; i<cn; ++i)
  198. if( !isalnum(cp[i]) && (cp[i] != '_') )
  199. break;
  200. }
  201. return i;
  202. }
  203. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  204. {
  205. cmChar_t qStr[]="\"";
  206. unsigned n = strlen(qStr);
  207. if( strncmp(qStr,cp,n) == 0 )
  208. {
  209. unsigned i;
  210. if((i = _cmLexScanTo(cp+n, cn-n, qStr)) == cmInvalidIdx )
  211. {
  212. _cmLexError( p, kMissingEndQuoteLexRC, "Missing string end quote.");
  213. return 0;
  214. }
  215. return n+i;
  216. }
  217. return 0;
  218. }
  219. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  220. {
  221. unsigned n = strlen(p->blockBegCmtStr);
  222. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  223. {
  224. unsigned i;
  225. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  226. {
  227. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  228. return 0;
  229. }
  230. return n + i;
  231. }
  232. return 0;
  233. }
  234. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  235. {
  236. unsigned n = strlen(p->lineCmtStr);
  237. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  238. {
  239. unsigned i;
  240. const char newlineStr[] = "\n";
  241. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  242. {
  243. // no EOL was found so the comment must be on the last line of the source
  244. return cn;
  245. }
  246. return n + i;
  247. }
  248. return 0;
  249. }
  250. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  251. {
  252. assert( funcPtr==NULL || userPtr==NULL );
  253. assert( !(funcPtr==NULL && userPtr==NULL));
  254. // if there is no space in the user token array - then expand it
  255. if( p->mfi == p->mfn )
  256. {
  257. int incr_cnt = 10;
  258. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  259. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  260. cmMemPtrFree(&p->mfp);
  261. p->mfp = np;
  262. p->mfn += incr_cnt;
  263. }
  264. p->mfp[p->mfi].tokenStr = NULL;
  265. p->mfp[p->mfi].typeId = typeId;
  266. p->mfp[p->mfi].funcPtr = funcPtr;
  267. p->mfp[p->mfi].userPtr = userPtr;
  268. p->mfp[p->mfi].enableFl = true;
  269. if( keyStr != NULL )
  270. {
  271. // allocate space for the token string and store it
  272. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  273. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  274. }
  275. p->mfi++;
  276. return kOkLexRC;
  277. }
  278. cmRC_t _cmLexReset( cmLex* p )
  279. {
  280. p->ci = 0;
  281. p->curTokenId = kErrorLexTId;
  282. p->curTokenCharIdx = cmInvalidIdx;
  283. p->curTokenCharCnt = 0;
  284. p->curLine = 0;
  285. p->curCol = 0;
  286. p->nextLine = 0;
  287. p->nextCol = 0;
  288. cmErrClearRC(&p->err);
  289. return kOkLexRC;
  290. }
  291. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  292. {
  293. p->cp = cp;
  294. p->cn = cn;
  295. return _cmLexReset(p);
  296. }
  297. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  298. {
  299. cmLexH h;
  300. cmChar_t dfltLineCmt[] = "//";
  301. cmChar_t dfltBlockBegCmt[] = "/*";
  302. cmChar_t dfltBlockEndCmt[] = "*/";
  303. cmLex* p = cmMemAllocZ( cmLex, 1 );
  304. cmErrSetup(&p->err,rpt,"Lexer");
  305. p->flags = flags;
  306. _cmLexSetTextBuffer( p, cp, cn );
  307. /*
  308. p->cp = (cn==0) ? NULL : cp;
  309. p->cn = (cp==NULL) ? 0 : cn;
  310. p->ci = 0;
  311. p->curTokenId = kErrorLexTId;
  312. p->curTokenCharIdx = cmInvalidIdx;
  313. p->curTokenCharCnt = 0;
  314. p->curLine = 0;
  315. p->curCol = 0;
  316. p->nextLine = 0;
  317. p->nextCol = 0;
  318. */
  319. int init_mfn = 10;
  320. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  321. p->mfn = init_mfn;
  322. p->mfi = 0;
  323. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  324. strcpy( p->lineCmtStr, dfltLineCmt );
  325. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  326. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  327. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  328. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  329. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  330. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  331. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  332. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  333. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  334. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  335. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  336. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  337. h.h = p;
  338. _cmLexReset(p);
  339. return h;
  340. }
  341. cmRC_t cmLexFinal( cmLexH* hp )
  342. {
  343. if( hp == NULL || cmLexIsValid(*hp)==false )
  344. return cmOkRC;
  345. cmLex* p = _cmLexHandleToPtr(*hp);
  346. if( p != NULL )
  347. {
  348. if( p->mfp != NULL )
  349. {
  350. unsigned i = 0;
  351. // free the user token strings
  352. for(; i<p->mfi; ++i)
  353. if( p->mfp[i].tokenStr != NULL )
  354. cmMemPtrFree(&p->mfp[i].tokenStr);
  355. // free the matcher array
  356. cmMemPtrFree(&p->mfp);
  357. p->mfi = 0;
  358. p->mfn = 0;
  359. }
  360. cmMemPtrFree(&p->lineCmtStr);
  361. cmMemPtrFree(&p->blockBegCmtStr);
  362. cmMemPtrFree(&p->blockEndCmtStr);
  363. cmMemPtrFree(&p->textBuf);
  364. // free the lexer object
  365. cmMemPtrFree(&p);
  366. hp->h = NULL;
  367. }
  368. return kOkLexRC;
  369. }
  370. cmRC_t cmLexReset( cmLexH h )
  371. {
  372. cmLex* p = _cmLexHandleToPtr(h);
  373. return _cmLexReset(p);
  374. }
  375. bool cmLexIsValid( cmLexH h )
  376. { return h.h != NULL; }
  377. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  378. {
  379. cmLex* p = _cmLexHandleToPtr(h);
  380. return _cmLexSetTextBuffer(p,cp,cn);
  381. }
  382. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  383. {
  384. cmRC_t rc = kOkLexRC;
  385. cmFileH_t fh = cmFileNullHandle;
  386. cmLex* p = _cmLexHandleToPtr(h);
  387. long n = 0;
  388. assert( fn != NULL && p != NULL );
  389. // open the file
  390. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  391. return kFileOpenErrLexRC;
  392. // seek to the end of the file
  393. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  394. return kFileSeekErrLexRC;
  395. // get the length of the file
  396. if( cmFileTell(fh,&n) != kOkFileRC )
  397. return kFileTellErrLexRC;
  398. // rewind to the beginning of the file
  399. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  400. return kFileSeekErrLexRC;
  401. // allocate the text buffer
  402. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  403. {
  404. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  405. goto errLabel;
  406. }
  407. // read the file into the buffer
  408. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  409. return kFileReadErrLexRC;
  410. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  411. goto errLabel;
  412. errLabel:
  413. // close the file
  414. if( cmFileClose(&fh) != kOkFileRC )
  415. return kFileCloseErrLexRC;
  416. return rc;
  417. }
  418. /*
  419. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  420. {
  421. cmRC_t rc = kOkLexRC;
  422. FILE* fp = NULL;
  423. cmLex* p = _cmLexHandleToPtr(h);
  424. unsigned n = 0;
  425. assert( fn != NULL && p != NULL );
  426. // open the file
  427. if((fp = fopen(fn,"rb")) == NULL )
  428. return _cmLexError(p,kFileOpenErrLexRC,"Unable to open the file:'%s'.",fn);
  429. // seek to the end
  430. if( fseek(fp,0,SEEK_END) != 0 )
  431. {
  432. rc= _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the end of '%s'.",fn);
  433. goto errLabel;
  434. }
  435. // get the length of the file
  436. if( (n=ftell(fp)) == 0 )
  437. {
  438. rc = _cmLexError(p,kFileOpenErrLexRC,"The file '%s' appears to be empty.",fn);
  439. goto errLabel;
  440. }
  441. // rewind the file
  442. if( fseek(fp,0,SEEK_SET) != 0 )
  443. {
  444. rc = _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the beginning of '%s'.",fn);
  445. goto errLabel;
  446. }
  447. // allocate the text buffer
  448. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  449. {
  450. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  451. goto errLabel;
  452. }
  453. // read the file into the text buffer
  454. if( fread(p->textBuf,n,1,fp) != 1 )
  455. {
  456. rc = _cmLexError(p,kFileReadErrLexRC,"File read failed on:'%s'.",fn);
  457. goto errLabel;
  458. }
  459. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  460. goto errLabel;
  461. errLabel:
  462. // close the file
  463. if( fclose(fp) != 0 )
  464. {
  465. rc = _cmLexError(p,kFileCloseErrLexRC,"File close failed on:'%s'.",fn);
  466. goto errLabel;
  467. }
  468. return rc;
  469. }
  470. */
  471. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  472. {
  473. unsigned i = 0;
  474. for(; i<p->mfi; ++i)
  475. {
  476. if( id != cmInvalidId && p->mfp[i].typeId == id )
  477. return p->mfp + i;
  478. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  479. return p->mfp + i;
  480. }
  481. return NULL;
  482. }
  483. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  484. {
  485. cmLex* p = _cmLexHandleToPtr(h);
  486. // prevent duplicate tokens
  487. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  488. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  489. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  490. }
  491. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  492. {
  493. cmLex* p = _cmLexHandleToPtr(h);
  494. // prevent duplicate tokens
  495. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  496. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  497. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  498. }
  499. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  500. {
  501. cmLex* p = _cmLexHandleToPtr(h);
  502. unsigned mi = 0;
  503. for(; mi<p->mfi; ++mi)
  504. if( p->mfp[mi].typeId == id )
  505. {
  506. p->mfp[mi].enableFl = enableFl;
  507. return cmOkRC;
  508. }
  509. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  510. }
  511. unsigned cmLexFilterFlags( cmLexH h )
  512. {
  513. cmLex* p = _cmLexHandleToPtr(h);
  514. return p->flags;
  515. }
  516. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  517. {
  518. cmLex* p = _cmLexHandleToPtr(h);
  519. p->flags = flags;
  520. }
  521. unsigned cmLexGetNextToken( cmLexH h )
  522. {
  523. cmLex* p = _cmLexHandleToPtr(h);
  524. if( cmErrLastRC(&p->err) != kOkLexRC )
  525. return kErrorLexTId;
  526. while( p->ci < p->cn )
  527. {
  528. unsigned i;
  529. unsigned mi = 0;
  530. unsigned maxCharCnt = 0;
  531. unsigned maxIdx = cmInvalidIdx;
  532. p->curTokenId = kErrorLexTId;
  533. p->curTokenCharIdx = cmInvalidIdx;
  534. p->curTokenCharCnt = 0;
  535. // try each mater
  536. for(; mi<p->mfi; ++mi)
  537. if( p->mfp[mi].enableFl )
  538. {
  539. unsigned charCnt = 0;
  540. if( p->mfp[mi].funcPtr != NULL )
  541. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  542. else
  543. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  544. if( cmErrLastRC(&p->err) != kOkLexRC )
  545. return kErrorLexTId;
  546. // if this matched token is longer then the prev. matched token or
  547. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  548. if( (charCnt > maxCharCnt)
  549. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  550. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  551. )
  552. {
  553. maxCharCnt = charCnt;
  554. maxIdx = mi;
  555. }
  556. }
  557. // no token was matched
  558. if( maxIdx == cmInvalidIdx )
  559. {
  560. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  561. {
  562. maxCharCnt = 1;
  563. }
  564. else
  565. {
  566. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  567. return kErrorLexTId;
  568. }
  569. }
  570. // update the current line and column position
  571. p->curLine = p->nextLine;
  572. p->curCol = p->nextCol;
  573. // find the next column and line position
  574. for(i=0; i<maxCharCnt; ++i)
  575. {
  576. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  577. {
  578. p->nextLine++;
  579. p->nextCol = 1;
  580. }
  581. else
  582. p->nextCol++;
  583. }
  584. bool returnFl = true;
  585. if( maxIdx != cmInvalidIdx )
  586. {
  587. // check the space token filter
  588. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  589. returnFl = false;
  590. // check the comment token filter
  591. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  592. returnFl = false;
  593. }
  594. // update the lexer state
  595. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  596. p->curTokenCharIdx = p->ci;
  597. p->curTokenCharCnt = maxCharCnt;
  598. // advance the text buffer
  599. p->ci += maxCharCnt;
  600. if( returnFl )
  601. return p->curTokenId;
  602. }
  603. cmErrSetRC(&p->err,kEofRC);
  604. return kEofLexTId;
  605. }
  606. unsigned cmLexTokenId( cmLexH h )
  607. {
  608. cmLex* p = _cmLexHandleToPtr(h);
  609. return p->curTokenId;
  610. }
  611. const cmChar_t* cmLexTokenText( cmLexH h )
  612. {
  613. cmLex* p = _cmLexHandleToPtr(h);
  614. if( p->curTokenCharIdx == cmInvalidIdx )
  615. return NULL;
  616. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  617. return p->cp + p->curTokenCharIdx + n;
  618. }
  619. unsigned cmLexTokenCharCount( cmLexH h )
  620. {
  621. cmLex* p = _cmLexHandleToPtr(h);
  622. if( p->curTokenCharIdx == cmInvalidIdx )
  623. return 0;
  624. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  625. return p->curTokenCharCnt - n;
  626. }
  627. int cmLexTokenInt( cmLexH h )
  628. { return strtol( cmLexTokenText(h),NULL,0 ); }
  629. unsigned cmLexTokenUInt( cmLexH h )
  630. { return strtol( cmLexTokenText(h),NULL,0 ); }
  631. float cmLexTokenFloat( cmLexH h )
  632. { return strtof( cmLexTokenText(h),NULL ); }
  633. double cmLexTokenDouble( cmLexH h )
  634. { return strtod( cmLexTokenText(h),NULL ); }
  635. unsigned cmLexCurrentLineNumber( cmLexH h )
  636. {
  637. cmLex* p = _cmLexHandleToPtr(h);
  638. return p->curLine + 1;
  639. }
  640. unsigned cmLexCurrentColumnNumber( cmLexH h )
  641. {
  642. cmLex* p = _cmLexHandleToPtr(h);
  643. return p->curCol + 1;
  644. }
  645. unsigned cmLexErrorRC( cmLexH h )
  646. {
  647. cmLex* p = _cmLexHandleToPtr(h);
  648. return cmErrLastRC(&p->err);
  649. }
  650. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  651. {
  652. cmLex* p = _cmLexHandleToPtr(h);
  653. switch( typeId )
  654. {
  655. case kErrorLexTId: return "<error>";
  656. case kEofLexTId: return "<EOF>";
  657. case kSpaceLexTId: return "<space>";
  658. case kRealLexTId: return "<real>";
  659. case kIntLexTId: return "<int>";
  660. case kHexLexTId: return "<hex>";
  661. case kIdentLexTId: return "<ident>";
  662. case kQStrLexTId: return "<qstr>";
  663. case kBlockCmtLexTId: return "<bcmt>";
  664. case kLineCmtLexTId: return "<lcmt>";
  665. default:
  666. {
  667. cmLexMatcher* mp;
  668. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  669. return "<unknown>";
  670. return mp->tokenStr;
  671. }
  672. }
  673. return "<invalid>";
  674. }
  675. const cmChar_t* cmLexRcToMsg( unsigned rc )
  676. {
  677. unsigned i=0;
  678. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  679. if( cmLexErrorArray[i].code == rc )
  680. break;
  681. return cmLexErrorArray[i].msg;
  682. }
  683. //{ { label:cmLexEx }
  684. //(
  685. // cmLexTest() gives a simple cmLex example.
  686. //)
  687. //(
  688. void cmLexTest( cmRpt_t* rpt)
  689. {
  690. cmChar_t buf[] =
  691. "123ident0\n 123.456\nident0\n"
  692. "0xa12+.2\n"
  693. "// comment \n"
  694. "/* block \n"
  695. "comment */"
  696. "\"quoted string\""
  697. "ident1"
  698. "// last line comment";
  699. // initialize a lexer with a buffer of text
  700. cmLexH h = cmLexInit(buf,strlen(buf),
  701. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  702. // verify that the lexer initialization succeded.
  703. if( cmLexIsValid(h) == false )
  704. {
  705. cmRptPrintf(rpt,"Lexer initialization failed.");
  706. return;
  707. }
  708. // register some additional recoginizers
  709. cmLexRegisterToken(h,kUserLexTId+1,"+");
  710. cmLexRegisterToken(h,kUserLexTId+2,"-");
  711. unsigned tid;
  712. // ask for token id's
  713. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  714. {
  715. // print information about each token
  716. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  717. cmLexCurrentLineNumber(h),
  718. cmLexCurrentColumnNumber(h),
  719. cmLexIdToLabel(h,tid),
  720. cmLexTokenCharCount(h),
  721. cmLexTokenText(h) ,
  722. cmLexTokenCharCount(h));
  723. // if the token is a number ...
  724. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  725. {
  726. // ... then request the numbers value
  727. int iv = cmLexTokenInt(h);
  728. double dv = cmLexTokenDouble(h);
  729. cmRptPrintf(rpt,"%i %f",iv,dv);
  730. }
  731. cmRptPrintf(rpt,"\n");
  732. // handle errors
  733. if( tid == kErrorLexTId )
  734. {
  735. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  736. break;
  737. }
  738. }
  739. // finalize the lexer
  740. cmLexFinal(&h);
  741. }
  742. //)
  743. //}