libcm is a C development framework with an emphasis on audio signal processing applications.
Du kannst nicht mehr als 25 Themen auswählen Themen müssen mit entweder einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

cmLex.c 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. typedef struct
  10. {
  11. unsigned code;
  12. const cmChar_t* msg;
  13. } cmLexErrorRecd;
  14. cmLexErrorRecd cmLexErrorArray[] =
  15. {
  16. { kOkLexRC, "No error. The operation completed successfully."},
  17. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  18. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  19. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  20. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  21. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  22. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  23. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  24. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  25. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  26. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  27. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  28. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  29. { kInvalidLexRC, "Unknown lexer error code." }
  30. };
  31. struct cmLex_str;
  32. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  33. // token match function record
  34. typedef struct
  35. {
  36. unsigned typeId; // token type this matcher recognizes
  37. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  38. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  39. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  40. bool enableFl; // true if this matcher is enabled
  41. } cmLexMatcher;
  42. typedef struct cmLex_str
  43. {
  44. cmErr_t err;
  45. const cmChar_t* cp; // character buffer
  46. unsigned cn; // count of characters in buffer
  47. unsigned ci; // current buffer index position
  48. unsigned flags; // lexer control flags
  49. unsigned curTokenId; // type id of the current token
  50. unsigned curTokenCharIdx; // index into cp[] of the current token
  51. unsigned curTokenCharCnt; // count of characters in the current token
  52. unsigned curLine; // line number of the current token
  53. unsigned curCol; // column number of the current token
  54. unsigned nextLine;
  55. unsigned nextCol;
  56. cmChar_t* blockBegCmtStr;
  57. cmChar_t* blockEndCmtStr;
  58. cmChar_t* lineCmtStr;
  59. cmLexMatcher* mfp; // base of matcher array
  60. unsigned mfi; // next available matcher array slot
  61. unsigned mfn; // count of elementes in mfp[]
  62. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  63. } cmLex;
  64. cmLexH cmLexNullH = { NULL };
  65. bool _cmLexIsNewline( cmChar_t c )
  66. { return c == '\n'; }
  67. bool _cmLexIsCommentTypeId( unsigned typeId )
  68. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  69. cmLex* _cmLexHandleToPtr( cmLexH h )
  70. {
  71. cmLex* p = (cmLex*)h.h;
  72. assert(p != NULL);
  73. return p;
  74. };
  75. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  76. {
  77. va_list vl;
  78. va_start(vl,fmt);
  79. unsigned bufCharCnt = 512;
  80. char buf[ bufCharCnt+1 ];
  81. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  82. unsigned sn = strlen(buf);
  83. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  84. buf[bufCharCnt]=0;
  85. cmErrMsg(&p->err,rc,"%s",buf);
  86. va_end(vl);
  87. return rc;
  88. }
  89. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  90. {
  91. unsigned i = 0;
  92. unsigned n = strlen(keyStr);
  93. if( n <= cn )
  94. for(; i<=cn-n; ++i)
  95. if( strncmp(cp + i, keyStr, n ) == 0 )
  96. return i+n;
  97. return cmInvalidIdx;
  98. }
  99. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  100. {
  101. unsigned n = strlen(keyStr);
  102. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  103. }
  104. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  105. {
  106. unsigned i=0;
  107. for(; i<cn; ++i)
  108. if( !isspace(cp[i]) )
  109. break;
  110. return i;
  111. }
  112. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  113. {
  114. unsigned i = 0;
  115. unsigned n = 0; // decimal point counter
  116. unsigned d = 0; // digit counter
  117. bool fl = false; // true if this real includes an exponent
  118. for(; i<cn && n<=1; ++i)
  119. {
  120. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  121. continue;
  122. if( isdigit(cp[i]) ) // allow digits
  123. {
  124. ++d;
  125. continue;
  126. }
  127. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  128. ++n;
  129. else
  130. break;
  131. }
  132. // if there was at least one digit and the next char is an 'e'
  133. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  134. {
  135. d=0;
  136. ++i;
  137. unsigned j = i;
  138. for(; i<cn; ++i)
  139. {
  140. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  141. continue;
  142. if( isdigit(cp[i]) )
  143. {
  144. ++d;
  145. continue;
  146. }
  147. // stop at the first non-digit
  148. break;
  149. }
  150. // an exp exists if digits follwed the 'e'
  151. fl = d > 0;
  152. }
  153. return i>1 && (n==1 || fl) ? i : 0;
  154. }
  155. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  156. {
  157. unsigned i = 0;
  158. bool signFl = false;
  159. for(; i<cn; ++i)
  160. {
  161. if( i==0 && cp[i]=='-' )
  162. {
  163. signFl = true;
  164. continue;
  165. }
  166. if( !isdigit(cp[i]) )
  167. break;
  168. }
  169. // BUG BUG BUG
  170. // If an integer is specified using 'e' notiation
  171. // (see _cmLexRealMatcher()) and the number of exponent places
  172. // specified following the 'e' is positive and >= number of
  173. // digits following the decimal point (in effect zeros are
  174. // padded on the right side) then the value is an integer.
  175. //
  176. // The current implementation recognizes all numeric strings
  177. // containing a decimal point as reals.
  178. return signFl && i==1 ? 0 : i;
  179. }
  180. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  181. {
  182. unsigned i = 0;
  183. if( cn < 3 )
  184. return 0;
  185. if( cp[0]=='0' && cp[1]=='x')
  186. for(i=2; i<cn; ++i)
  187. if( !isxdigit(cp[i]) )
  188. break;
  189. return i;
  190. }
  191. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  192. {
  193. unsigned i = 0;
  194. if( isalpha(cp[0]) || (cp[0]== '_'))
  195. {
  196. i = 1;
  197. for(; i<cn; ++i)
  198. if( !isalnum(cp[i]) && (cp[i] != '_') )
  199. break;
  200. }
  201. return i;
  202. }
  203. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  204. {
  205. cmChar_t qStr[]="\"";
  206. unsigned n = strlen(qStr);
  207. if( strncmp(qStr,cp,n) == 0 )
  208. {
  209. unsigned i;
  210. if((i = _cmLexScanTo(cp+n, cn-n, qStr)) == cmInvalidIdx )
  211. {
  212. _cmLexError( p, kMissingEndQuoteLexRC, "Missing string end quote.");
  213. return 0;
  214. }
  215. return n+i;
  216. }
  217. return 0;
  218. }
  219. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  220. {
  221. unsigned n = strlen(p->blockBegCmtStr);
  222. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  223. {
  224. unsigned i;
  225. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  226. {
  227. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  228. return 0;
  229. }
  230. return n + i;
  231. }
  232. return 0;
  233. }
  234. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  235. {
  236. unsigned n = strlen(p->lineCmtStr);
  237. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  238. {
  239. unsigned i;
  240. const char newlineStr[] = "\n";
  241. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  242. {
  243. // no EOL was found so the comment must be on the last line of the source
  244. return cn;
  245. }
  246. return n + i;
  247. }
  248. return 0;
  249. }
  250. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  251. {
  252. assert( funcPtr==NULL || userPtr==NULL );
  253. assert( !(funcPtr==NULL && userPtr==NULL));
  254. // if there is no space in the user token array - then expand it
  255. if( p->mfi == p->mfn )
  256. {
  257. int incr_cnt = 10;
  258. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  259. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  260. cmMemPtrFree(&p->mfp);
  261. p->mfp = np;
  262. p->mfn += incr_cnt;
  263. }
  264. p->mfp[p->mfi].tokenStr = NULL;
  265. p->mfp[p->mfi].typeId = typeId;
  266. p->mfp[p->mfi].funcPtr = funcPtr;
  267. p->mfp[p->mfi].userPtr = userPtr;
  268. p->mfp[p->mfi].enableFl = true;
  269. if( keyStr != NULL )
  270. {
  271. // allocate space for the token string and store it
  272. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  273. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  274. }
  275. p->mfi++;
  276. return kOkLexRC;
  277. }
  278. cmRC_t _cmLexReset( cmLex* p )
  279. {
  280. p->ci = 0;
  281. p->curTokenId = kErrorLexTId;
  282. p->curTokenCharIdx = cmInvalidIdx;
  283. p->curTokenCharCnt = 0;
  284. p->curLine = 0;
  285. p->curCol = 0;
  286. p->nextLine = 0;
  287. p->nextCol = 0;
  288. cmErrClearRC(&p->err);
  289. return kOkLexRC;
  290. }
  291. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  292. {
  293. p->cp = cp;
  294. p->cn = cn;
  295. return _cmLexReset(p);
  296. }
  297. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  298. {
  299. cmLexH h;
  300. cmChar_t dfltLineCmt[] = "//";
  301. cmChar_t dfltBlockBegCmt[] = "/*";
  302. cmChar_t dfltBlockEndCmt[] = "*/";
  303. cmLex* p = cmMemAllocZ( cmLex, 1 );
  304. cmErrSetup(&p->err,rpt,"Lexer");
  305. p->flags = flags;
  306. _cmLexSetTextBuffer( p, cp, cn );
  307. /*
  308. p->cp = (cn==0) ? NULL : cp;
  309. p->cn = (cp==NULL) ? 0 : cn;
  310. p->ci = 0;
  311. p->curTokenId = kErrorLexTId;
  312. p->curTokenCharIdx = cmInvalidIdx;
  313. p->curTokenCharCnt = 0;
  314. p->curLine = 0;
  315. p->curCol = 0;
  316. p->nextLine = 0;
  317. p->nextCol = 0;
  318. */
  319. int init_mfn = 10;
  320. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  321. p->mfn = init_mfn;
  322. p->mfi = 0;
  323. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  324. strcpy( p->lineCmtStr, dfltLineCmt );
  325. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  326. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  327. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  328. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  329. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  330. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  331. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  332. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  333. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  334. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  335. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  336. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  337. h.h = p;
  338. _cmLexReset(p);
  339. return h;
  340. }
  341. cmRC_t cmLexFinal( cmLexH* hp )
  342. {
  343. if( hp == NULL || cmLexIsValid(*hp)==false )
  344. return cmOkRC;
  345. cmLex* p = _cmLexHandleToPtr(*hp);
  346. if( p != NULL )
  347. {
  348. if( p->mfp != NULL )
  349. {
  350. unsigned i = 0;
  351. // free the user token strings
  352. for(; i<p->mfi; ++i)
  353. if( p->mfp[i].tokenStr != NULL )
  354. cmMemPtrFree(&p->mfp[i].tokenStr);
  355. // free the matcher array
  356. cmMemPtrFree(&p->mfp);
  357. p->mfi = 0;
  358. p->mfn = 0;
  359. }
  360. cmMemPtrFree(&p->lineCmtStr);
  361. cmMemPtrFree(&p->blockBegCmtStr);
  362. cmMemPtrFree(&p->blockEndCmtStr);
  363. cmMemPtrFree(&p->textBuf);
  364. // free the lexer object
  365. cmMemPtrFree(&p);
  366. hp->h = NULL;
  367. }
  368. return kOkLexRC;
  369. }
  370. cmRC_t cmLexReset( cmLexH h )
  371. {
  372. cmLex* p = _cmLexHandleToPtr(h);
  373. return _cmLexReset(p);
  374. }
  375. bool cmLexIsValid( cmLexH h )
  376. { return h.h != NULL; }
  377. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  378. {
  379. cmLex* p = _cmLexHandleToPtr(h);
  380. return _cmLexSetTextBuffer(p,cp,cn);
  381. }
  382. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  383. {
  384. cmRC_t rc = kOkLexRC;
  385. cmFileH_t fh = cmFileNullHandle;
  386. cmLex* p = _cmLexHandleToPtr(h);
  387. long n = 0;
  388. assert( fn != NULL && p != NULL );
  389. // open the file
  390. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  391. return kFileOpenErrLexRC;
  392. // seek to the end of the file
  393. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  394. return kFileSeekErrLexRC;
  395. // get the length of the file
  396. if( cmFileTell(fh,&n) != kOkFileRC )
  397. return kFileTellErrLexRC;
  398. // rewind to the beginning of the file
  399. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  400. return kFileSeekErrLexRC;
  401. // allocate the text buffer
  402. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  403. {
  404. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  405. goto errLabel;
  406. }
  407. // read the file into the buffer
  408. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  409. return kFileReadErrLexRC;
  410. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  411. goto errLabel;
  412. errLabel:
  413. // close the file
  414. if( cmFileClose(&fh) != kOkFileRC )
  415. return kFileCloseErrLexRC;
  416. return rc;
  417. }
  418. /*
  419. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  420. {
  421. cmRC_t rc = kOkLexRC;
  422. FILE* fp = NULL;
  423. cmLex* p = _cmLexHandleToPtr(h);
  424. unsigned n = 0;
  425. assert( fn != NULL && p != NULL );
  426. // open the file
  427. if((fp = fopen(fn,"rb")) == NULL )
  428. return _cmLexError(p,kFileOpenErrLexRC,"Unable to open the file:'%s'.",fn);
  429. // seek to the end
  430. if( fseek(fp,0,SEEK_END) != 0 )
  431. {
  432. rc= _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the end of '%s'.",fn);
  433. goto errLabel;
  434. }
  435. // get the length of the file
  436. if( (n=ftell(fp)) == 0 )
  437. {
  438. rc = _cmLexError(p,kFileOpenErrLexRC,"The file '%s' appears to be empty.",fn);
  439. goto errLabel;
  440. }
  441. // rewind the file
  442. if( fseek(fp,0,SEEK_SET) != 0 )
  443. {
  444. rc = _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the beginning of '%s'.",fn);
  445. goto errLabel;
  446. }
  447. // allocate the text buffer
  448. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  449. {
  450. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  451. goto errLabel;
  452. }
  453. // read the file into the text buffer
  454. if( fread(p->textBuf,n,1,fp) != 1 )
  455. {
  456. rc = _cmLexError(p,kFileReadErrLexRC,"File read failed on:'%s'.",fn);
  457. goto errLabel;
  458. }
  459. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  460. goto errLabel;
  461. errLabel:
  462. // close the file
  463. if( fclose(fp) != 0 )
  464. {
  465. rc = _cmLexError(p,kFileCloseErrLexRC,"File close failed on:'%s'.",fn);
  466. goto errLabel;
  467. }
  468. return rc;
  469. }
  470. */
  471. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  472. {
  473. unsigned i = 0;
  474. for(; i<p->mfi; ++i)
  475. {
  476. if( id != cmInvalidId && p->mfp[i].typeId == id )
  477. return p->mfp + i;
  478. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  479. return p->mfp + i;
  480. }
  481. return NULL;
  482. }
  483. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  484. {
  485. cmLex* p = _cmLexHandleToPtr(h);
  486. // prevent duplicate tokens
  487. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  488. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  489. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  490. }
  491. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  492. {
  493. cmLex* p = _cmLexHandleToPtr(h);
  494. // prevent duplicate tokens
  495. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  496. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  497. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  498. }
  499. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  500. {
  501. cmLex* p = _cmLexHandleToPtr(h);
  502. unsigned mi = 0;
  503. for(; mi<p->mfi; ++mi)
  504. if( p->mfp[mi].typeId == id )
  505. {
  506. p->mfp[mi].enableFl = enableFl;
  507. return cmOkRC;
  508. }
  509. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  510. }
  511. unsigned cmLexFilterFlags( cmLexH h )
  512. {
  513. cmLex* p = _cmLexHandleToPtr(h);
  514. return p->flags;
  515. }
  516. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  517. {
  518. cmLex* p = _cmLexHandleToPtr(h);
  519. p->flags = flags;
  520. }
  521. unsigned cmLexGetNextToken( cmLexH h )
  522. {
  523. cmLex* p = _cmLexHandleToPtr(h);
  524. if( cmErrLastRC(&p->err) != kOkLexRC )
  525. return kErrorLexTId;
  526. while( p->ci < p->cn )
  527. {
  528. unsigned i;
  529. unsigned mi = 0;
  530. unsigned maxCharCnt = 0;
  531. unsigned maxIdx = cmInvalidIdx;
  532. p->curTokenId = kErrorLexTId;
  533. p->curTokenCharIdx = cmInvalidIdx;
  534. p->curTokenCharCnt = 0;
  535. // try each mater
  536. for(; mi<p->mfi; ++mi)
  537. if( p->mfp[mi].enableFl )
  538. {
  539. unsigned charCnt = 0;
  540. if( p->mfp[mi].funcPtr != NULL )
  541. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  542. else
  543. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  544. if( cmErrLastRC(&p->err) != kOkLexRC )
  545. return kErrorLexTId;
  546. // if this matched token is longer then the prev. matched token or
  547. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  548. if( (charCnt > maxCharCnt)
  549. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  550. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  551. )
  552. {
  553. maxCharCnt = charCnt;
  554. maxIdx = mi;
  555. }
  556. }
  557. // no token was matched
  558. if( maxIdx == cmInvalidIdx )
  559. {
  560. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  561. {
  562. maxCharCnt = 1;
  563. }
  564. else
  565. {
  566. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  567. return kErrorLexTId;
  568. }
  569. }
  570. // update the current line and column position
  571. p->curLine = p->nextLine;
  572. p->curCol = p->nextCol;
  573. // find the next column and line position
  574. for(i=0; i<maxCharCnt; ++i)
  575. {
  576. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  577. {
  578. p->nextLine++;
  579. p->nextCol = 1;
  580. }
  581. else
  582. p->nextCol++;
  583. }
  584. bool returnFl = true;
  585. if( maxIdx != cmInvalidIdx )
  586. {
  587. // check the space token filter
  588. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  589. returnFl = false;
  590. // check the comment token filter
  591. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  592. returnFl = false;
  593. }
  594. // update the lexer state
  595. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  596. p->curTokenCharIdx = p->ci;
  597. p->curTokenCharCnt = maxCharCnt;
  598. // advance the text buffer
  599. p->ci += maxCharCnt;
  600. if( returnFl )
  601. return p->curTokenId;
  602. }
  603. cmErrSetRC(&p->err,kEofRC);
  604. return kEofLexTId;
  605. }
  606. unsigned cmLexTokenId( cmLexH h )
  607. {
  608. cmLex* p = _cmLexHandleToPtr(h);
  609. return p->curTokenId;
  610. }
  611. const cmChar_t* cmLexTokenText( cmLexH h )
  612. {
  613. cmLex* p = _cmLexHandleToPtr(h);
  614. if( p->curTokenCharIdx == cmInvalidIdx )
  615. return NULL;
  616. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  617. return p->cp + p->curTokenCharIdx + n;
  618. }
  619. unsigned cmLexTokenCharCount( cmLexH h )
  620. {
  621. cmLex* p = _cmLexHandleToPtr(h);
  622. if( p->curTokenCharIdx == cmInvalidIdx )
  623. return 0;
  624. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  625. return p->curTokenCharCnt - n;
  626. }
  627. int cmLexTokenInt( cmLexH h )
  628. { return strtol( cmLexTokenText(h),NULL,0 ); }
  629. unsigned cmLexTokenUInt( cmLexH h )
  630. { return strtol( cmLexTokenText(h),NULL,0 ); }
  631. float cmLexTokenFloat( cmLexH h )
  632. { return strtof( cmLexTokenText(h),NULL ); }
  633. double cmLexTokenDouble( cmLexH h )
  634. { return strtod( cmLexTokenText(h),NULL ); }
  635. unsigned cmLexCurrentLineNumber( cmLexH h )
  636. {
  637. cmLex* p = _cmLexHandleToPtr(h);
  638. return p->curLine + 1;
  639. }
  640. unsigned cmLexCurrentColumnNumber( cmLexH h )
  641. {
  642. cmLex* p = _cmLexHandleToPtr(h);
  643. return p->curCol + 1;
  644. }
  645. unsigned cmLexErrorRC( cmLexH h )
  646. {
  647. cmLex* p = _cmLexHandleToPtr(h);
  648. return cmErrLastRC(&p->err);
  649. }
  650. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  651. {
  652. cmLex* p = _cmLexHandleToPtr(h);
  653. switch( typeId )
  654. {
  655. case kErrorLexTId: return "<error>";
  656. case kEofLexTId: return "<EOF>";
  657. case kSpaceLexTId: return "<space>";
  658. case kRealLexTId: return "<real>";
  659. case kIntLexTId: return "<int>";
  660. case kHexLexTId: return "<hex>";
  661. case kIdentLexTId: return "<ident>";
  662. case kQStrLexTId: return "<qstr>";
  663. case kBlockCmtLexTId: return "<bcmt>";
  664. case kLineCmtLexTId: return "<lcmt>";
  665. default:
  666. {
  667. cmLexMatcher* mp;
  668. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  669. return "<unknown>";
  670. return mp->tokenStr;
  671. }
  672. }
  673. return "<invalid>";
  674. }
  675. const cmChar_t* cmLexRcToMsg( unsigned rc )
  676. {
  677. unsigned i=0;
  678. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  679. if( cmLexErrorArray[i].code == rc )
  680. break;
  681. return cmLexErrorArray[i].msg;
  682. }
  683. //{ { label:cmLexEx }
  684. //(
  685. // cmLexTest() gives a simple cmLex example.
  686. //)
  687. //(
  688. void cmLexTest( cmRpt_t* rpt)
  689. {
  690. cmChar_t buf[] =
  691. "123ident0\n 123.456\nident0\n"
  692. "0xa12+.2\n"
  693. "// comment \n"
  694. "/* block \n"
  695. "comment */"
  696. "\"quoted string\""
  697. "ident1"
  698. "// last line comment";
  699. // initialize a lexer with a buffer of text
  700. cmLexH h = cmLexInit(buf,strlen(buf),
  701. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  702. // verify that the lexer initialization succeded.
  703. if( cmLexIsValid(h) == false )
  704. {
  705. cmRptPrintf(rpt,"Lexer initialization failed.");
  706. return;
  707. }
  708. // register some additional recoginizers
  709. cmLexRegisterToken(h,kUserLexTId+1,"+");
  710. cmLexRegisterToken(h,kUserLexTId+2,"-");
  711. unsigned tid;
  712. // ask for token id's
  713. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  714. {
  715. // print information about each token
  716. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  717. cmLexCurrentLineNumber(h),
  718. cmLexCurrentColumnNumber(h),
  719. cmLexIdToLabel(h,tid),
  720. cmLexTokenCharCount(h),
  721. cmLexTokenText(h) ,
  722. cmLexTokenCharCount(h));
  723. // if the token is a number ...
  724. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  725. {
  726. // ... then request the numbers value
  727. int iv = cmLexTokenInt(h);
  728. double dv = cmLexTokenDouble(h);
  729. cmRptPrintf(rpt,"%i %f",iv,dv);
  730. }
  731. cmRptPrintf(rpt,"\n");
  732. // handle errors
  733. if( tid == kErrorLexTId )
  734. {
  735. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  736. break;
  737. }
  738. }
  739. // finalize the lexer
  740. cmLexFinal(&h);
  741. }
  742. //)
  743. //}