libcm is a C development framework with an emphasis on audio signal processing applications.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. typedef struct
  10. {
  11. unsigned code;
  12. const cmChar_t* msg;
  13. } cmLexErrorRecd;
  14. cmLexErrorRecd cmLexErrorArray[] =
  15. {
  16. { kOkLexRC, "No error. The operation completed successfully."},
  17. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  18. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  19. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  20. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  21. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  22. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  23. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  24. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  25. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  26. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  27. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  28. { kInvalidLexRC, "Unknown lexer error code." }
  29. };
  30. struct cmLex_str;
  31. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  32. // token match function record
  33. typedef struct
  34. {
  35. unsigned typeId; // token type this matcher recognizes
  36. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  37. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  38. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  39. } cmLexMatcher;
  40. typedef struct cmLex_str
  41. {
  42. cmErr_t err;
  43. const cmChar_t* cp; // character buffer
  44. unsigned cn; // count of characters in buffer
  45. unsigned ci; // current buffer index position
  46. unsigned flags; // lexer control flags
  47. unsigned curTokenId; // type id of the current token
  48. unsigned curTokenCharIdx; // index into cp[] of the current token
  49. unsigned curTokenCharCnt; // count of characters in the current token
  50. unsigned curLine; // line number of the current token
  51. unsigned curCol; // column number of the current token
  52. unsigned nextLine;
  53. unsigned nextCol;
  54. cmChar_t* blockBegCmtStr;
  55. cmChar_t* blockEndCmtStr;
  56. cmChar_t* lineCmtStr;
  57. cmLexMatcher* mfp; // base of matcher array
  58. unsigned mfi; // next available matcher array slot
  59. unsigned mfn; // count of elementes in mfp[]
  60. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  61. } cmLex;
  62. cmLexH cmLexNullH = { NULL };
  63. bool _cmLexIsNewline( cmChar_t c )
  64. { return c == '\n'; }
  65. bool _cmLexIsCommentTypeId( unsigned typeId )
  66. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  67. cmLex* _cmLexHandleToPtr( cmLexH h )
  68. {
  69. cmLex* p = (cmLex*)h.h;
  70. assert(p != NULL);
  71. return p;
  72. };
  73. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  74. {
  75. va_list vl;
  76. va_start(vl,fmt);
  77. unsigned bufCharCnt = 512;
  78. char buf[ bufCharCnt+1 ];
  79. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  80. unsigned sn = strlen(buf);
  81. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  82. buf[bufCharCnt]=0;
  83. cmErrMsg(&p->err,rc,"%s",buf);
  84. va_end(vl);
  85. return rc;
  86. }
  87. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  88. {
  89. unsigned i = 0;
  90. unsigned n = strlen(keyStr);
  91. if( n <= cn )
  92. for(; i<=cn-n; ++i)
  93. if( strncmp(cp + i, keyStr, n ) == 0 )
  94. return i+n;
  95. return cmInvalidIdx;
  96. }
  97. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  98. {
  99. unsigned n = strlen(keyStr);
  100. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  101. }
  102. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  103. {
  104. unsigned i=0;
  105. for(; i<cn; ++i)
  106. if( !isspace(cp[i]) )
  107. break;
  108. return i;
  109. }
  110. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  111. {
  112. unsigned i = 0;
  113. unsigned n = 0; // decimal point counter
  114. unsigned d = 0; // digit counter
  115. bool fl = false; // true if this real includes an exponent
  116. for(; i<cn && n<=1; ++i)
  117. {
  118. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  119. continue;
  120. if( isdigit(cp[i]) ) // allow digits
  121. {
  122. ++d;
  123. continue;
  124. }
  125. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  126. ++n;
  127. else
  128. break;
  129. }
  130. // if there was at least one digit and the next char is an 'e'
  131. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  132. {
  133. d=0;
  134. ++i;
  135. unsigned j = i;
  136. for(; i<cn; ++i)
  137. {
  138. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  139. continue;
  140. if( isdigit(cp[i]) )
  141. {
  142. ++d;
  143. continue;
  144. }
  145. // stop at the first non-digit
  146. break;
  147. }
  148. // an exp exists if digits follwed the 'e'
  149. fl = d > 0;
  150. }
  151. return i>1 && (n==1 || fl) ? i : 0;
  152. }
  153. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  154. {
  155. unsigned i = 0;
  156. for(; i<cn; ++i)
  157. {
  158. if( i==0 && cp[i]=='-' )
  159. continue;
  160. if( !isdigit(cp[i]) )
  161. break;
  162. }
  163. // BUG BUG BUG
  164. // If an integer is specified using 'e' notiation
  165. // (see _cmLexRealMatcher()) and the number of exponent places
  166. // specified following the 'e' is positive and >= number of
  167. // digits following the decimal point (in effect zeros are
  168. // padded on the right side) then the value is an integer.
  169. //
  170. // The current implementation recognizes all numeric strings
  171. // containing a decimal point as reals.
  172. return i;
  173. }
  174. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  175. {
  176. unsigned i = 0;
  177. if( cn < 3 )
  178. return 0;
  179. if( cp[0]=='0' && cp[1]=='x')
  180. for(i=2; i<cn; ++i)
  181. if( !isxdigit(cp[i]) )
  182. break;
  183. return i;
  184. }
  185. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  186. {
  187. unsigned i = 0;
  188. if( isalpha(cp[0]) || (cp[0]== '_'))
  189. {
  190. i = 1;
  191. for(; i<cn; ++i)
  192. if( !isalnum(cp[i]) && (cp[i] != '_') )
  193. break;
  194. }
  195. return i;
  196. }
  197. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  198. {
  199. cmChar_t qStr[]="\"";
  200. unsigned n = strlen(qStr);
  201. if( strncmp(qStr,cp,n) == 0 )
  202. {
  203. unsigned i;
  204. if((i = _cmLexScanTo(cp+n, cn-n, qStr)) == cmInvalidIdx )
  205. {
  206. _cmLexError( p, kMissingEndQuoteLexRC, "Missing string end quote.");
  207. return 0;
  208. }
  209. return n+i;
  210. }
  211. return 0;
  212. }
  213. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  214. {
  215. unsigned n = strlen(p->blockBegCmtStr);
  216. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  217. {
  218. unsigned i;
  219. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  220. {
  221. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  222. return 0;
  223. }
  224. return n + i;
  225. }
  226. return 0;
  227. }
  228. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  229. {
  230. unsigned n = strlen(p->lineCmtStr);
  231. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  232. {
  233. unsigned i;
  234. const char newlineStr[] = "\n";
  235. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  236. {
  237. // no EOL was found so the comment must be on the last line of the source
  238. return cn;
  239. }
  240. return n + i;
  241. }
  242. return 0;
  243. }
  244. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  245. {
  246. assert( funcPtr==NULL || userPtr==NULL );
  247. assert( !(funcPtr==NULL && userPtr==NULL));
  248. // if there is no space in the user token array - then expand it
  249. if( p->mfi == p->mfn )
  250. {
  251. int incr_cnt = 10;
  252. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  253. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  254. cmMemPtrFree(&p->mfp);
  255. p->mfp = np;
  256. p->mfn += incr_cnt;
  257. }
  258. p->mfp[p->mfi].tokenStr = NULL;
  259. p->mfp[p->mfi].typeId = typeId;
  260. p->mfp[p->mfi].funcPtr = funcPtr;
  261. p->mfp[p->mfi].userPtr = userPtr;
  262. if( keyStr != NULL )
  263. {
  264. // allocate space for the token string and store it
  265. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  266. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  267. }
  268. p->mfi++;
  269. return kOkLexRC;
  270. }
  271. cmRC_t _cmLexReset( cmLex* p )
  272. {
  273. p->ci = 0;
  274. p->curTokenId = kErrorLexTId;
  275. p->curTokenCharIdx = cmInvalidIdx;
  276. p->curTokenCharCnt = 0;
  277. p->curLine = 0;
  278. p->curCol = 0;
  279. p->nextLine = 0;
  280. p->nextCol = 0;
  281. cmErrClearRC(&p->err);
  282. return kOkLexRC;
  283. }
  284. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  285. {
  286. p->cp = cp;
  287. p->cn = cn;
  288. return _cmLexReset(p);
  289. }
  290. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  291. {
  292. cmLexH h;
  293. cmChar_t dfltLineCmt[] = "//";
  294. cmChar_t dfltBlockBegCmt[] = "/*";
  295. cmChar_t dfltBlockEndCmt[] = "*/";
  296. cmLex* p = cmMemAllocZ( cmLex, 1 );
  297. cmErrSetup(&p->err,rpt,"Lexer");
  298. p->flags = flags;
  299. _cmLexSetTextBuffer( p, cp, cn );
  300. /*
  301. p->cp = (cn==0) ? NULL : cp;
  302. p->cn = (cp==NULL) ? 0 : cn;
  303. p->ci = 0;
  304. p->curTokenId = kErrorLexTId;
  305. p->curTokenCharIdx = cmInvalidIdx;
  306. p->curTokenCharCnt = 0;
  307. p->curLine = 0;
  308. p->curCol = 0;
  309. p->nextLine = 0;
  310. p->nextCol = 0;
  311. */
  312. int init_mfn = 10;
  313. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  314. p->mfn = init_mfn;
  315. p->mfi = 0;
  316. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  317. strcpy( p->lineCmtStr, dfltLineCmt );
  318. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  319. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  320. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  321. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  322. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  323. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  324. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  325. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  326. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  327. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  328. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  329. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  330. h.h = p;
  331. _cmLexReset(p);
  332. return h;
  333. }
  334. cmRC_t cmLexFinal( cmLexH* hp )
  335. {
  336. if( hp == NULL || cmLexIsValid(*hp)==false )
  337. return cmOkRC;
  338. cmLex* p = _cmLexHandleToPtr(*hp);
  339. if( p != NULL )
  340. {
  341. if( p->mfp != NULL )
  342. {
  343. unsigned i = 0;
  344. // free the user token strings
  345. for(; i<p->mfi; ++i)
  346. if( p->mfp[i].tokenStr != NULL )
  347. cmMemPtrFree(&p->mfp[i].tokenStr);
  348. // free the matcher array
  349. cmMemPtrFree(&p->mfp);
  350. p->mfi = 0;
  351. p->mfn = 0;
  352. }
  353. cmMemPtrFree(&p->lineCmtStr);
  354. cmMemPtrFree(&p->blockBegCmtStr);
  355. cmMemPtrFree(&p->blockEndCmtStr);
  356. cmMemPtrFree(&p->textBuf);
  357. // free the lexer object
  358. cmMemPtrFree(&p);
  359. hp->h = NULL;
  360. }
  361. return kOkLexRC;
  362. }
  363. cmRC_t cmLexReset( cmLexH h )
  364. {
  365. cmLex* p = _cmLexHandleToPtr(h);
  366. return _cmLexReset(p);
  367. }
  368. bool cmLexIsValid( cmLexH h )
  369. { return h.h != NULL; }
  370. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  371. {
  372. cmLex* p = _cmLexHandleToPtr(h);
  373. return _cmLexSetTextBuffer(p,cp,cn);
  374. }
  375. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  376. {
  377. cmRC_t rc = kOkLexRC;
  378. cmFileH_t fh = cmFileNullHandle;
  379. cmLex* p = _cmLexHandleToPtr(h);
  380. long n = 0;
  381. assert( fn != NULL && p != NULL );
  382. // open the file
  383. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  384. return kFileOpenErrLexRC;
  385. // seek to the end of the file
  386. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  387. return kFileSeekErrLexRC;
  388. // get the length of the file
  389. if( cmFileTell(fh,&n) != kOkFileRC )
  390. return kFileTellErrLexRC;
  391. // rewind to the beginning of the file
  392. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  393. return kFileSeekErrLexRC;
  394. // allocate the text buffer
  395. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  396. {
  397. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  398. goto errLabel;
  399. }
  400. // read the file into the buffer
  401. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  402. return kFileReadErrLexRC;
  403. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  404. goto errLabel;
  405. errLabel:
  406. // close the file
  407. if( cmFileClose(&fh) != kOkFileRC )
  408. return kFileCloseErrLexRC;
  409. return rc;
  410. }
  411. /*
  412. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  413. {
  414. cmRC_t rc = kOkLexRC;
  415. FILE* fp = NULL;
  416. cmLex* p = _cmLexHandleToPtr(h);
  417. unsigned n = 0;
  418. assert( fn != NULL && p != NULL );
  419. // open the file
  420. if((fp = fopen(fn,"rb")) == NULL )
  421. return _cmLexError(p,kFileOpenErrLexRC,"Unable to open the file:'%s'.",fn);
  422. // seek to the end
  423. if( fseek(fp,0,SEEK_END) != 0 )
  424. {
  425. rc= _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the end of '%s'.",fn);
  426. goto errLabel;
  427. }
  428. // get the length of the file
  429. if( (n=ftell(fp)) == 0 )
  430. {
  431. rc = _cmLexError(p,kFileOpenErrLexRC,"The file '%s' appears to be empty.",fn);
  432. goto errLabel;
  433. }
  434. // rewind the file
  435. if( fseek(fp,0,SEEK_SET) != 0 )
  436. {
  437. rc = _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the beginning of '%s'.",fn);
  438. goto errLabel;
  439. }
  440. // allocate the text buffer
  441. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  442. {
  443. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  444. goto errLabel;
  445. }
  446. // read the file into the text buffer
  447. if( fread(p->textBuf,n,1,fp) != 1 )
  448. {
  449. rc = _cmLexError(p,kFileReadErrLexRC,"File read failed on:'%s'.",fn);
  450. goto errLabel;
  451. }
  452. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  453. goto errLabel;
  454. errLabel:
  455. // close the file
  456. if( fclose(fp) != 0 )
  457. {
  458. rc = _cmLexError(p,kFileCloseErrLexRC,"File close failed on:'%s'.",fn);
  459. goto errLabel;
  460. }
  461. return rc;
  462. }
  463. */
  464. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  465. {
  466. unsigned i = 0;
  467. for(; i<p->mfi; ++i)
  468. {
  469. if( id != cmInvalidId && p->mfp[i].typeId == id )
  470. return p->mfp + i;
  471. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  472. return p->mfp + i;
  473. }
  474. return NULL;
  475. }
  476. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  477. {
  478. cmLex* p = _cmLexHandleToPtr(h);
  479. // prevent duplicate tokens
  480. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  481. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  482. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  483. }
  484. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  485. {
  486. cmLex* p = _cmLexHandleToPtr(h);
  487. // prevent duplicate tokens
  488. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  489. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  490. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  491. }
  492. unsigned cmLexGetNextToken( cmLexH h )
  493. {
  494. cmLex* p = _cmLexHandleToPtr(h);
  495. if( cmErrLastRC(&p->err) != kOkLexRC )
  496. return kErrorLexTId;
  497. while( p->ci < p->cn )
  498. {
  499. unsigned i;
  500. unsigned mi = 0;
  501. unsigned maxCharCnt = 0;
  502. unsigned maxIdx = cmInvalidIdx;
  503. p->curTokenId = kErrorLexTId;
  504. p->curTokenCharIdx = cmInvalidIdx;
  505. p->curTokenCharCnt = 0;
  506. for(; mi<p->mfi; ++mi)
  507. {
  508. unsigned charCnt = 0;
  509. if( p->mfp[mi].funcPtr != NULL )
  510. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  511. else
  512. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  513. if( cmErrLastRC(&p->err) != kOkLexRC )
  514. return kErrorLexTId;
  515. // if this matched token is longer then the prev. matched token or
  516. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  517. if( (charCnt > maxCharCnt) || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId ) )
  518. {
  519. maxCharCnt = charCnt;
  520. maxIdx = mi;
  521. }
  522. }
  523. // no token was matched
  524. if( maxIdx == cmInvalidIdx )
  525. {
  526. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  527. return kErrorLexTId;
  528. }
  529. // update the current line and column position
  530. p->curLine = p->nextLine;
  531. p->curCol = p->nextCol;
  532. // find the next column and line position
  533. for(i=0; i<maxCharCnt; ++i)
  534. {
  535. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  536. {
  537. p->nextLine++;
  538. p->nextCol = 1;
  539. }
  540. else
  541. p->nextCol++;
  542. }
  543. bool returnFl = true;
  544. // check the space token filter
  545. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  546. returnFl = false;
  547. // check the comment token filter
  548. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  549. returnFl = false;
  550. // update the lexer state
  551. p->curTokenId = p->mfp[ maxIdx ].typeId;
  552. p->curTokenCharIdx = p->ci;
  553. p->curTokenCharCnt = maxCharCnt;
  554. // advance the text buffer
  555. p->ci += maxCharCnt;
  556. if( returnFl )
  557. return p->curTokenId;
  558. }
  559. cmErrSetRC(&p->err,kEofRC);
  560. return kEofLexTId;
  561. }
  562. unsigned cmLexTokenId( cmLexH h )
  563. {
  564. cmLex* p = _cmLexHandleToPtr(h);
  565. return p->curTokenId;
  566. }
  567. const cmChar_t* cmLexTokenText( cmLexH h )
  568. {
  569. cmLex* p = _cmLexHandleToPtr(h);
  570. if( p->curTokenCharIdx == cmInvalidIdx )
  571. return NULL;
  572. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  573. return p->cp + p->curTokenCharIdx + n;
  574. }
  575. unsigned cmLexTokenCharCount( cmLexH h )
  576. {
  577. cmLex* p = _cmLexHandleToPtr(h);
  578. if( p->curTokenCharIdx == cmInvalidIdx )
  579. return 0;
  580. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  581. return p->curTokenCharCnt - n;
  582. }
  583. int cmLexTokenInt( cmLexH h )
  584. { return strtol( cmLexTokenText(h),NULL,0 ); }
  585. unsigned cmLexTokenUInt( cmLexH h )
  586. { return strtol( cmLexTokenText(h),NULL,0 ); }
  587. float cmLexTokenFloat( cmLexH h )
  588. { return strtof( cmLexTokenText(h),NULL ); }
  589. double cmLexTokenDouble( cmLexH h )
  590. { return strtod( cmLexTokenText(h),NULL ); }
  591. unsigned cmLexCurrentLineNumber( cmLexH h )
  592. {
  593. cmLex* p = _cmLexHandleToPtr(h);
  594. return p->curLine + 1;
  595. }
  596. unsigned cmLexCurrentColumnNumber( cmLexH h )
  597. {
  598. cmLex* p = _cmLexHandleToPtr(h);
  599. return p->curCol + 1;
  600. }
  601. unsigned cmLexErrorRC( cmLexH h )
  602. {
  603. cmLex* p = _cmLexHandleToPtr(h);
  604. return cmErrLastRC(&p->err);
  605. }
  606. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  607. {
  608. cmLex* p = _cmLexHandleToPtr(h);
  609. switch( typeId )
  610. {
  611. case kErrorLexTId: return "<error>";
  612. case kEofLexTId: return "<EOF>";
  613. case kSpaceLexTId: return "<space>";
  614. case kRealLexTId: return "<real>";
  615. case kIntLexTId: return "<int>";
  616. case kHexLexTId: return "<hex>";
  617. case kIdentLexTId: return "<ident>";
  618. case kQStrLexTId: return "<qstr>";
  619. case kBlockCmtLexTId: return "<bcmt>";
  620. case kLineCmtLexTId: return "<lcmt>";
  621. default:
  622. {
  623. cmLexMatcher* mp;
  624. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  625. return "<unknown>";
  626. return mp->tokenStr;
  627. }
  628. }
  629. return "<invalid>";
  630. }
  631. const cmChar_t* cmLexRcToMsg( unsigned rc )
  632. {
  633. unsigned i=0;
  634. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  635. if( cmLexErrorArray[i].code == rc )
  636. break;
  637. return cmLexErrorArray[i].msg;
  638. }
  639. //{ { label:cmLexEx }
  640. //(
  641. // cmLexTest() gives a simple cmLex example.
  642. //)
  643. //(
  644. void cmLexTest( cmRpt_t* rpt)
  645. {
  646. cmChar_t buf[] =
  647. "123ident0\n 123.456\nident0\n"
  648. "0xa12+.2\n"
  649. "// comment \n"
  650. "/* block \n"
  651. "comment */"
  652. "\"quoted string\""
  653. "ident1"
  654. "// last line comment";
  655. // initialize a lexer with a buffer of text
  656. cmLexH h = cmLexInit(buf,strlen(buf),
  657. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  658. // verify that the lexer initialization succeded.
  659. if( cmLexIsValid(h) == false )
  660. {
  661. cmRptPrintf(rpt,"Lexer initialization failed.");
  662. return;
  663. }
  664. // register some additional recoginizers
  665. cmLexRegisterToken(h,kUserLexTId+1,"+");
  666. cmLexRegisterToken(h,kUserLexTId+2,"-");
  667. unsigned tid;
  668. // ask for token id's
  669. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  670. {
  671. // print information about each token
  672. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  673. cmLexCurrentLineNumber(h),
  674. cmLexCurrentColumnNumber(h),
  675. cmLexIdToLabel(h,tid),
  676. cmLexTokenCharCount(h),
  677. cmLexTokenText(h) ,
  678. cmLexTokenCharCount(h));
  679. // if the token is a number ...
  680. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  681. {
  682. // ... then request the numbers value
  683. int iv = cmLexTokenInt(h);
  684. double dv = cmLexTokenDouble(h);
  685. cmRptPrintf(rpt,"%i %f",iv,dv);
  686. }
  687. cmRptPrintf(rpt,"\n");
  688. // handle errors
  689. if( tid == kErrorLexTId )
  690. {
  691. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  692. break;
  693. }
  694. }
  695. // finalize the lexer
  696. cmLexFinal(&h);
  697. }
  698. //)
  699. //}