libcm is a C development framework with an emphasis on audio signal processing applications.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. enum
  10. {
  11. kRealFloatLexFl = 0x01,
  12. kIntUnsignedLexFl = 0x02
  13. };
  14. typedef struct
  15. {
  16. unsigned code;
  17. const cmChar_t* msg;
  18. } cmLexErrorRecd;
  19. cmLexErrorRecd cmLexErrorArray[] =
  20. {
  21. { kOkLexRC, "No error. The operation completed successfully."},
  22. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  23. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  24. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  25. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  26. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  27. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  28. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  29. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  30. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  31. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  32. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  33. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  34. { kSignErrorLexRC, "A signed integer has a 'u' or 'U' suffix."},
  35. { kInvalidLexRC, "Unknown lexer error code." }
  36. };
  37. struct cmLex_str;
  38. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  39. // token match function record
  40. typedef struct
  41. {
  42. unsigned typeId; // token type this matcher recognizes
  43. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  44. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  45. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  46. bool enableFl; // true if this matcher is enabled
  47. } cmLexMatcher;
  48. typedef struct cmLex_str
  49. {
  50. cmErr_t err;
  51. const cmChar_t* cp; // character buffer
  52. unsigned cn; // count of characters in buffer
  53. unsigned ci; // current buffer index position
  54. unsigned flags; // lexer control flags
  55. unsigned curTokenId; // type id of the current token
  56. unsigned curTokenCharIdx; // index into cp[] of the current token
  57. unsigned curTokenCharCnt; // count of characters in the current token
  58. unsigned curLine; // line number of the current token
  59. unsigned curCol; // column number of the current token
  60. unsigned nextLine;
  61. unsigned nextCol;
  62. cmChar_t* blockBegCmtStr;
  63. cmChar_t* blockEndCmtStr;
  64. cmChar_t* lineCmtStr;
  65. cmLexMatcher* mfp; // base of matcher array
  66. unsigned mfi; // next available matcher array slot
  67. unsigned mfn; // count of elementes in mfp[]
  68. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  69. unsigned attrFlags; // used to store the int and real suffix type flags
  70. } cmLex;
  71. cmLexH cmLexNullH = { NULL };
  72. bool _cmLexIsNewline( cmChar_t c )
  73. { return c == '\n'; }
  74. bool _cmLexIsCommentTypeId( unsigned typeId )
  75. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  76. cmLex* _cmLexHandleToPtr( cmLexH h )
  77. {
  78. cmLex* p = (cmLex*)h.h;
  79. assert(p != NULL);
  80. return p;
  81. };
  82. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  83. {
  84. va_list vl;
  85. va_start(vl,fmt);
  86. unsigned bufCharCnt = 512;
  87. char buf[ bufCharCnt+1 ];
  88. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  89. unsigned sn = strlen(buf);
  90. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  91. buf[bufCharCnt]=0;
  92. cmErrMsg(&p->err,rc,"%s",buf);
  93. va_end(vl);
  94. return rc;
  95. }
  96. // Locate 'keyStr' in cp[cn] and return the index into cp[cn] of the character
  97. // following the last char in 'keyStr'. If keyStr is not found return cmInvalidIdx.
  98. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  99. {
  100. unsigned i = 0;
  101. unsigned n = strlen(keyStr);
  102. if( n <= cn )
  103. for(; i<=cn-n; ++i)
  104. if( strncmp(cp + i, keyStr, n ) == 0 )
  105. return i+n;
  106. return cmInvalidIdx;
  107. }
  108. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  109. {
  110. unsigned n = strlen(keyStr);
  111. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  112. }
  113. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  114. {
  115. unsigned i=0;
  116. for(; i<cn; ++i)
  117. if( !isspace(cp[i]) )
  118. break;
  119. return i;
  120. }
  121. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  122. {
  123. unsigned i = 0;
  124. unsigned n = 0; // decimal point counter
  125. unsigned d = 0; // digit counter
  126. bool fl = false; // expo flag
  127. for(; i<cn && n<=1; ++i)
  128. {
  129. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  130. continue;
  131. if( isdigit(cp[i]) ) // allow digits
  132. {
  133. ++d;
  134. continue;
  135. }
  136. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  137. ++n;
  138. else
  139. break;
  140. }
  141. // if there was at least one digit and the next char is an 'e'
  142. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  143. {
  144. unsigned e=0;
  145. ++i;
  146. unsigned j = i;
  147. fl = false;
  148. for(; i<cn; ++i)
  149. {
  150. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  151. continue;
  152. if( isdigit(cp[i]) )
  153. {
  154. ++e;
  155. ++d;
  156. continue;
  157. }
  158. // stop at the first non-digit
  159. break;
  160. }
  161. // an exp exists if digits follwed the 'e'
  162. fl = e > 0;
  163. }
  164. // if at least one digit was found
  165. if( d>0 )
  166. {
  167. // Note that this path allows a string w/o a decimal pt to trigger a match.
  168. if(i<cn)
  169. {
  170. // if the real has a suffix
  171. switch(cp[i])
  172. {
  173. case 'F':
  174. case 'f':
  175. p->attrFlags = cmSetFlag(p->attrFlags,kRealFloatLexFl);
  176. ++i;
  177. break;
  178. }
  179. }
  180. // match w/o suffix return
  181. if( d>0 && (fl || n==1 || cmIsFlag(p->attrFlags,kRealFloatLexFl)) )
  182. return i;
  183. }
  184. return 0; // no-match return
  185. }
  186. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  187. {
  188. unsigned i = 0;
  189. bool signFl = false;
  190. unsigned digitCnt = 0;
  191. for(; i<cn; ++i)
  192. {
  193. if( i==0 && cp[i]=='-' )
  194. {
  195. signFl = true;
  196. continue;
  197. }
  198. if( !isdigit(cp[i]) )
  199. break;
  200. ++digitCnt;
  201. }
  202. // BUG BUG BUG
  203. // If an integer is specified using 'e' notiation
  204. // (see _cmLexRealMatcher()) and the number of exponent places
  205. // specified following the 'e' is positive and >= number of
  206. // digits following the decimal point (in effect zeros are
  207. // padded on the right side) then the value is an integer.
  208. //
  209. // The current implementation recognizes all numeric strings
  210. // containing a decimal point as reals.
  211. // if no integer was found
  212. if( digitCnt==0)
  213. return 0;
  214. // check for suffix
  215. if(i<cn )
  216. {
  217. switch(cp[i])
  218. {
  219. case 'u':
  220. case 'U':
  221. if( signFl )
  222. _cmLexError(p,kSignErrorLexRC,"A signed integer has a 'u' or 'U' suffix.");
  223. else
  224. {
  225. p->attrFlags = cmSetFlag(p->attrFlags,kIntUnsignedLexFl);
  226. ++i;
  227. }
  228. break;
  229. default:
  230. break;
  231. }
  232. }
  233. return i;
  234. }
  235. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  236. {
  237. unsigned i = 0;
  238. if( cn < 3 )
  239. return 0;
  240. if( cp[0]=='0' && cp[1]=='x')
  241. for(i=2; i<cn; ++i)
  242. if( !isxdigit(cp[i]) )
  243. break;
  244. return i;
  245. }
  246. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  247. {
  248. unsigned i = 0;
  249. if( isalpha(cp[0]) || (cp[0]== '_'))
  250. {
  251. i = 1;
  252. for(; i<cn; ++i)
  253. if( !isalnum(cp[i]) && (cp[i] != '_') )
  254. break;
  255. }
  256. return i;
  257. }
  258. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  259. {
  260. bool escFl = false;
  261. unsigned i = 0;
  262. if( cp[i] != '"' )
  263. return 0;
  264. for(i=1; i<cn; ++i)
  265. {
  266. if( escFl )
  267. {
  268. escFl = false;
  269. continue;
  270. }
  271. if( cp[i] == '\\' )
  272. {
  273. escFl = true;
  274. continue;
  275. }
  276. if( cp[i] == '"' )
  277. return i+1;
  278. }
  279. return _cmLexError(p, kMissingEndQuoteLexRC, "Missing string literal end quote.");
  280. }
  281. unsigned _cmLexQCharMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  282. {
  283. unsigned i = 0;
  284. if( i >= cn || cp[i]!='\'' )
  285. return 0;
  286. i+=2;
  287. if( i >= cn || cp[i]!='\'')
  288. return 0;
  289. return 3;
  290. }
  291. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  292. {
  293. unsigned n = strlen(p->blockBegCmtStr);
  294. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  295. {
  296. unsigned i;
  297. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  298. {
  299. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  300. return 0;
  301. }
  302. return n + i;
  303. }
  304. return 0;
  305. }
  306. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  307. {
  308. unsigned n = strlen(p->lineCmtStr);
  309. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  310. {
  311. unsigned i;
  312. const char newlineStr[] = "\n";
  313. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  314. {
  315. // no EOL was found so the comment must be on the last line of the source
  316. return cn;
  317. }
  318. return n + i;
  319. }
  320. return 0;
  321. }
  322. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  323. {
  324. assert( funcPtr==NULL || userPtr==NULL );
  325. assert( !(funcPtr==NULL && userPtr==NULL));
  326. // if there is no space in the user token array - then expand it
  327. if( p->mfi == p->mfn )
  328. {
  329. int incr_cnt = 10;
  330. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  331. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  332. cmMemPtrFree(&p->mfp);
  333. p->mfp = np;
  334. p->mfn += incr_cnt;
  335. }
  336. p->mfp[p->mfi].tokenStr = NULL;
  337. p->mfp[p->mfi].typeId = typeId;
  338. p->mfp[p->mfi].funcPtr = funcPtr;
  339. p->mfp[p->mfi].userPtr = userPtr;
  340. p->mfp[p->mfi].enableFl = true;
  341. if( keyStr != NULL )
  342. {
  343. // allocate space for the token string and store it
  344. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  345. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  346. }
  347. p->mfi++;
  348. return kOkLexRC;
  349. }
  350. cmRC_t _cmLexReset( cmLex* p )
  351. {
  352. p->ci = 0;
  353. p->curTokenId = kErrorLexTId;
  354. p->curTokenCharIdx = cmInvalidIdx;
  355. p->curTokenCharCnt = 0;
  356. p->curLine = 0;
  357. p->curCol = 0;
  358. p->nextLine = 0;
  359. p->nextCol = 0;
  360. cmErrClearRC(&p->err);
  361. return kOkLexRC;
  362. }
  363. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  364. {
  365. p->cp = cp;
  366. p->cn = cn;
  367. return _cmLexReset(p);
  368. }
  369. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  370. {
  371. cmLexH h;
  372. cmChar_t dfltLineCmt[] = "//";
  373. cmChar_t dfltBlockBegCmt[] = "/*";
  374. cmChar_t dfltBlockEndCmt[] = "*/";
  375. cmLex* p = cmMemAllocZ( cmLex, 1 );
  376. cmErrSetup(&p->err,rpt,"Lexer");
  377. p->flags = flags;
  378. _cmLexSetTextBuffer( p, cp, cn );
  379. int init_mfn = 10;
  380. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  381. p->mfn = init_mfn;
  382. p->mfi = 0;
  383. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  384. strcpy( p->lineCmtStr, dfltLineCmt );
  385. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  386. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  387. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  388. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  389. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  390. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  391. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  392. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  393. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  394. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  395. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  396. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  397. if( cmIsFlag(flags,kReturnQCharLexFl) )
  398. _cmLexInstallMatcher( p, kQCharLexTId, _cmLexQCharMatcher, NULL, NULL );
  399. h.h = p;
  400. _cmLexReset(p);
  401. return h;
  402. }
  403. cmRC_t cmLexFinal( cmLexH* hp )
  404. {
  405. if( hp == NULL || cmLexIsValid(*hp)==false )
  406. return cmOkRC;
  407. cmLex* p = _cmLexHandleToPtr(*hp);
  408. if( p != NULL )
  409. {
  410. if( p->mfp != NULL )
  411. {
  412. unsigned i = 0;
  413. // free the user token strings
  414. for(; i<p->mfi; ++i)
  415. if( p->mfp[i].tokenStr != NULL )
  416. cmMemPtrFree(&p->mfp[i].tokenStr);
  417. // free the matcher array
  418. cmMemPtrFree(&p->mfp);
  419. p->mfi = 0;
  420. p->mfn = 0;
  421. }
  422. cmMemPtrFree(&p->lineCmtStr);
  423. cmMemPtrFree(&p->blockBegCmtStr);
  424. cmMemPtrFree(&p->blockEndCmtStr);
  425. cmMemPtrFree(&p->textBuf);
  426. // free the lexer object
  427. cmMemPtrFree(&p);
  428. hp->h = NULL;
  429. }
  430. return kOkLexRC;
  431. }
  432. cmRC_t cmLexReset( cmLexH h )
  433. {
  434. cmLex* p = _cmLexHandleToPtr(h);
  435. return _cmLexReset(p);
  436. }
  437. bool cmLexIsValid( cmLexH h )
  438. { return h.h != NULL; }
  439. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  440. {
  441. cmLex* p = _cmLexHandleToPtr(h);
  442. return _cmLexSetTextBuffer(p,cp,cn);
  443. }
  444. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  445. {
  446. cmRC_t rc = kOkLexRC;
  447. cmFileH_t fh = cmFileNullHandle;
  448. cmLex* p = _cmLexHandleToPtr(h);
  449. long n = 0;
  450. assert( fn != NULL && p != NULL );
  451. // open the file
  452. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  453. return kFileOpenErrLexRC;
  454. // seek to the end of the file
  455. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  456. return kFileSeekErrLexRC;
  457. // get the length of the file
  458. if( cmFileTell(fh,&n) != kOkFileRC )
  459. return kFileTellErrLexRC;
  460. // rewind to the beginning of the file
  461. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  462. return kFileSeekErrLexRC;
  463. // allocate the text buffer
  464. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  465. {
  466. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  467. goto errLabel;
  468. }
  469. // read the file into the buffer
  470. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  471. return kFileReadErrLexRC;
  472. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  473. goto errLabel;
  474. errLabel:
  475. // close the file
  476. if( cmFileClose(&fh) != kOkFileRC )
  477. return kFileCloseErrLexRC;
  478. return rc;
  479. }
  480. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  481. {
  482. unsigned i = 0;
  483. for(; i<p->mfi; ++i)
  484. {
  485. if( id != cmInvalidId && p->mfp[i].typeId == id )
  486. return p->mfp + i;
  487. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  488. return p->mfp + i;
  489. }
  490. return NULL;
  491. }
  492. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  493. {
  494. cmLex* p = _cmLexHandleToPtr(h);
  495. // prevent duplicate tokens
  496. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  497. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  498. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  499. }
  500. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  501. {
  502. cmLex* p = _cmLexHandleToPtr(h);
  503. // prevent duplicate tokens
  504. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  505. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  506. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  507. }
  508. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  509. {
  510. cmLex* p = _cmLexHandleToPtr(h);
  511. unsigned mi = 0;
  512. for(; mi<p->mfi; ++mi)
  513. if( p->mfp[mi].typeId == id )
  514. {
  515. p->mfp[mi].enableFl = enableFl;
  516. return cmOkRC;
  517. }
  518. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  519. }
  520. unsigned cmLexFilterFlags( cmLexH h )
  521. {
  522. cmLex* p = _cmLexHandleToPtr(h);
  523. return p->flags;
  524. }
  525. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  526. {
  527. cmLex* p = _cmLexHandleToPtr(h);
  528. p->flags = flags;
  529. }
  530. unsigned cmLexGetNextToken( cmLexH h )
  531. {
  532. cmLex* p = _cmLexHandleToPtr(h);
  533. if( cmErrLastRC(&p->err) != kOkLexRC )
  534. return kErrorLexTId;
  535. while( p->ci < p->cn )
  536. {
  537. unsigned i;
  538. unsigned mi = 0;
  539. unsigned maxCharCnt = 0;
  540. unsigned maxIdx = cmInvalidIdx;
  541. p->curTokenId = kErrorLexTId;
  542. p->curTokenCharIdx = cmInvalidIdx;
  543. p->curTokenCharCnt = 0;
  544. p->attrFlags = 0;
  545. // try each matcher
  546. for(; mi<p->mfi; ++mi)
  547. if( p->mfp[mi].enableFl )
  548. {
  549. unsigned charCnt = 0;
  550. if( p->mfp[mi].funcPtr != NULL )
  551. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  552. else
  553. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  554. // notice if the matcher set the error code
  555. if( cmErrLastRC(&p->err) != kOkLexRC )
  556. return kErrorLexTId;
  557. // if this matched token is longer then the prev. matched token or
  558. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  559. if( (charCnt > maxCharCnt)
  560. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  561. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  562. )
  563. {
  564. maxCharCnt = charCnt;
  565. maxIdx = mi;
  566. }
  567. }
  568. // no token was matched
  569. if( maxIdx == cmInvalidIdx )
  570. {
  571. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  572. {
  573. maxCharCnt = 1;
  574. }
  575. else
  576. {
  577. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  578. return kErrorLexTId;
  579. }
  580. }
  581. // update the current line and column position
  582. p->curLine = p->nextLine;
  583. p->curCol = p->nextCol;
  584. // find the next column and line position
  585. for(i=0; i<maxCharCnt; ++i)
  586. {
  587. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  588. {
  589. p->nextLine++;
  590. p->nextCol = 1;
  591. }
  592. else
  593. p->nextCol++;
  594. }
  595. bool returnFl = true;
  596. if( maxIdx != cmInvalidIdx )
  597. {
  598. // check the space token filter
  599. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  600. returnFl = false;
  601. // check the comment token filter
  602. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  603. returnFl = false;
  604. }
  605. // update the lexer state
  606. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  607. p->curTokenCharIdx = p->ci;
  608. p->curTokenCharCnt = maxCharCnt;
  609. // advance the text buffer
  610. p->ci += maxCharCnt;
  611. if( returnFl )
  612. return p->curTokenId;
  613. }
  614. cmErrSetRC(&p->err,kEofRC);
  615. return kEofLexTId;
  616. }
  617. unsigned cmLexTokenId( cmLexH h )
  618. {
  619. cmLex* p = _cmLexHandleToPtr(h);
  620. return p->curTokenId;
  621. }
  622. const cmChar_t* cmLexTokenText( cmLexH h )
  623. {
  624. cmLex* p = _cmLexHandleToPtr(h);
  625. if( p->curTokenCharIdx == cmInvalidIdx )
  626. return NULL;
  627. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  628. return p->cp + p->curTokenCharIdx + n;
  629. }
  630. unsigned cmLexTokenCharCount( cmLexH h )
  631. {
  632. cmLex* p = _cmLexHandleToPtr(h);
  633. if( p->curTokenCharIdx == cmInvalidIdx )
  634. return 0;
  635. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  636. return p->curTokenCharCnt - n;
  637. }
  638. int cmLexTokenInt( cmLexH h )
  639. { return strtol( cmLexTokenText(h),NULL,0 ); }
  640. unsigned cmLexTokenUInt( cmLexH h )
  641. { return strtol( cmLexTokenText(h),NULL,0 ); }
  642. float cmLexTokenFloat( cmLexH h )
  643. { return strtof( cmLexTokenText(h),NULL ); }
  644. double cmLexTokenDouble( cmLexH h )
  645. { return strtod( cmLexTokenText(h),NULL ); }
  646. bool cmLexTokenIsUnsigned( cmLexH h )
  647. {
  648. cmLex* p = _cmLexHandleToPtr(h);
  649. return p->curTokenId == kIntLexTId && cmIsFlag(p->attrFlags,kIntUnsignedLexFl);
  650. }
  651. bool cmLexTokenIsSinglePrecision( cmLexH h )
  652. {
  653. cmLex* p = _cmLexHandleToPtr(h);
  654. return p->curTokenId == kRealLexTId && cmIsFlag(p->attrFlags,kRealFloatLexFl);
  655. }
  656. unsigned cmLexCurrentLineNumber( cmLexH h )
  657. {
  658. cmLex* p = _cmLexHandleToPtr(h);
  659. return p->curLine + 1;
  660. }
  661. unsigned cmLexCurrentColumnNumber( cmLexH h )
  662. {
  663. cmLex* p = _cmLexHandleToPtr(h);
  664. return p->curCol + 1;
  665. }
  666. unsigned cmLexErrorRC( cmLexH h )
  667. {
  668. cmLex* p = _cmLexHandleToPtr(h);
  669. return cmErrLastRC(&p->err);
  670. }
  671. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  672. {
  673. cmLex* p = _cmLexHandleToPtr(h);
  674. switch( typeId )
  675. {
  676. case kErrorLexTId: return "<error>";
  677. case kEofLexTId: return "<EOF>";
  678. case kSpaceLexTId: return "<space>";
  679. case kRealLexTId: return "<real>";
  680. case kIntLexTId: return "<int>";
  681. case kHexLexTId: return "<hex>";
  682. case kIdentLexTId: return "<ident>";
  683. case kQStrLexTId: return "<qstr>";
  684. case kBlockCmtLexTId: return "<bcmt>";
  685. case kLineCmtLexTId: return "<lcmt>";
  686. default:
  687. {
  688. cmLexMatcher* mp;
  689. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  690. return "<unknown>";
  691. return mp->tokenStr;
  692. }
  693. }
  694. return "<invalid>";
  695. }
  696. const cmChar_t* cmLexRcToMsg( unsigned rc )
  697. {
  698. unsigned i=0;
  699. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  700. if( cmLexErrorArray[i].code == rc )
  701. break;
  702. return cmLexErrorArray[i].msg;
  703. }
  704. //{ { label:cmLexEx }
  705. //(
  706. // cmLexTest() gives a simple cmLex example.
  707. //)
  708. //(
  709. void cmLexTest( cmRpt_t* rpt)
  710. {
  711. cmChar_t buf[] =
  712. "123ident0\n 123.456\nident0\n"
  713. "0xa12+.2\n"
  714. "// comment \n"
  715. "/* block \n"
  716. "comment */"
  717. "\"quoted string\""
  718. "ident1"
  719. "// last line comment";
  720. // initialize a lexer with a buffer of text
  721. cmLexH h = cmLexInit(buf,strlen(buf),
  722. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  723. // verify that the lexer initialization succeded.
  724. if( cmLexIsValid(h) == false )
  725. {
  726. cmRptPrintf(rpt,"Lexer initialization failed.");
  727. return;
  728. }
  729. // register some additional recoginizers
  730. cmLexRegisterToken(h,kUserLexTId+1,"+");
  731. cmLexRegisterToken(h,kUserLexTId+2,"-");
  732. unsigned tid;
  733. // ask for token id's
  734. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  735. {
  736. // print information about each token
  737. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  738. cmLexCurrentLineNumber(h),
  739. cmLexCurrentColumnNumber(h),
  740. cmLexIdToLabel(h,tid),
  741. cmLexTokenCharCount(h),
  742. cmLexTokenText(h) ,
  743. cmLexTokenCharCount(h));
  744. // if the token is a number ...
  745. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  746. {
  747. // ... then request the numbers value
  748. int iv = cmLexTokenInt(h);
  749. double dv = cmLexTokenDouble(h);
  750. cmRptPrintf(rpt,"%i %f",iv,dv);
  751. }
  752. cmRptPrintf(rpt,"\n");
  753. // handle errors
  754. if( tid == kErrorLexTId )
  755. {
  756. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  757. break;
  758. }
  759. }
  760. // finalize the lexer
  761. cmLexFinal(&h);
  762. }
  763. //)
  764. //}