libcm is a C development framework with an emphasis on audio signal processing applications.
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

cmLex.c 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. enum
  10. {
  11. kRealFloatLexFl = 0x01,
  12. kIntUnsignedLexFl = 0x02
  13. };
  14. typedef struct
  15. {
  16. unsigned code;
  17. const cmChar_t* msg;
  18. } cmLexErrorRecd;
  19. cmLexErrorRecd cmLexErrorArray[] =
  20. {
  21. { kOkLexRC, "No error. The operation completed successfully."},
  22. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  23. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  24. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  25. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  26. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  27. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  28. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  29. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  30. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  31. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  32. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  33. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  34. { kSignErrorLexRC, "A signed integer has a 'u' or 'U' suffix."},
  35. { kInvalidLexRC, "Unknown lexer error code." }
  36. };
  37. struct cmLex_str;
  38. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  39. // token match function record
  40. typedef struct
  41. {
  42. unsigned typeId; // token type this matcher recognizes
  43. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  44. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  45. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  46. bool enableFl; // true if this matcher is enabled
  47. } cmLexMatcher;
  48. typedef struct cmLex_str
  49. {
  50. cmErr_t err;
  51. const cmChar_t* cp; // character buffer
  52. unsigned cn; // count of characters in buffer
  53. unsigned ci; // current buffer index position
  54. unsigned flags; // lexer control flags
  55. unsigned curTokenId; // type id of the current token
  56. unsigned curTokenCharIdx; // index into cp[] of the current token
  57. unsigned curTokenCharCnt; // count of characters in the current token
  58. unsigned curLine; // line number of the current token
  59. unsigned curCol; // column number of the current token
  60. unsigned nextLine;
  61. unsigned nextCol;
  62. cmChar_t* blockBegCmtStr;
  63. cmChar_t* blockEndCmtStr;
  64. cmChar_t* lineCmtStr;
  65. cmLexMatcher* mfp; // base of matcher array
  66. unsigned mfi; // next available matcher array slot
  67. unsigned mfn; // count of elementes in mfp[]
  68. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  69. unsigned attrFlags; // used to store the int and real suffix type flags
  70. } cmLex;
  71. cmLexH cmLexNullH = { NULL };
  72. bool _cmLexIsNewline( cmChar_t c )
  73. { return c == '\n'; }
  74. bool _cmLexIsCommentTypeId( unsigned typeId )
  75. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  76. cmLex* _cmLexHandleToPtr( cmLexH h )
  77. {
  78. cmLex* p = (cmLex*)h.h;
  79. assert(p != NULL);
  80. return p;
  81. };
  82. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  83. {
  84. va_list vl;
  85. va_start(vl,fmt);
  86. unsigned bufCharCnt = 512;
  87. char buf[ bufCharCnt+1 ];
  88. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  89. unsigned sn = strlen(buf);
  90. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  91. buf[bufCharCnt]=0;
  92. cmErrMsg(&p->err,rc,"%s",buf);
  93. va_end(vl);
  94. return rc;
  95. }
  96. // Locate 'keyStr' in cp[cn] and return the index into cp[cn] of the character
  97. // following the last char in 'keyStr'. If keyStr is not found return cmInvalidIdx.
  98. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  99. {
  100. unsigned i = 0;
  101. unsigned n = strlen(keyStr);
  102. if( n <= cn )
  103. for(; i<=cn-n; ++i)
  104. if( strncmp(cp + i, keyStr, n ) == 0 )
  105. return i+n;
  106. return cmInvalidIdx;
  107. }
  108. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  109. {
  110. unsigned n = strlen(keyStr);
  111. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  112. }
  113. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  114. {
  115. unsigned i=0;
  116. for(; i<cn; ++i)
  117. if( !isspace(cp[i]) )
  118. break;
  119. return i;
  120. }
  121. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  122. {
  123. unsigned i = 0;
  124. unsigned n = 0; // decimal point counter
  125. unsigned d = 0; // digit counter
  126. bool fl = false; // expo flag
  127. for(; i<cn && n<=1; ++i)
  128. {
  129. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  130. continue;
  131. if( isdigit(cp[i]) ) // allow digits
  132. {
  133. ++d;
  134. continue;
  135. }
  136. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  137. ++n;
  138. else
  139. break;
  140. }
  141. // if there was at least one digit and the next char is an 'e'
  142. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  143. {
  144. unsigned e=0;
  145. ++i;
  146. unsigned j = i;
  147. fl = false;
  148. for(; i<cn; ++i)
  149. {
  150. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  151. continue;
  152. if( isdigit(cp[i]) )
  153. {
  154. ++e;
  155. ++d;
  156. continue;
  157. }
  158. // stop at the first non-digit
  159. break;
  160. }
  161. // an exp exists if digits follwed the 'e'
  162. fl = e > 0;
  163. }
  164. // if at least one digit was found
  165. if( d>0 )
  166. {
  167. // Note that this path allows a string w/o a decimal pt to trigger a match.
  168. if(i<cn)
  169. {
  170. // if the real has a suffix
  171. switch(cp[i])
  172. {
  173. case 'F':
  174. case 'f':
  175. p->attrFlags = cmSetFlag(p->attrFlags,kRealFloatLexFl);
  176. ++i;
  177. break;
  178. }
  179. }
  180. // match w/o suffix return
  181. if( d>0 && (fl || n==1 || cmIsFlag(p->attrFlags,kRealFloatLexFl)) )
  182. return i;
  183. }
  184. return 0; // no-match return
  185. }
  186. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  187. {
  188. unsigned i = 0;
  189. bool signFl = false;
  190. unsigned digitCnt = 0;
  191. for(; i<cn; ++i)
  192. {
  193. if( i==0 && cp[i]=='-' )
  194. {
  195. signFl = true;
  196. continue;
  197. }
  198. if( !isdigit(cp[i]) )
  199. break;
  200. ++digitCnt;
  201. }
  202. // BUG BUG BUG
  203. // If an integer is specified using 'e' notiation
  204. // (see _cmLexRealMatcher()) and the number of exponent places
  205. // specified following the 'e' is positive and >= number of
  206. // digits following the decimal point (in effect zeros are
  207. // padded on the right side) then the value is an integer.
  208. //
  209. // The current implementation recognizes all numeric strings
  210. // containing a decimal point as reals.
  211. // if no integer was found
  212. if( digitCnt==0)
  213. return 0;
  214. // check for suffix
  215. if(i<cn )
  216. {
  217. switch(cp[i])
  218. {
  219. case 'u':
  220. case 'U':
  221. if( signFl )
  222. _cmLexError(p,kSignErrorLexRC,"A signed integer has a 'u' or 'U' suffix.");
  223. else
  224. {
  225. p->attrFlags = cmSetFlag(p->attrFlags,kIntUnsignedLexFl);
  226. ++i;
  227. }
  228. break;
  229. default:
  230. break;
  231. }
  232. }
  233. return i;
  234. }
  235. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  236. {
  237. unsigned i = 0;
  238. if( cn < 3 )
  239. return 0;
  240. if( cp[0]=='0' && cp[1]=='x')
  241. for(i=2; i<cn; ++i)
  242. if( !isxdigit(cp[i]) )
  243. break;
  244. return i;
  245. }
  246. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  247. {
  248. unsigned i = 0;
  249. if( isalpha(cp[0]) || (cp[0]== '_'))
  250. {
  251. i = 1;
  252. for(; i<cn; ++i)
  253. if( !isalnum(cp[i]) && (cp[i] != '_') )
  254. break;
  255. }
  256. return i;
  257. }
  258. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  259. {
  260. bool escFl = false;
  261. unsigned i = 0;
  262. if( cp[i] != '"' )
  263. return 0;
  264. for(i=1; i<cn; ++i)
  265. {
  266. if( escFl )
  267. {
  268. escFl = false;
  269. continue;
  270. }
  271. if( cp[i] == '\\' )
  272. {
  273. escFl = true;
  274. continue;
  275. }
  276. if( cp[i] == '"' )
  277. return i+1;
  278. }
  279. return _cmLexError(p, kMissingEndQuoteLexRC, "Missing string literal end quote.");
  280. }
  281. unsigned _cmLexQCharMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  282. {
  283. unsigned i = 0;
  284. if( i >= cn || cp[i]!='\'' )
  285. return 0;
  286. i+=2;
  287. if( i >= cn || cp[i]!='\'')
  288. return 0;
  289. return 3;
  290. }
  291. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  292. {
  293. unsigned n = strlen(p->blockBegCmtStr);
  294. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  295. {
  296. unsigned i;
  297. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  298. {
  299. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  300. return 0;
  301. }
  302. return n + i;
  303. }
  304. return 0;
  305. }
  306. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  307. {
  308. unsigned n = strlen(p->lineCmtStr);
  309. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  310. {
  311. unsigned i;
  312. const char newlineStr[] = "\n";
  313. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  314. {
  315. // no EOL was found so the comment must be on the last line of the source
  316. return cn;
  317. }
  318. return n + i;
  319. }
  320. return 0;
  321. }
  322. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  323. {
  324. assert( funcPtr==NULL || userPtr==NULL );
  325. assert( !(funcPtr==NULL && userPtr==NULL));
  326. // if there is no space in the user token array - then expand it
  327. if( p->mfi == p->mfn )
  328. {
  329. int incr_cnt = 10;
  330. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  331. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  332. cmMemPtrFree(&p->mfp);
  333. p->mfp = np;
  334. p->mfn += incr_cnt;
  335. }
  336. p->mfp[p->mfi].tokenStr = NULL;
  337. p->mfp[p->mfi].typeId = typeId;
  338. p->mfp[p->mfi].funcPtr = funcPtr;
  339. p->mfp[p->mfi].userPtr = userPtr;
  340. p->mfp[p->mfi].enableFl = true;
  341. if( keyStr != NULL )
  342. {
  343. // allocate space for the token string and store it
  344. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  345. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  346. }
  347. p->mfi++;
  348. return kOkLexRC;
  349. }
  350. cmRC_t _cmLexReset( cmLex* p )
  351. {
  352. p->ci = 0;
  353. p->curTokenId = kErrorLexTId;
  354. p->curTokenCharIdx = cmInvalidIdx;
  355. p->curTokenCharCnt = 0;
  356. p->curLine = 0;
  357. p->curCol = 0;
  358. p->nextLine = 0;
  359. p->nextCol = 0;
  360. cmErrClearRC(&p->err);
  361. return kOkLexRC;
  362. }
  363. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  364. {
  365. p->cp = cp;
  366. p->cn = cn;
  367. return _cmLexReset(p);
  368. }
  369. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  370. {
  371. cmLexH h;
  372. cmChar_t dfltLineCmt[] = "//";
  373. cmChar_t dfltBlockBegCmt[] = "/*";
  374. cmChar_t dfltBlockEndCmt[] = "*/";
  375. cmLex* p = cmMemAllocZ( cmLex, 1 );
  376. cmErrSetup(&p->err,rpt,"Lexer");
  377. p->flags = flags;
  378. _cmLexSetTextBuffer( p, cp, cn );
  379. int init_mfn = 10;
  380. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  381. p->mfn = init_mfn;
  382. p->mfi = 0;
  383. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  384. strcpy( p->lineCmtStr, dfltLineCmt );
  385. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  386. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  387. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  388. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  389. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  390. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  391. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  392. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  393. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  394. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  395. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  396. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  397. if( cmIsFlag(flags,kReturnQCharLexFl) )
  398. _cmLexInstallMatcher( p, kQCharLexTId, _cmLexQCharMatcher, NULL, NULL );
  399. h.h = p;
  400. _cmLexReset(p);
  401. return h;
  402. }
  403. cmRC_t cmLexFinal( cmLexH* hp )
  404. {
  405. if( hp == NULL || cmLexIsValid(*hp)==false )
  406. return cmOkRC;
  407. cmLex* p = _cmLexHandleToPtr(*hp);
  408. if( p != NULL )
  409. {
  410. if( p->mfp != NULL )
  411. {
  412. unsigned i = 0;
  413. // free the user token strings
  414. for(; i<p->mfi; ++i)
  415. if( p->mfp[i].tokenStr != NULL )
  416. cmMemPtrFree(&p->mfp[i].tokenStr);
  417. // free the matcher array
  418. cmMemPtrFree(&p->mfp);
  419. p->mfi = 0;
  420. p->mfn = 0;
  421. }
  422. cmMemPtrFree(&p->lineCmtStr);
  423. cmMemPtrFree(&p->blockBegCmtStr);
  424. cmMemPtrFree(&p->blockEndCmtStr);
  425. cmMemPtrFree(&p->textBuf);
  426. // free the lexer object
  427. cmMemPtrFree(&p);
  428. hp->h = NULL;
  429. }
  430. return kOkLexRC;
  431. }
  432. cmRC_t cmLexReset( cmLexH h )
  433. {
  434. cmLex* p = _cmLexHandleToPtr(h);
  435. return _cmLexReset(p);
  436. }
  437. bool cmLexIsValid( cmLexH h )
  438. { return h.h != NULL; }
  439. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  440. {
  441. cmLex* p = _cmLexHandleToPtr(h);
  442. return _cmLexSetTextBuffer(p,cp,cn);
  443. }
  444. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  445. {
  446. cmRC_t rc = kOkLexRC;
  447. cmFileH_t fh = cmFileNullHandle;
  448. cmLex* p = _cmLexHandleToPtr(h);
  449. long n = 0;
  450. assert( fn != NULL && p != NULL );
  451. // open the file
  452. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  453. return kFileOpenErrLexRC;
  454. // seek to the end of the file
  455. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  456. return kFileSeekErrLexRC;
  457. // get the length of the file
  458. if( cmFileTell(fh,&n) != kOkFileRC )
  459. return kFileTellErrLexRC;
  460. // rewind to the beginning of the file
  461. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  462. return kFileSeekErrLexRC;
  463. // allocate the text buffer
  464. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  465. {
  466. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  467. goto errLabel;
  468. }
  469. // read the file into the buffer
  470. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  471. return kFileReadErrLexRC;
  472. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  473. goto errLabel;
  474. errLabel:
  475. // close the file
  476. if( cmFileClose(&fh) != kOkFileRC )
  477. return kFileCloseErrLexRC;
  478. return rc;
  479. }
  480. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  481. {
  482. unsigned i = 0;
  483. for(; i<p->mfi; ++i)
  484. {
  485. if( id != cmInvalidId && p->mfp[i].typeId == id )
  486. return p->mfp + i;
  487. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  488. return p->mfp + i;
  489. }
  490. return NULL;
  491. }
  492. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  493. {
  494. cmLex* p = _cmLexHandleToPtr(h);
  495. // prevent duplicate tokens
  496. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  497. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  498. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  499. }
  500. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  501. {
  502. cmLex* p = _cmLexHandleToPtr(h);
  503. // prevent duplicate tokens
  504. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  505. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  506. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  507. }
  508. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  509. {
  510. cmLex* p = _cmLexHandleToPtr(h);
  511. unsigned mi = 0;
  512. for(; mi<p->mfi; ++mi)
  513. if( p->mfp[mi].typeId == id )
  514. {
  515. p->mfp[mi].enableFl = enableFl;
  516. return cmOkRC;
  517. }
  518. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  519. }
  520. unsigned cmLexFilterFlags( cmLexH h )
  521. {
  522. cmLex* p = _cmLexHandleToPtr(h);
  523. return p->flags;
  524. }
  525. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  526. {
  527. cmLex* p = _cmLexHandleToPtr(h);
  528. p->flags = flags;
  529. }
  530. unsigned cmLexGetNextToken( cmLexH h )
  531. {
  532. cmLex* p = _cmLexHandleToPtr(h);
  533. if( cmErrLastRC(&p->err) != kOkLexRC )
  534. return kErrorLexTId;
  535. while( p->ci < p->cn )
  536. {
  537. unsigned i;
  538. unsigned mi = 0;
  539. unsigned maxCharCnt = 0;
  540. unsigned maxIdx = cmInvalidIdx;
  541. p->curTokenId = kErrorLexTId;
  542. p->curTokenCharIdx = cmInvalidIdx;
  543. p->curTokenCharCnt = 0;
  544. p->attrFlags = 0;
  545. // try each matcher
  546. for(; mi<p->mfi; ++mi)
  547. if( p->mfp[mi].enableFl )
  548. {
  549. unsigned charCnt = 0;
  550. if( p->mfp[mi].funcPtr != NULL )
  551. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  552. else
  553. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  554. // notice if the matcher set the error code
  555. if( cmErrLastRC(&p->err) != kOkLexRC )
  556. return kErrorLexTId;
  557. // if this matched token is longer then the prev. matched token or
  558. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  559. if( (charCnt > maxCharCnt)
  560. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  561. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  562. )
  563. {
  564. maxCharCnt = charCnt;
  565. maxIdx = mi;
  566. }
  567. }
  568. // no token was matched
  569. if( maxIdx == cmInvalidIdx )
  570. {
  571. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  572. {
  573. maxCharCnt = 1;
  574. }
  575. else
  576. {
  577. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  578. return kErrorLexTId;
  579. }
  580. }
  581. // update the current line and column position
  582. p->curLine = p->nextLine;
  583. p->curCol = p->nextCol;
  584. // find the next column and line position
  585. for(i=0; i<maxCharCnt; ++i)
  586. {
  587. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  588. {
  589. p->nextLine++;
  590. p->nextCol = 1;
  591. }
  592. else
  593. p->nextCol++;
  594. }
  595. bool returnFl = true;
  596. if( maxIdx != cmInvalidIdx )
  597. {
  598. // check the space token filter
  599. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  600. returnFl = false;
  601. // check the comment token filter
  602. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  603. returnFl = false;
  604. }
  605. // update the lexer state
  606. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  607. p->curTokenCharIdx = p->ci;
  608. p->curTokenCharCnt = maxCharCnt;
  609. // advance the text buffer
  610. p->ci += maxCharCnt;
  611. if( returnFl )
  612. return p->curTokenId;
  613. }
  614. cmErrSetRC(&p->err,kEofRC);
  615. return kEofLexTId;
  616. }
  617. unsigned cmLexTokenId( cmLexH h )
  618. {
  619. cmLex* p = _cmLexHandleToPtr(h);
  620. return p->curTokenId;
  621. }
  622. const cmChar_t* cmLexTokenText( cmLexH h )
  623. {
  624. cmLex* p = _cmLexHandleToPtr(h);
  625. if( p->curTokenCharIdx == cmInvalidIdx )
  626. return NULL;
  627. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  628. return p->cp + p->curTokenCharIdx + n;
  629. }
  630. unsigned cmLexTokenCharCount( cmLexH h )
  631. {
  632. cmLex* p = _cmLexHandleToPtr(h);
  633. if( p->curTokenCharIdx == cmInvalidIdx )
  634. return 0;
  635. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  636. return p->curTokenCharCnt - n;
  637. }
  638. int cmLexTokenInt( cmLexH h )
  639. { return strtol( cmLexTokenText(h),NULL,0 ); }
  640. unsigned cmLexTokenUInt( cmLexH h )
  641. { return strtol( cmLexTokenText(h),NULL,0 ); }
  642. float cmLexTokenFloat( cmLexH h )
  643. { return strtof( cmLexTokenText(h),NULL ); }
  644. double cmLexTokenDouble( cmLexH h )
  645. { return strtod( cmLexTokenText(h),NULL ); }
  646. bool cmLexTokenIsUnsigned( cmLexH h )
  647. {
  648. cmLex* p = _cmLexHandleToPtr(h);
  649. return p->curTokenId == kIntLexTId && cmIsFlag(p->attrFlags,kIntUnsignedLexFl);
  650. }
  651. bool cmLexTokenIsSinglePrecision( cmLexH h )
  652. {
  653. cmLex* p = _cmLexHandleToPtr(h);
  654. return p->curTokenId == kRealLexTId && cmIsFlag(p->attrFlags,kRealFloatLexFl);
  655. }
  656. unsigned cmLexCurrentLineNumber( cmLexH h )
  657. {
  658. cmLex* p = _cmLexHandleToPtr(h);
  659. return p->curLine + 1;
  660. }
  661. unsigned cmLexCurrentColumnNumber( cmLexH h )
  662. {
  663. cmLex* p = _cmLexHandleToPtr(h);
  664. return p->curCol + 1;
  665. }
  666. unsigned cmLexErrorRC( cmLexH h )
  667. {
  668. cmLex* p = _cmLexHandleToPtr(h);
  669. return cmErrLastRC(&p->err);
  670. }
  671. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  672. {
  673. cmLex* p = _cmLexHandleToPtr(h);
  674. switch( typeId )
  675. {
  676. case kErrorLexTId: return "<error>";
  677. case kEofLexTId: return "<EOF>";
  678. case kSpaceLexTId: return "<space>";
  679. case kRealLexTId: return "<real>";
  680. case kIntLexTId: return "<int>";
  681. case kHexLexTId: return "<hex>";
  682. case kIdentLexTId: return "<ident>";
  683. case kQStrLexTId: return "<qstr>";
  684. case kBlockCmtLexTId: return "<bcmt>";
  685. case kLineCmtLexTId: return "<lcmt>";
  686. default:
  687. {
  688. cmLexMatcher* mp;
  689. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  690. return "<unknown>";
  691. return mp->tokenStr;
  692. }
  693. }
  694. return "<invalid>";
  695. }
  696. const cmChar_t* cmLexRcToMsg( unsigned rc )
  697. {
  698. unsigned i=0;
  699. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  700. if( cmLexErrorArray[i].code == rc )
  701. break;
  702. return cmLexErrorArray[i].msg;
  703. }
  704. //{ { label:cmLexEx }
  705. //(
  706. // cmLexTest() gives a simple cmLex example.
  707. //)
  708. //(
  709. void cmLexTest( cmRpt_t* rpt)
  710. {
  711. cmChar_t buf[] =
  712. "123ident0\n 123.456\nident0\n"
  713. "0xa12+.2\n"
  714. "// comment \n"
  715. "/* block \n"
  716. "comment */"
  717. "\"quoted string\""
  718. "ident1"
  719. "// last line comment";
  720. // initialize a lexer with a buffer of text
  721. cmLexH h = cmLexInit(buf,strlen(buf),
  722. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  723. // verify that the lexer initialization succeded.
  724. if( cmLexIsValid(h) == false )
  725. {
  726. cmRptPrintf(rpt,"Lexer initialization failed.");
  727. return;
  728. }
  729. // register some additional recoginizers
  730. cmLexRegisterToken(h,kUserLexTId+1,"+");
  731. cmLexRegisterToken(h,kUserLexTId+2,"-");
  732. unsigned tid;
  733. // ask for token id's
  734. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  735. {
  736. // print information about each token
  737. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  738. cmLexCurrentLineNumber(h),
  739. cmLexCurrentColumnNumber(h),
  740. cmLexIdToLabel(h,tid),
  741. cmLexTokenCharCount(h),
  742. cmLexTokenText(h) ,
  743. cmLexTokenCharCount(h));
  744. // if the token is a number ...
  745. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  746. {
  747. // ... then request the numbers value
  748. int iv = cmLexTokenInt(h);
  749. double dv = cmLexTokenDouble(h);
  750. cmRptPrintf(rpt,"%i %f",iv,dv);
  751. }
  752. cmRptPrintf(rpt,"\n");
  753. // handle errors
  754. if( tid == kErrorLexTId )
  755. {
  756. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  757. break;
  758. }
  759. }
  760. // finalize the lexer
  761. cmLexFinal(&h);
  762. }
  763. //)
  764. //}