libcm is a C development framework with an emphasis on audio signal processing applications.
您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符

cmLex.c 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. enum
  10. {
  11. kRealFloatLexFl = 0x01,
  12. kIntUnsignedLexFl = 0x02
  13. };
  14. typedef struct
  15. {
  16. unsigned code;
  17. const cmChar_t* msg;
  18. } cmLexErrorRecd;
  19. cmLexErrorRecd cmLexErrorArray[] =
  20. {
  21. { kOkLexRC, "No error. The operation completed successfully."},
  22. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  23. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  24. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  25. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  26. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  27. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  28. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  29. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  30. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  31. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  32. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  33. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  34. { kSignErrorLexRC, "A signed integer has a 'u' or 'U' suffix."},
  35. { kInvalidLexRC, "Unknown lexer error code." }
  36. };
  37. struct cmLex_str;
  38. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  39. // token match function record
  40. typedef struct
  41. {
  42. unsigned typeId; // token type this matcher recognizes
  43. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  44. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  45. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  46. bool enableFl; // true if this matcher is enabled
  47. } cmLexMatcher;
  48. typedef struct cmLex_str
  49. {
  50. cmErr_t err;
  51. const cmChar_t* cp; // character buffer
  52. unsigned cn; // count of characters in buffer
  53. unsigned ci; // current buffer index position
  54. unsigned flags; // lexer control flags
  55. unsigned curTokenId; // type id of the current token
  56. unsigned curTokenCharIdx; // index into cp[] of the current token
  57. unsigned curTokenCharCnt; // count of characters in the current token
  58. unsigned curLine; // line number of the current token
  59. unsigned curCol; // column number of the current token
  60. unsigned nextLine;
  61. unsigned nextCol;
  62. cmChar_t* blockBegCmtStr;
  63. cmChar_t* blockEndCmtStr;
  64. cmChar_t* lineCmtStr;
  65. cmLexMatcher* mfp; // base of matcher array
  66. unsigned mfi; // next available matcher array slot
  67. unsigned mfn; // count of elementes in mfp[]
  68. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  69. unsigned attrFlags; // used to store the int and real suffix type flags
  70. } cmLex;
  71. cmLexH cmLexNullH = { NULL };
  72. bool _cmLexIsNewline( cmChar_t c )
  73. { return c == '\n'; }
  74. bool _cmLexIsCommentTypeId( unsigned typeId )
  75. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  76. cmLex* _cmLexHandleToPtr( cmLexH h )
  77. {
  78. cmLex* p = (cmLex*)h.h;
  79. assert(p != NULL);
  80. return p;
  81. };
  82. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  83. {
  84. va_list vl;
  85. va_start(vl,fmt);
  86. unsigned bufCharCnt = 512;
  87. char buf[ bufCharCnt+1 ];
  88. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  89. unsigned sn = strlen(buf);
  90. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  91. buf[bufCharCnt]=0;
  92. cmErrMsg(&p->err,rc,"%s",buf);
  93. va_end(vl);
  94. return rc;
  95. }
  96. // Locate 'keyStr' in cp[cn] and return the index into cp[cn] of the character
  97. // following the last char in 'keyStr'. If keyStr is not found return cmInvalidIdx.
  98. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  99. {
  100. unsigned i = 0;
  101. unsigned n = strlen(keyStr);
  102. if( n <= cn )
  103. for(; i<=cn-n; ++i)
  104. if( strncmp(cp + i, keyStr, n ) == 0 )
  105. return i+n;
  106. return cmInvalidIdx;
  107. }
  108. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  109. {
  110. unsigned n = strlen(keyStr);
  111. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  112. }
  113. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  114. {
  115. unsigned i=0;
  116. for(; i<cn; ++i)
  117. if( !isspace(cp[i]) )
  118. break;
  119. return i;
  120. }
  121. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  122. {
  123. unsigned i = 0;
  124. unsigned n = 0; // decimal point counter
  125. unsigned d = 0; // digit counter
  126. bool fl = false; // expo flag
  127. for(; i<cn && n<=1; ++i)
  128. {
  129. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  130. continue;
  131. if( isdigit(cp[i]) ) // allow digits
  132. {
  133. ++d;
  134. continue;
  135. }
  136. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  137. ++n;
  138. else
  139. break;
  140. }
  141. // if there was at least one digit and the next char is an 'e'
  142. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  143. {
  144. unsigned e=0;
  145. ++i;
  146. unsigned j = i;
  147. fl = false;
  148. for(; i<cn; ++i)
  149. {
  150. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  151. continue;
  152. if( isdigit(cp[i]) )
  153. {
  154. ++e;
  155. ++d;
  156. continue;
  157. }
  158. // stop at the first non-digit
  159. break;
  160. }
  161. // an exp exists if digits follwed the 'e'
  162. fl = e > 0;
  163. }
  164. // if at least one digit was found
  165. if( d>0 )
  166. {
  167. // Note that this path allows a string w/o a decimal pt to trigger a match.
  168. if(i<cn)
  169. {
  170. // if the real has a suffix
  171. switch(cp[i])
  172. {
  173. case 'F':
  174. case 'f':
  175. p->attrFlags = cmSetFlag(p->attrFlags,kRealFloatLexFl);
  176. ++i;
  177. break;
  178. }
  179. }
  180. // match w/o suffix return
  181. if( d>0 && (fl || n==1 || cmIsFlag(p->attrFlags,kRealFloatLexFl)) )
  182. return i;
  183. }
  184. return 0; // no-match return
  185. }
  186. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  187. {
  188. unsigned i = 0;
  189. bool signFl = false;
  190. for(; i<cn; ++i)
  191. {
  192. if( i==0 && cp[i]=='-' )
  193. {
  194. signFl = true;
  195. continue;
  196. }
  197. if( !isdigit(cp[i]) )
  198. break;
  199. }
  200. // BUG BUG BUG
  201. // If an integer is specified using 'e' notiation
  202. // (see _cmLexRealMatcher()) and the number of exponent places
  203. // specified following the 'e' is positive and >= number of
  204. // digits following the decimal point (in effect zeros are
  205. // padded on the right side) then the value is an integer.
  206. //
  207. // The current implementation recognizes all numeric strings
  208. // containing a decimal point as reals.
  209. // if no integer was found
  210. if( (signFl && i==0) || i==0 )
  211. return 0;
  212. // check for suffix
  213. if(i<cn )
  214. {
  215. switch(cp[i])
  216. {
  217. case 'u':
  218. case 'U':
  219. if( signFl )
  220. _cmLexError(p,kSignErrorLexRC,"A signed integer has a 'u' or 'U' suffix.");
  221. else
  222. {
  223. p->attrFlags = cmSetFlag(p->attrFlags,kIntUnsignedLexFl);
  224. ++i;
  225. }
  226. break;
  227. default:
  228. break;
  229. }
  230. }
  231. return i;
  232. }
  233. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  234. {
  235. unsigned i = 0;
  236. if( cn < 3 )
  237. return 0;
  238. if( cp[0]=='0' && cp[1]=='x')
  239. for(i=2; i<cn; ++i)
  240. if( !isxdigit(cp[i]) )
  241. break;
  242. return i;
  243. }
  244. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  245. {
  246. unsigned i = 0;
  247. if( isalpha(cp[0]) || (cp[0]== '_'))
  248. {
  249. i = 1;
  250. for(; i<cn; ++i)
  251. if( !isalnum(cp[i]) && (cp[i] != '_') )
  252. break;
  253. }
  254. return i;
  255. }
  256. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  257. {
  258. bool escFl = false;
  259. unsigned i = 0;
  260. if( cp[i] != '"' )
  261. return 0;
  262. for(i=1; i<cn; ++i)
  263. {
  264. if( escFl )
  265. {
  266. escFl = false;
  267. continue;
  268. }
  269. if( cp[i] == '\\' )
  270. {
  271. escFl = true;
  272. continue;
  273. }
  274. if( cp[i] == '"' )
  275. return i+1;
  276. }
  277. return _cmLexError(p, kMissingEndQuoteLexRC, "Missing string literal end quote.");
  278. }
  279. unsigned _cmLexQCharMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  280. {
  281. unsigned i = 0;
  282. if( i >= cn || cp[i]!='\'' )
  283. return 0;
  284. i+=2;
  285. if( i >= cn || cp[i]!='\'')
  286. return 0;
  287. return 3;
  288. }
  289. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  290. {
  291. unsigned n = strlen(p->blockBegCmtStr);
  292. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  293. {
  294. unsigned i;
  295. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  296. {
  297. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  298. return 0;
  299. }
  300. return n + i;
  301. }
  302. return 0;
  303. }
  304. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  305. {
  306. unsigned n = strlen(p->lineCmtStr);
  307. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  308. {
  309. unsigned i;
  310. const char newlineStr[] = "\n";
  311. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  312. {
  313. // no EOL was found so the comment must be on the last line of the source
  314. return cn;
  315. }
  316. return n + i;
  317. }
  318. return 0;
  319. }
  320. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  321. {
  322. assert( funcPtr==NULL || userPtr==NULL );
  323. assert( !(funcPtr==NULL && userPtr==NULL));
  324. // if there is no space in the user token array - then expand it
  325. if( p->mfi == p->mfn )
  326. {
  327. int incr_cnt = 10;
  328. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  329. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  330. cmMemPtrFree(&p->mfp);
  331. p->mfp = np;
  332. p->mfn += incr_cnt;
  333. }
  334. p->mfp[p->mfi].tokenStr = NULL;
  335. p->mfp[p->mfi].typeId = typeId;
  336. p->mfp[p->mfi].funcPtr = funcPtr;
  337. p->mfp[p->mfi].userPtr = userPtr;
  338. p->mfp[p->mfi].enableFl = true;
  339. if( keyStr != NULL )
  340. {
  341. // allocate space for the token string and store it
  342. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  343. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  344. }
  345. p->mfi++;
  346. return kOkLexRC;
  347. }
  348. cmRC_t _cmLexReset( cmLex* p )
  349. {
  350. p->ci = 0;
  351. p->curTokenId = kErrorLexTId;
  352. p->curTokenCharIdx = cmInvalidIdx;
  353. p->curTokenCharCnt = 0;
  354. p->curLine = 0;
  355. p->curCol = 0;
  356. p->nextLine = 0;
  357. p->nextCol = 0;
  358. cmErrClearRC(&p->err);
  359. return kOkLexRC;
  360. }
  361. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  362. {
  363. p->cp = cp;
  364. p->cn = cn;
  365. return _cmLexReset(p);
  366. }
  367. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  368. {
  369. cmLexH h;
  370. cmChar_t dfltLineCmt[] = "//";
  371. cmChar_t dfltBlockBegCmt[] = "/*";
  372. cmChar_t dfltBlockEndCmt[] = "*/";
  373. cmLex* p = cmMemAllocZ( cmLex, 1 );
  374. cmErrSetup(&p->err,rpt,"Lexer");
  375. p->flags = flags;
  376. _cmLexSetTextBuffer( p, cp, cn );
  377. int init_mfn = 10;
  378. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  379. p->mfn = init_mfn;
  380. p->mfi = 0;
  381. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  382. strcpy( p->lineCmtStr, dfltLineCmt );
  383. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  384. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  385. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  386. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  387. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  388. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  389. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  390. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  391. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  392. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  393. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  394. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  395. if( cmIsFlag(flags,kReturnQCharLexFl) )
  396. _cmLexInstallMatcher( p, kQCharLexTId, _cmLexQCharMatcher, NULL, NULL );
  397. h.h = p;
  398. _cmLexReset(p);
  399. return h;
  400. }
  401. cmRC_t cmLexFinal( cmLexH* hp )
  402. {
  403. if( hp == NULL || cmLexIsValid(*hp)==false )
  404. return cmOkRC;
  405. cmLex* p = _cmLexHandleToPtr(*hp);
  406. if( p != NULL )
  407. {
  408. if( p->mfp != NULL )
  409. {
  410. unsigned i = 0;
  411. // free the user token strings
  412. for(; i<p->mfi; ++i)
  413. if( p->mfp[i].tokenStr != NULL )
  414. cmMemPtrFree(&p->mfp[i].tokenStr);
  415. // free the matcher array
  416. cmMemPtrFree(&p->mfp);
  417. p->mfi = 0;
  418. p->mfn = 0;
  419. }
  420. cmMemPtrFree(&p->lineCmtStr);
  421. cmMemPtrFree(&p->blockBegCmtStr);
  422. cmMemPtrFree(&p->blockEndCmtStr);
  423. cmMemPtrFree(&p->textBuf);
  424. // free the lexer object
  425. cmMemPtrFree(&p);
  426. hp->h = NULL;
  427. }
  428. return kOkLexRC;
  429. }
  430. cmRC_t cmLexReset( cmLexH h )
  431. {
  432. cmLex* p = _cmLexHandleToPtr(h);
  433. return _cmLexReset(p);
  434. }
  435. bool cmLexIsValid( cmLexH h )
  436. { return h.h != NULL; }
  437. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  438. {
  439. cmLex* p = _cmLexHandleToPtr(h);
  440. return _cmLexSetTextBuffer(p,cp,cn);
  441. }
  442. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  443. {
  444. cmRC_t rc = kOkLexRC;
  445. cmFileH_t fh = cmFileNullHandle;
  446. cmLex* p = _cmLexHandleToPtr(h);
  447. long n = 0;
  448. assert( fn != NULL && p != NULL );
  449. // open the file
  450. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  451. return kFileOpenErrLexRC;
  452. // seek to the end of the file
  453. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  454. return kFileSeekErrLexRC;
  455. // get the length of the file
  456. if( cmFileTell(fh,&n) != kOkFileRC )
  457. return kFileTellErrLexRC;
  458. // rewind to the beginning of the file
  459. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  460. return kFileSeekErrLexRC;
  461. // allocate the text buffer
  462. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  463. {
  464. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  465. goto errLabel;
  466. }
  467. // read the file into the buffer
  468. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  469. return kFileReadErrLexRC;
  470. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  471. goto errLabel;
  472. errLabel:
  473. // close the file
  474. if( cmFileClose(&fh) != kOkFileRC )
  475. return kFileCloseErrLexRC;
  476. return rc;
  477. }
  478. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  479. {
  480. unsigned i = 0;
  481. for(; i<p->mfi; ++i)
  482. {
  483. if( id != cmInvalidId && p->mfp[i].typeId == id )
  484. return p->mfp + i;
  485. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  486. return p->mfp + i;
  487. }
  488. return NULL;
  489. }
  490. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  491. {
  492. cmLex* p = _cmLexHandleToPtr(h);
  493. // prevent duplicate tokens
  494. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  495. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  496. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  497. }
  498. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  499. {
  500. cmLex* p = _cmLexHandleToPtr(h);
  501. // prevent duplicate tokens
  502. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  503. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  504. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  505. }
  506. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  507. {
  508. cmLex* p = _cmLexHandleToPtr(h);
  509. unsigned mi = 0;
  510. for(; mi<p->mfi; ++mi)
  511. if( p->mfp[mi].typeId == id )
  512. {
  513. p->mfp[mi].enableFl = enableFl;
  514. return cmOkRC;
  515. }
  516. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  517. }
  518. unsigned cmLexFilterFlags( cmLexH h )
  519. {
  520. cmLex* p = _cmLexHandleToPtr(h);
  521. return p->flags;
  522. }
  523. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  524. {
  525. cmLex* p = _cmLexHandleToPtr(h);
  526. p->flags = flags;
  527. }
  528. unsigned cmLexGetNextToken( cmLexH h )
  529. {
  530. cmLex* p = _cmLexHandleToPtr(h);
  531. if( cmErrLastRC(&p->err) != kOkLexRC )
  532. return kErrorLexTId;
  533. while( p->ci < p->cn )
  534. {
  535. unsigned i;
  536. unsigned mi = 0;
  537. unsigned maxCharCnt = 0;
  538. unsigned maxIdx = cmInvalidIdx;
  539. p->curTokenId = kErrorLexTId;
  540. p->curTokenCharIdx = cmInvalidIdx;
  541. p->curTokenCharCnt = 0;
  542. p->attrFlags = 0;
  543. // try each matcher
  544. for(; mi<p->mfi; ++mi)
  545. if( p->mfp[mi].enableFl )
  546. {
  547. unsigned charCnt = 0;
  548. if( p->mfp[mi].funcPtr != NULL )
  549. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  550. else
  551. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  552. // notice if the matcher set the error code
  553. if( cmErrLastRC(&p->err) != kOkLexRC )
  554. return kErrorLexTId;
  555. // if this matched token is longer then the prev. matched token or
  556. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  557. if( (charCnt > maxCharCnt)
  558. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  559. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  560. )
  561. {
  562. maxCharCnt = charCnt;
  563. maxIdx = mi;
  564. }
  565. }
  566. // no token was matched
  567. if( maxIdx == cmInvalidIdx )
  568. {
  569. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  570. {
  571. maxCharCnt = 1;
  572. }
  573. else
  574. {
  575. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  576. return kErrorLexTId;
  577. }
  578. }
  579. // update the current line and column position
  580. p->curLine = p->nextLine;
  581. p->curCol = p->nextCol;
  582. // find the next column and line position
  583. for(i=0; i<maxCharCnt; ++i)
  584. {
  585. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  586. {
  587. p->nextLine++;
  588. p->nextCol = 1;
  589. }
  590. else
  591. p->nextCol++;
  592. }
  593. bool returnFl = true;
  594. if( maxIdx != cmInvalidIdx )
  595. {
  596. // check the space token filter
  597. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  598. returnFl = false;
  599. // check the comment token filter
  600. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  601. returnFl = false;
  602. }
  603. // update the lexer state
  604. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  605. p->curTokenCharIdx = p->ci;
  606. p->curTokenCharCnt = maxCharCnt;
  607. // advance the text buffer
  608. p->ci += maxCharCnt;
  609. if( returnFl )
  610. return p->curTokenId;
  611. }
  612. cmErrSetRC(&p->err,kEofRC);
  613. return kEofLexTId;
  614. }
  615. unsigned cmLexTokenId( cmLexH h )
  616. {
  617. cmLex* p = _cmLexHandleToPtr(h);
  618. return p->curTokenId;
  619. }
  620. const cmChar_t* cmLexTokenText( cmLexH h )
  621. {
  622. cmLex* p = _cmLexHandleToPtr(h);
  623. if( p->curTokenCharIdx == cmInvalidIdx )
  624. return NULL;
  625. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  626. return p->cp + p->curTokenCharIdx + n;
  627. }
  628. unsigned cmLexTokenCharCount( cmLexH h )
  629. {
  630. cmLex* p = _cmLexHandleToPtr(h);
  631. if( p->curTokenCharIdx == cmInvalidIdx )
  632. return 0;
  633. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  634. return p->curTokenCharCnt - n;
  635. }
  636. int cmLexTokenInt( cmLexH h )
  637. { return strtol( cmLexTokenText(h),NULL,0 ); }
  638. unsigned cmLexTokenUInt( cmLexH h )
  639. { return strtol( cmLexTokenText(h),NULL,0 ); }
  640. float cmLexTokenFloat( cmLexH h )
  641. { return strtof( cmLexTokenText(h),NULL ); }
  642. double cmLexTokenDouble( cmLexH h )
  643. { return strtod( cmLexTokenText(h),NULL ); }
  644. bool cmLexTokenIsUnsigned( cmLexH h )
  645. {
  646. cmLex* p = _cmLexHandleToPtr(h);
  647. return p->curTokenId == kIntLexTId && cmIsFlag(p->attrFlags,kIntUnsignedLexFl);
  648. }
  649. bool cmLexTokenIsSinglePrecision( cmLexH h )
  650. {
  651. cmLex* p = _cmLexHandleToPtr(h);
  652. return p->curTokenId == kRealLexTId && cmIsFlag(p->attrFlags,kRealFloatLexFl);
  653. }
  654. unsigned cmLexCurrentLineNumber( cmLexH h )
  655. {
  656. cmLex* p = _cmLexHandleToPtr(h);
  657. return p->curLine + 1;
  658. }
  659. unsigned cmLexCurrentColumnNumber( cmLexH h )
  660. {
  661. cmLex* p = _cmLexHandleToPtr(h);
  662. return p->curCol + 1;
  663. }
  664. unsigned cmLexErrorRC( cmLexH h )
  665. {
  666. cmLex* p = _cmLexHandleToPtr(h);
  667. return cmErrLastRC(&p->err);
  668. }
  669. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  670. {
  671. cmLex* p = _cmLexHandleToPtr(h);
  672. switch( typeId )
  673. {
  674. case kErrorLexTId: return "<error>";
  675. case kEofLexTId: return "<EOF>";
  676. case kSpaceLexTId: return "<space>";
  677. case kRealLexTId: return "<real>";
  678. case kIntLexTId: return "<int>";
  679. case kHexLexTId: return "<hex>";
  680. case kIdentLexTId: return "<ident>";
  681. case kQStrLexTId: return "<qstr>";
  682. case kBlockCmtLexTId: return "<bcmt>";
  683. case kLineCmtLexTId: return "<lcmt>";
  684. default:
  685. {
  686. cmLexMatcher* mp;
  687. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  688. return "<unknown>";
  689. return mp->tokenStr;
  690. }
  691. }
  692. return "<invalid>";
  693. }
  694. const cmChar_t* cmLexRcToMsg( unsigned rc )
  695. {
  696. unsigned i=0;
  697. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  698. if( cmLexErrorArray[i].code == rc )
  699. break;
  700. return cmLexErrorArray[i].msg;
  701. }
  702. //{ { label:cmLexEx }
  703. //(
  704. // cmLexTest() gives a simple cmLex example.
  705. //)
  706. //(
  707. void cmLexTest( cmRpt_t* rpt)
  708. {
  709. cmChar_t buf[] =
  710. "123ident0\n 123.456\nident0\n"
  711. "0xa12+.2\n"
  712. "// comment \n"
  713. "/* block \n"
  714. "comment */"
  715. "\"quoted string\""
  716. "ident1"
  717. "// last line comment";
  718. // initialize a lexer with a buffer of text
  719. cmLexH h = cmLexInit(buf,strlen(buf),
  720. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  721. // verify that the lexer initialization succeded.
  722. if( cmLexIsValid(h) == false )
  723. {
  724. cmRptPrintf(rpt,"Lexer initialization failed.");
  725. return;
  726. }
  727. // register some additional recoginizers
  728. cmLexRegisterToken(h,kUserLexTId+1,"+");
  729. cmLexRegisterToken(h,kUserLexTId+2,"-");
  730. unsigned tid;
  731. // ask for token id's
  732. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  733. {
  734. // print information about each token
  735. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  736. cmLexCurrentLineNumber(h),
  737. cmLexCurrentColumnNumber(h),
  738. cmLexIdToLabel(h,tid),
  739. cmLexTokenCharCount(h),
  740. cmLexTokenText(h) ,
  741. cmLexTokenCharCount(h));
  742. // if the token is a number ...
  743. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  744. {
  745. // ... then request the numbers value
  746. int iv = cmLexTokenInt(h);
  747. double dv = cmLexTokenDouble(h);
  748. cmRptPrintf(rpt,"%i %f",iv,dv);
  749. }
  750. cmRptPrintf(rpt,"\n");
  751. // handle errors
  752. if( tid == kErrorLexTId )
  753. {
  754. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  755. break;
  756. }
  757. }
  758. // finalize the lexer
  759. cmLexFinal(&h);
  760. }
  761. //)
  762. //}