libcm is a C development framework with an emphasis on audio signal processing applications.
Você não pode selecionar mais de 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.

cmLex.c 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. typedef struct
  10. {
  11. unsigned code;
  12. const cmChar_t* msg;
  13. } cmLexErrorRecd;
  14. cmLexErrorRecd cmLexErrorArray[] =
  15. {
  16. { kOkLexRC, "No error. The operation completed successfully."},
  17. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  18. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  19. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  20. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  21. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  22. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  23. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  24. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  25. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  26. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  27. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  28. { kInvalidLexRC, "Unknown lexer error code." }
  29. };
  30. struct cmLex_str;
  31. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  32. // token match function record
  33. typedef struct
  34. {
  35. unsigned typeId; // token type this matcher recognizes
  36. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  37. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  38. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  39. } cmLexMatcher;
  40. typedef struct cmLex_str
  41. {
  42. cmErr_t err;
  43. const cmChar_t* cp; // character buffer
  44. unsigned cn; // count of characters in buffer
  45. unsigned ci; // current buffer index position
  46. unsigned flags; // lexer control flags
  47. unsigned curTokenId; // type id of the current token
  48. unsigned curTokenCharIdx; // index into cp[] of the current token
  49. unsigned curTokenCharCnt; // count of characters in the current token
  50. unsigned curLine; // line number of the current token
  51. unsigned curCol; // column number of the current token
  52. unsigned nextLine;
  53. unsigned nextCol;
  54. cmChar_t* blockBegCmtStr;
  55. cmChar_t* blockEndCmtStr;
  56. cmChar_t* lineCmtStr;
  57. cmLexMatcher* mfp; // base of matcher array
  58. unsigned mfi; // next available matcher array slot
  59. unsigned mfn; // count of elementes in mfp[]
  60. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  61. } cmLex;
  62. cmLexH cmLexNullH = { NULL };
  63. bool _cmLexIsNewline( cmChar_t c )
  64. { return c == '\n'; }
  65. bool _cmLexIsCommentTypeId( unsigned typeId )
  66. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  67. cmLex* _cmLexHandleToPtr( cmLexH h )
  68. {
  69. cmLex* p = (cmLex*)h.h;
  70. assert(p != NULL);
  71. return p;
  72. };
  73. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  74. {
  75. va_list vl;
  76. va_start(vl,fmt);
  77. unsigned bufCharCnt = 512;
  78. char buf[ bufCharCnt+1 ];
  79. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  80. unsigned sn = strlen(buf);
  81. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  82. buf[bufCharCnt]=0;
  83. cmErrMsg(&p->err,rc,"%s",buf);
  84. va_end(vl);
  85. return rc;
  86. }
  87. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  88. {
  89. unsigned i = 0;
  90. unsigned n = strlen(keyStr);
  91. if( n <= cn )
  92. for(; i<=cn-n; ++i)
  93. if( strncmp(cp + i, keyStr, n ) == 0 )
  94. return i+n;
  95. return cmInvalidIdx;
  96. }
  97. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  98. {
  99. unsigned n = strlen(keyStr);
  100. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  101. }
  102. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  103. {
  104. unsigned i=0;
  105. for(; i<cn; ++i)
  106. if( !isspace(cp[i]) )
  107. break;
  108. return i;
  109. }
  110. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  111. {
  112. unsigned i = 0;
  113. unsigned n = 0; // decimal point counter
  114. unsigned d = 0; // digit counter
  115. bool fl = false; // true if this real includes an exponent
  116. for(; i<cn && n<=1; ++i)
  117. {
  118. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  119. continue;
  120. if( isdigit(cp[i]) ) // allow digits
  121. {
  122. ++d;
  123. continue;
  124. }
  125. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  126. ++n;
  127. else
  128. break;
  129. }
  130. // if there was at least one digit and the next char is an 'e'
  131. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  132. {
  133. d=0;
  134. ++i;
  135. unsigned j = i;
  136. for(; i<cn; ++i)
  137. {
  138. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  139. continue;
  140. if( isdigit(cp[i]) )
  141. {
  142. ++d;
  143. continue;
  144. }
  145. // stop at the first non-digit
  146. break;
  147. }
  148. // an exp exists if digits follwed the 'e'
  149. fl = d > 0;
  150. }
  151. return i>1 && (n==1 || fl) ? i : 0;
  152. }
  153. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  154. {
  155. unsigned i = 0;
  156. for(; i<cn; ++i)
  157. {
  158. if( i==0 && cp[i]=='-' )
  159. continue;
  160. if( !isdigit(cp[i]) )
  161. break;
  162. }
  163. // BUG BUG BUG
  164. // If an integer is specified using 'e' notiation
  165. // (see _cmLexRealMatcher()) and the number of exponent places
  166. // specified following the 'e' is positive and >= number of
  167. // digits following the decimal point (in effect zeros are
  168. // padded on the right side) then the value is an integer.
  169. //
  170. // The current implementation recognizes all numeric strings
  171. // containing a decimal point as reals.
  172. return i;
  173. }
  174. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  175. {
  176. unsigned i = 0;
  177. if( cn < 3 )
  178. return 0;
  179. if( cp[0]=='0' && cp[1]=='x')
  180. for(i=2; i<cn; ++i)
  181. if( !isxdigit(cp[i]) )
  182. break;
  183. return i;
  184. }
  185. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  186. {
  187. unsigned i = 0;
  188. if( isalpha(cp[0]) || (cp[0]== '_'))
  189. {
  190. i = 1;
  191. for(; i<cn; ++i)
  192. if( !isalnum(cp[i]) && (cp[i] != '_') )
  193. break;
  194. }
  195. return i;
  196. }
  197. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  198. {
  199. cmChar_t qStr[]="\"";
  200. unsigned n = strlen(qStr);
  201. if( strncmp(qStr,cp,n) == 0 )
  202. {
  203. unsigned i;
  204. if((i = _cmLexScanTo(cp+n, cn-n, qStr)) == cmInvalidIdx )
  205. {
  206. _cmLexError( p, kMissingEndQuoteLexRC, "Missing string end quote.");
  207. return 0;
  208. }
  209. return n+i;
  210. }
  211. return 0;
  212. }
  213. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  214. {
  215. unsigned n = strlen(p->blockBegCmtStr);
  216. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  217. {
  218. unsigned i;
  219. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  220. {
  221. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  222. return 0;
  223. }
  224. return n + i;
  225. }
  226. return 0;
  227. }
  228. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  229. {
  230. unsigned n = strlen(p->lineCmtStr);
  231. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  232. {
  233. unsigned i;
  234. const char newlineStr[] = "\n";
  235. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  236. {
  237. // no EOL was found so the comment must be on the last line of the source
  238. return cn;
  239. }
  240. return n + i;
  241. }
  242. return 0;
  243. }
  244. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  245. {
  246. assert( funcPtr==NULL || userPtr==NULL );
  247. assert( !(funcPtr==NULL && userPtr==NULL));
  248. // if there is no space in the user token array - then expand it
  249. if( p->mfi == p->mfn )
  250. {
  251. int incr_cnt = 10;
  252. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  253. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  254. cmMemPtrFree(&p->mfp);
  255. p->mfp = np;
  256. p->mfn += incr_cnt;
  257. }
  258. p->mfp[p->mfi].tokenStr = NULL;
  259. p->mfp[p->mfi].typeId = typeId;
  260. p->mfp[p->mfi].funcPtr = funcPtr;
  261. p->mfp[p->mfi].userPtr = userPtr;
  262. if( keyStr != NULL )
  263. {
  264. // allocate space for the token string and store it
  265. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  266. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  267. }
  268. p->mfi++;
  269. return kOkLexRC;
  270. }
  271. cmRC_t _cmLexReset( cmLex* p )
  272. {
  273. p->ci = 0;
  274. p->curTokenId = kErrorLexTId;
  275. p->curTokenCharIdx = cmInvalidIdx;
  276. p->curTokenCharCnt = 0;
  277. p->curLine = 0;
  278. p->curCol = 0;
  279. p->nextLine = 0;
  280. p->nextCol = 0;
  281. cmErrClearRC(&p->err);
  282. return kOkLexRC;
  283. }
  284. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  285. {
  286. p->cp = cp;
  287. p->cn = cn;
  288. return _cmLexReset(p);
  289. }
  290. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  291. {
  292. cmLexH h;
  293. cmChar_t dfltLineCmt[] = "//";
  294. cmChar_t dfltBlockBegCmt[] = "/*";
  295. cmChar_t dfltBlockEndCmt[] = "*/";
  296. cmLex* p = cmMemAllocZ( cmLex, 1 );
  297. cmErrSetup(&p->err,rpt,"Lexer");
  298. p->flags = flags;
  299. _cmLexSetTextBuffer( p, cp, cn );
  300. /*
  301. p->cp = (cn==0) ? NULL : cp;
  302. p->cn = (cp==NULL) ? 0 : cn;
  303. p->ci = 0;
  304. p->curTokenId = kErrorLexTId;
  305. p->curTokenCharIdx = cmInvalidIdx;
  306. p->curTokenCharCnt = 0;
  307. p->curLine = 0;
  308. p->curCol = 0;
  309. p->nextLine = 0;
  310. p->nextCol = 0;
  311. */
  312. int init_mfn = 10;
  313. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  314. p->mfn = init_mfn;
  315. p->mfi = 0;
  316. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  317. strcpy( p->lineCmtStr, dfltLineCmt );
  318. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  319. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  320. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  321. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  322. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  323. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  324. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  325. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  326. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  327. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  328. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  329. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  330. h.h = p;
  331. _cmLexReset(p);
  332. return h;
  333. }
  334. cmRC_t cmLexFinal( cmLexH* hp )
  335. {
  336. if( hp == NULL || cmLexIsValid(*hp)==false )
  337. return cmOkRC;
  338. cmLex* p = _cmLexHandleToPtr(*hp);
  339. if( p != NULL )
  340. {
  341. if( p->mfp != NULL )
  342. {
  343. unsigned i = 0;
  344. // free the user token strings
  345. for(; i<p->mfi; ++i)
  346. if( p->mfp[i].tokenStr != NULL )
  347. cmMemPtrFree(&p->mfp[i].tokenStr);
  348. // free the matcher array
  349. cmMemPtrFree(&p->mfp);
  350. p->mfi = 0;
  351. p->mfn = 0;
  352. }
  353. cmMemPtrFree(&p->lineCmtStr);
  354. cmMemPtrFree(&p->blockBegCmtStr);
  355. cmMemPtrFree(&p->blockEndCmtStr);
  356. cmMemPtrFree(&p->textBuf);
  357. // free the lexer object
  358. cmMemPtrFree(&p);
  359. hp->h = NULL;
  360. }
  361. return kOkLexRC;
  362. }
  363. cmRC_t cmLexReset( cmLexH h )
  364. {
  365. cmLex* p = _cmLexHandleToPtr(h);
  366. return _cmLexReset(p);
  367. }
  368. bool cmLexIsValid( cmLexH h )
  369. { return h.h != NULL; }
  370. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  371. {
  372. cmLex* p = _cmLexHandleToPtr(h);
  373. return _cmLexSetTextBuffer(p,cp,cn);
  374. }
  375. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  376. {
  377. cmRC_t rc = kOkLexRC;
  378. cmFileH_t fh = cmFileNullHandle;
  379. cmLex* p = _cmLexHandleToPtr(h);
  380. long n = 0;
  381. assert( fn != NULL && p != NULL );
  382. // open the file
  383. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  384. return kFileOpenErrLexRC;
  385. // seek to the end of the file
  386. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  387. return kFileSeekErrLexRC;
  388. // get the length of the file
  389. if( cmFileTell(fh,&n) != kOkFileRC )
  390. return kFileTellErrLexRC;
  391. // rewind to the beginning of the file
  392. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  393. return kFileSeekErrLexRC;
  394. // allocate the text buffer
  395. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  396. {
  397. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  398. goto errLabel;
  399. }
  400. // read the file into the buffer
  401. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  402. return kFileReadErrLexRC;
  403. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  404. goto errLabel;
  405. errLabel:
  406. // close the file
  407. if( cmFileClose(&fh) != kOkFileRC )
  408. return kFileCloseErrLexRC;
  409. return rc;
  410. }
  411. /*
  412. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  413. {
  414. cmRC_t rc = kOkLexRC;
  415. FILE* fp = NULL;
  416. cmLex* p = _cmLexHandleToPtr(h);
  417. unsigned n = 0;
  418. assert( fn != NULL && p != NULL );
  419. // open the file
  420. if((fp = fopen(fn,"rb")) == NULL )
  421. return _cmLexError(p,kFileOpenErrLexRC,"Unable to open the file:'%s'.",fn);
  422. // seek to the end
  423. if( fseek(fp,0,SEEK_END) != 0 )
  424. {
  425. rc= _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the end of '%s'.",fn);
  426. goto errLabel;
  427. }
  428. // get the length of the file
  429. if( (n=ftell(fp)) == 0 )
  430. {
  431. rc = _cmLexError(p,kFileOpenErrLexRC,"The file '%s' appears to be empty.",fn);
  432. goto errLabel;
  433. }
  434. // rewind the file
  435. if( fseek(fp,0,SEEK_SET) != 0 )
  436. {
  437. rc = _cmLexError(p,kFileSeekErrLexRC,"Unable to seek to the beginning of '%s'.",fn);
  438. goto errLabel;
  439. }
  440. // allocate the text buffer
  441. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  442. {
  443. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  444. goto errLabel;
  445. }
  446. // read the file into the text buffer
  447. if( fread(p->textBuf,n,1,fp) != 1 )
  448. {
  449. rc = _cmLexError(p,kFileReadErrLexRC,"File read failed on:'%s'.",fn);
  450. goto errLabel;
  451. }
  452. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  453. goto errLabel;
  454. errLabel:
  455. // close the file
  456. if( fclose(fp) != 0 )
  457. {
  458. rc = _cmLexError(p,kFileCloseErrLexRC,"File close failed on:'%s'.",fn);
  459. goto errLabel;
  460. }
  461. return rc;
  462. }
  463. */
  464. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  465. {
  466. unsigned i = 0;
  467. for(; i<p->mfi; ++i)
  468. {
  469. if( id != cmInvalidId && p->mfp[i].typeId == id )
  470. return p->mfp + i;
  471. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  472. return p->mfp + i;
  473. }
  474. return NULL;
  475. }
  476. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  477. {
  478. cmLex* p = _cmLexHandleToPtr(h);
  479. // prevent duplicate tokens
  480. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  481. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  482. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  483. }
  484. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  485. {
  486. cmLex* p = _cmLexHandleToPtr(h);
  487. // prevent duplicate tokens
  488. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  489. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  490. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  491. }
  492. unsigned cmLexGetNextToken( cmLexH h )
  493. {
  494. cmLex* p = _cmLexHandleToPtr(h);
  495. if( cmErrLastRC(&p->err) != kOkLexRC )
  496. return kErrorLexTId;
  497. while( p->ci < p->cn )
  498. {
  499. unsigned i;
  500. unsigned mi = 0;
  501. unsigned maxCharCnt = 0;
  502. unsigned maxIdx = cmInvalidIdx;
  503. p->curTokenId = kErrorLexTId;
  504. p->curTokenCharIdx = cmInvalidIdx;
  505. p->curTokenCharCnt = 0;
  506. for(; mi<p->mfi; ++mi)
  507. {
  508. unsigned charCnt = 0;
  509. if( p->mfp[mi].funcPtr != NULL )
  510. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  511. else
  512. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  513. if( cmErrLastRC(&p->err) != kOkLexRC )
  514. return kErrorLexTId;
  515. // if this matched token is longer then the prev. matched token or
  516. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  517. if( (charCnt > maxCharCnt) || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId ) )
  518. {
  519. maxCharCnt = charCnt;
  520. maxIdx = mi;
  521. }
  522. }
  523. // no token was matched
  524. if( maxIdx == cmInvalidIdx )
  525. {
  526. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  527. return kErrorLexTId;
  528. }
  529. // update the current line and column position
  530. p->curLine = p->nextLine;
  531. p->curCol = p->nextCol;
  532. // find the next column and line position
  533. for(i=0; i<maxCharCnt; ++i)
  534. {
  535. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  536. {
  537. p->nextLine++;
  538. p->nextCol = 1;
  539. }
  540. else
  541. p->nextCol++;
  542. }
  543. bool returnFl = true;
  544. // check the space token filter
  545. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  546. returnFl = false;
  547. // check the comment token filter
  548. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  549. returnFl = false;
  550. // update the lexer state
  551. p->curTokenId = p->mfp[ maxIdx ].typeId;
  552. p->curTokenCharIdx = p->ci;
  553. p->curTokenCharCnt = maxCharCnt;
  554. // advance the text buffer
  555. p->ci += maxCharCnt;
  556. if( returnFl )
  557. return p->curTokenId;
  558. }
  559. cmErrSetRC(&p->err,kEofRC);
  560. return kEofLexTId;
  561. }
  562. unsigned cmLexTokenId( cmLexH h )
  563. {
  564. cmLex* p = _cmLexHandleToPtr(h);
  565. return p->curTokenId;
  566. }
  567. const cmChar_t* cmLexTokenText( cmLexH h )
  568. {
  569. cmLex* p = _cmLexHandleToPtr(h);
  570. if( p->curTokenCharIdx == cmInvalidIdx )
  571. return NULL;
  572. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  573. return p->cp + p->curTokenCharIdx + n;
  574. }
  575. unsigned cmLexTokenCharCount( cmLexH h )
  576. {
  577. cmLex* p = _cmLexHandleToPtr(h);
  578. if( p->curTokenCharIdx == cmInvalidIdx )
  579. return 0;
  580. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  581. return p->curTokenCharCnt - n;
  582. }
  583. int cmLexTokenInt( cmLexH h )
  584. { return strtol( cmLexTokenText(h),NULL,0 ); }
  585. unsigned cmLexTokenUInt( cmLexH h )
  586. { return strtol( cmLexTokenText(h),NULL,0 ); }
  587. float cmLexTokenFloat( cmLexH h )
  588. { return strtof( cmLexTokenText(h),NULL ); }
  589. double cmLexTokenDouble( cmLexH h )
  590. { return strtod( cmLexTokenText(h),NULL ); }
  591. unsigned cmLexCurrentLineNumber( cmLexH h )
  592. {
  593. cmLex* p = _cmLexHandleToPtr(h);
  594. return p->curLine + 1;
  595. }
  596. unsigned cmLexCurrentColumnNumber( cmLexH h )
  597. {
  598. cmLex* p = _cmLexHandleToPtr(h);
  599. return p->curCol + 1;
  600. }
  601. unsigned cmLexErrorRC( cmLexH h )
  602. {
  603. cmLex* p = _cmLexHandleToPtr(h);
  604. return cmErrLastRC(&p->err);
  605. }
  606. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  607. {
  608. cmLex* p = _cmLexHandleToPtr(h);
  609. switch( typeId )
  610. {
  611. case kErrorLexTId: return "<error>";
  612. case kEofLexTId: return "<EOF>";
  613. case kSpaceLexTId: return "<space>";
  614. case kRealLexTId: return "<real>";
  615. case kIntLexTId: return "<int>";
  616. case kHexLexTId: return "<hex>";
  617. case kIdentLexTId: return "<ident>";
  618. case kQStrLexTId: return "<qstr>";
  619. case kBlockCmtLexTId: return "<bcmt>";
  620. case kLineCmtLexTId: return "<lcmt>";
  621. default:
  622. {
  623. cmLexMatcher* mp;
  624. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  625. return "<unknown>";
  626. return mp->tokenStr;
  627. }
  628. }
  629. return "<invalid>";
  630. }
  631. const cmChar_t* cmLexRcToMsg( unsigned rc )
  632. {
  633. unsigned i=0;
  634. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  635. if( cmLexErrorArray[i].code == rc )
  636. break;
  637. return cmLexErrorArray[i].msg;
  638. }
  639. //{ { label:cmLexEx }
  640. //(
  641. // cmLexTest() gives a simple cmLex example.
  642. //)
  643. //(
  644. void cmLexTest( cmRpt_t* rpt)
  645. {
  646. cmChar_t buf[] =
  647. "123ident0\n 123.456\nident0\n"
  648. "0xa12+.2\n"
  649. "// comment \n"
  650. "/* block \n"
  651. "comment */"
  652. "\"quoted string\""
  653. "ident1"
  654. "// last line comment";
  655. // initialize a lexer with a buffer of text
  656. cmLexH h = cmLexInit(buf,strlen(buf),
  657. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  658. // verify that the lexer initialization succeded.
  659. if( cmLexIsValid(h) == false )
  660. {
  661. cmRptPrintf(rpt,"Lexer initialization failed.");
  662. return;
  663. }
  664. // register some additional recoginizers
  665. cmLexRegisterToken(h,kUserLexTId+1,"+");
  666. cmLexRegisterToken(h,kUserLexTId+2,"-");
  667. unsigned tid;
  668. // ask for token id's
  669. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  670. {
  671. // print information about each token
  672. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  673. cmLexCurrentLineNumber(h),
  674. cmLexCurrentColumnNumber(h),
  675. cmLexIdToLabel(h,tid),
  676. cmLexTokenCharCount(h),
  677. cmLexTokenText(h) ,
  678. cmLexTokenCharCount(h));
  679. // if the token is a number ...
  680. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  681. {
  682. // ... then request the numbers value
  683. int iv = cmLexTokenInt(h);
  684. double dv = cmLexTokenDouble(h);
  685. cmRptPrintf(rpt,"%i %f",iv,dv);
  686. }
  687. cmRptPrintf(rpt,"\n");
  688. // handle errors
  689. if( tid == kErrorLexTId )
  690. {
  691. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  692. break;
  693. }
  694. }
  695. // finalize the lexer
  696. cmLexFinal(&h);
  697. }
  698. //)
  699. //}