libcm is a C development framework with an emphasis on audio signal processing applications.
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

cmLex.c 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970
  1. #include "cmPrefix.h"
  2. #include "cmGlobal.h"
  3. #include "cmRpt.h"
  4. #include "cmLex.h"
  5. #include "cmErr.h"
  6. #include "cmMem.h"
  7. #include "cmMallocDebug.h"
  8. #include "cmFile.h"
  9. enum
  10. {
  11. kRealFloatLexFl = 0x01,
  12. kIntUnsignedLexFl = 0x02
  13. };
  14. typedef struct
  15. {
  16. unsigned code;
  17. const cmChar_t* msg;
  18. } cmLexErrorRecd;
  19. cmLexErrorRecd cmLexErrorArray[] =
  20. {
  21. { kOkLexRC, "No error. The operation completed successfully."},
  22. { kDuplicateTokenLexRC, "The text or id passed as a user token is already in use by another token."},
  23. { kMissingCmtEndLexRC, "The end of a block comment could not be found."},
  24. { kMissingEndQuoteLexRC, "The end of a quoted string could not be found."},
  25. { kNoMatchLexRC, "The lexer encountered a string which could not be classified."},
  26. { kFileOpenErrLexRC, "File open failed on cmLexSetFile()"},
  27. { kFileSeekErrLexRC, "File seek failed on cmLexSetFile()"},
  28. { kFileTellErrLexRC, "File tell failed on cmLexSetFile()"},
  29. { kFileReadErrLexRC, "File read failed on cmLexSetFile()"},
  30. { kFileCloseErrLexRC, "File close failed on cmLexSetFile()"},
  31. { kMemAllocErrLexRC, "An attempted memory allocation failed"},
  32. { kEofRC, "The end of the input text was encountered (this is a normal condition not an error)"},
  33. { kInvalidLexTIdLexRC, "An invalid token id was encountered."},
  34. { kSignErrorLexRC, "A signed integer has a 'u' or 'U' suffix."},
  35. { kInvalidLexRC, "Unknown lexer error code." }
  36. };
  37. struct cmLex_str;
  38. typedef unsigned (*cmLexMatcherFuncPtr_t)( struct cmLex_str* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr );
  39. // token match function record
  40. typedef struct
  41. {
  42. unsigned typeId; // token type this matcher recognizes
  43. cmLexMatcherFuncPtr_t funcPtr; // recognizer function (only used if userPtr==NULL)
  44. cmChar_t* tokenStr; // fixed string data used by the recognizer (only used if userPtr==NULL)
  45. cmLexUserMatcherPtr_t userPtr; // user defined recognizer function (only used if funcPtr==NULL)
  46. bool enableFl; // true if this matcher is enabled
  47. } cmLexMatcher;
  48. typedef struct cmLex_str
  49. {
  50. cmErr_t err;
  51. const cmChar_t* cp; // character buffer
  52. unsigned cn; // count of characters in buffer
  53. unsigned ci; // current buffer index position
  54. unsigned flags; // lexer control flags
  55. unsigned curTokenId; // type id of the current token
  56. unsigned curTokenCharIdx; // index into cp[] of the current token
  57. unsigned curTokenCharCnt; // count of characters in the current token
  58. unsigned curLine; // line number of the current token
  59. unsigned curCol; // column number of the current token
  60. unsigned nextLine;
  61. unsigned nextCol;
  62. cmChar_t* blockBegCmtStr;
  63. cmChar_t* blockEndCmtStr;
  64. cmChar_t* lineCmtStr;
  65. cmLexMatcher* mfp; // base of matcher array
  66. unsigned mfi; // next available matcher array slot
  67. unsigned mfn; // count of elementes in mfp[]
  68. cmChar_t* textBuf; // text buf used by cmLexSetFile()
  69. unsigned attrFlags; // used to store the int and real suffix type flags
  70. } cmLex;
  71. cmLexH cmLexNullH = { NULL };
  72. bool _cmLexIsNewline( cmChar_t c )
  73. { return c == '\n'; }
  74. bool _cmLexIsCommentTypeId( unsigned typeId )
  75. { return typeId == kBlockCmtLexTId || typeId == kLineCmtLexTId; }
  76. cmLex* _cmLexHandleToPtr( cmLexH h )
  77. {
  78. cmLex* p = (cmLex*)h.h;
  79. assert(p != NULL);
  80. return p;
  81. };
  82. cmRC_t _cmLexError( cmLex* p, unsigned rc, const char* fmt, ... )
  83. {
  84. va_list vl;
  85. va_start(vl,fmt);
  86. unsigned bufCharCnt = 512;
  87. char buf[ bufCharCnt+1 ];
  88. snprintf(buf,bufCharCnt,"Error on line:%i ", p->curLine);
  89. unsigned sn = strlen(buf);
  90. vsnprintf(buf+sn,bufCharCnt-sn,fmt,vl);
  91. buf[bufCharCnt]=0;
  92. cmErrMsg(&p->err,rc,"%s",buf);
  93. va_end(vl);
  94. return rc;
  95. }
  96. // Locate 'keyStr' in cp[cn] and return the index into cp[cn] of the character
  97. // following the last char in 'keyStr'. If keyStr is not found return cmInvalidIdx.
  98. unsigned _cmLexScanTo( const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  99. {
  100. unsigned i = 0;
  101. unsigned n = strlen(keyStr);
  102. if( n <= cn )
  103. for(; i<=cn-n; ++i)
  104. if( strncmp(cp + i, keyStr, n ) == 0 )
  105. return i+n;
  106. return cmInvalidIdx;
  107. }
  108. unsigned _cmLexExactStringMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  109. {
  110. unsigned n = strlen(keyStr);
  111. return strncmp(keyStr,cp,n) == 0 ? n : 0;
  112. }
  113. unsigned _cmLexSpaceMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  114. {
  115. unsigned i=0;
  116. for(; i<cn; ++i)
  117. if( !isspace(cp[i]) )
  118. break;
  119. return i;
  120. }
  121. unsigned _cmLexRealMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  122. {
  123. unsigned i = 0;
  124. unsigned n = 0; // decimal point counter
  125. unsigned d = 0; // digit counter
  126. bool fl = false; // expo flag
  127. for(; i<cn && n<=1; ++i)
  128. {
  129. if( i==0 && cp[i]=='-' ) // allow a leading '-'
  130. continue;
  131. if( isdigit(cp[i]) ) // allow digits
  132. {
  133. ++d;
  134. continue;
  135. }
  136. if( cp[i] == '.' && n==0 ) // allow exactly one decimal point
  137. ++n;
  138. else
  139. break;
  140. }
  141. // if there was at least one digit and the next char is an 'e'
  142. if( d>0 && i<cn && (cp[i] == 'e' || cp[i] == 'E') )
  143. {
  144. unsigned e=0;
  145. ++i;
  146. unsigned j = i;
  147. fl = false;
  148. for(; i<cn; ++i)
  149. {
  150. if( i==j && cp[i]=='-' ) // allow the char following the e to be '-'
  151. continue;
  152. if( isdigit(cp[i]) )
  153. {
  154. ++e;
  155. ++d;
  156. continue;
  157. }
  158. // stop at the first non-digit
  159. break;
  160. }
  161. // an exp exists if digits follwed the 'e'
  162. fl = e > 0;
  163. }
  164. // if at least one digit was found
  165. if( d>0 )
  166. {
  167. // Note that this path allows a string w/o a decimal pt to trigger a match.
  168. if(i<cn)
  169. {
  170. // if the real has a suffix
  171. switch(cp[i])
  172. {
  173. case 'F':
  174. case 'f':
  175. p->attrFlags = cmSetFlag(p->attrFlags,kRealFloatLexFl);
  176. ++i;
  177. break;
  178. }
  179. }
  180. // match w/o suffix return
  181. if( d>0 && (fl || n==1 || cmIsFlag(p->attrFlags,kRealFloatLexFl)) )
  182. return i;
  183. }
  184. return 0; // no-match return
  185. }
  186. unsigned _cmLexIntMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  187. {
  188. unsigned i = 0;
  189. bool signFl = false;
  190. for(; i<cn; ++i)
  191. {
  192. if( i==0 && cp[i]=='-' )
  193. {
  194. signFl = true;
  195. continue;
  196. }
  197. if( !isdigit(cp[i]) )
  198. break;
  199. }
  200. // BUG BUG BUG
  201. // If an integer is specified using 'e' notiation
  202. // (see _cmLexRealMatcher()) and the number of exponent places
  203. // specified following the 'e' is positive and >= number of
  204. // digits following the decimal point (in effect zeros are
  205. // padded on the right side) then the value is an integer.
  206. //
  207. // The current implementation recognizes all numeric strings
  208. // containing a decimal point as reals.
  209. // if no integer was found
  210. if( (signFl && i==0) || i==0 )
  211. return 0;
  212. // check for suffix
  213. if(i<cn )
  214. {
  215. switch(cp[i])
  216. {
  217. case 'u':
  218. case 'U':
  219. if( signFl )
  220. _cmLexError(p,kSignErrorLexRC,"A signed integer has a 'u' or 'U' suffix.");
  221. else
  222. {
  223. p->attrFlags = cmSetFlag(p->attrFlags,kIntUnsignedLexFl);
  224. ++i;
  225. }
  226. break;
  227. default:
  228. break;
  229. }
  230. }
  231. return i;
  232. }
  233. unsigned _cmLexHexMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  234. {
  235. unsigned i = 0;
  236. if( cn < 3 )
  237. return 0;
  238. if( cp[0]=='0' && cp[1]=='x')
  239. for(i=2; i<cn; ++i)
  240. if( !isxdigit(cp[i]) )
  241. break;
  242. return i;
  243. }
  244. unsigned _cmLexIdentMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  245. {
  246. unsigned i = 0;
  247. if( isalpha(cp[0]) || (cp[0]== '_'))
  248. {
  249. i = 1;
  250. for(; i<cn; ++i)
  251. if( !isalnum(cp[i]) && (cp[i] != '_') )
  252. break;
  253. }
  254. return i;
  255. }
  256. unsigned _cmLexQStrMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  257. {
  258. cmChar_t qStr[]="\"";
  259. unsigned n = strlen(qStr);
  260. if( strncmp(qStr,cp,n) == 0 )
  261. {
  262. unsigned i;
  263. if((i = _cmLexScanTo(cp+n, cn-n, qStr)) == cmInvalidIdx )
  264. {
  265. _cmLexError( p, kMissingEndQuoteLexRC, "Missing string end quote.");
  266. return 0;
  267. }
  268. return n+i;
  269. }
  270. return 0;
  271. }
  272. unsigned _cmLexQCharMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  273. {
  274. unsigned i = 0;
  275. if( i >= cn || cp[i]!='\'' )
  276. return 0;
  277. i+=2;
  278. if( i >= cn || cp[i]!='\'')
  279. return 0;
  280. return 3;
  281. }
  282. unsigned _cmLexBlockCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  283. {
  284. unsigned n = strlen(p->blockBegCmtStr);
  285. if( strncmp( p->blockBegCmtStr, cp, n ) == 0 )
  286. {
  287. unsigned i;
  288. if((i = _cmLexScanTo(cp + n, cn-n,p->blockEndCmtStr)) == cmInvalidIdx )
  289. {
  290. _cmLexError(p, kMissingCmtEndLexRC, "Missing end of block comment.");
  291. return 0;
  292. }
  293. return n + i;
  294. }
  295. return 0;
  296. }
  297. unsigned _cmLexLineCmtMatcher( cmLex* p, const cmChar_t* cp, unsigned cn, const cmChar_t* keyStr )
  298. {
  299. unsigned n = strlen(p->lineCmtStr);
  300. if( strncmp( p->lineCmtStr, cp, n ) == 0)
  301. {
  302. unsigned i;
  303. const char newlineStr[] = "\n";
  304. if((i = _cmLexScanTo(cp + n, cn-n, newlineStr)) == cmInvalidIdx )
  305. {
  306. // no EOL was found so the comment must be on the last line of the source
  307. return cn;
  308. }
  309. return n + i;
  310. }
  311. return 0;
  312. }
  313. cmRC_t _cmLexInstallMatcher( cmLex* p, unsigned typeId, cmLexMatcherFuncPtr_t funcPtr, const cmChar_t* keyStr, cmLexUserMatcherPtr_t userPtr )
  314. {
  315. assert( funcPtr==NULL || userPtr==NULL );
  316. assert( !(funcPtr==NULL && userPtr==NULL));
  317. // if there is no space in the user token array - then expand it
  318. if( p->mfi == p->mfn )
  319. {
  320. int incr_cnt = 10;
  321. cmLexMatcher* np = cmMemAllocZ( cmLexMatcher, p->mfn + incr_cnt );
  322. memcpy(np,p->mfp,p->mfi*sizeof(cmLexMatcher));
  323. cmMemPtrFree(&p->mfp);
  324. p->mfp = np;
  325. p->mfn += incr_cnt;
  326. }
  327. p->mfp[p->mfi].tokenStr = NULL;
  328. p->mfp[p->mfi].typeId = typeId;
  329. p->mfp[p->mfi].funcPtr = funcPtr;
  330. p->mfp[p->mfi].userPtr = userPtr;
  331. p->mfp[p->mfi].enableFl = true;
  332. if( keyStr != NULL )
  333. {
  334. // allocate space for the token string and store it
  335. p->mfp[p->mfi].tokenStr = cmMemAlloc( cmChar_t, sizeof(cmChar_t) * (strlen(keyStr)+1) );
  336. strcpy(p->mfp[p->mfi].tokenStr, keyStr );
  337. }
  338. p->mfi++;
  339. return kOkLexRC;
  340. }
  341. cmRC_t _cmLexReset( cmLex* p )
  342. {
  343. p->ci = 0;
  344. p->curTokenId = kErrorLexTId;
  345. p->curTokenCharIdx = cmInvalidIdx;
  346. p->curTokenCharCnt = 0;
  347. p->curLine = 0;
  348. p->curCol = 0;
  349. p->nextLine = 0;
  350. p->nextCol = 0;
  351. cmErrClearRC(&p->err);
  352. return kOkLexRC;
  353. }
  354. cmRC_t _cmLexSetTextBuffer( cmLex* p, const cmChar_t* cp, unsigned cn )
  355. {
  356. p->cp = cp;
  357. p->cn = cn;
  358. return _cmLexReset(p);
  359. }
  360. cmLexH cmLexInit( const cmChar_t* cp, unsigned cn, unsigned flags, cmRpt_t* rpt )
  361. {
  362. cmLexH h;
  363. cmChar_t dfltLineCmt[] = "//";
  364. cmChar_t dfltBlockBegCmt[] = "/*";
  365. cmChar_t dfltBlockEndCmt[] = "*/";
  366. cmLex* p = cmMemAllocZ( cmLex, 1 );
  367. cmErrSetup(&p->err,rpt,"Lexer");
  368. p->flags = flags;
  369. _cmLexSetTextBuffer( p, cp, cn );
  370. int init_mfn = 10;
  371. p->mfp = cmMemAllocZ( cmLexMatcher, init_mfn );
  372. p->mfn = init_mfn;
  373. p->mfi = 0;
  374. p->lineCmtStr = cmMemAlloc( cmChar_t, strlen(dfltLineCmt)+1 );
  375. strcpy( p->lineCmtStr, dfltLineCmt );
  376. p->blockBegCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockBegCmt)+1 );
  377. strcpy( p->blockBegCmtStr, dfltBlockBegCmt );
  378. p->blockEndCmtStr = cmMemAlloc( cmChar_t, strlen(dfltBlockEndCmt)+1 );
  379. strcpy( p->blockEndCmtStr, dfltBlockEndCmt );
  380. _cmLexInstallMatcher( p, kSpaceLexTId, _cmLexSpaceMatcher, NULL, NULL );
  381. _cmLexInstallMatcher( p, kRealLexTId, _cmLexRealMatcher, NULL, NULL );
  382. _cmLexInstallMatcher( p, kIntLexTId, _cmLexIntMatcher, NULL, NULL );
  383. _cmLexInstallMatcher( p, kHexLexTId, _cmLexHexMatcher, NULL, NULL );
  384. _cmLexInstallMatcher( p, kIdentLexTId, _cmLexIdentMatcher, NULL, NULL );
  385. _cmLexInstallMatcher( p, kQStrLexTId, _cmLexQStrMatcher, NULL, NULL );
  386. _cmLexInstallMatcher( p, kBlockCmtLexTId, _cmLexBlockCmtMatcher, NULL, NULL );
  387. _cmLexInstallMatcher( p, kLineCmtLexTId, _cmLexLineCmtMatcher, NULL, NULL );
  388. if( cmIsFlag(flags,kReturnQCharLexFl) )
  389. _cmLexInstallMatcher( p, kQCharLexTId, _cmLexQCharMatcher, NULL, NULL );
  390. h.h = p;
  391. _cmLexReset(p);
  392. return h;
  393. }
  394. cmRC_t cmLexFinal( cmLexH* hp )
  395. {
  396. if( hp == NULL || cmLexIsValid(*hp)==false )
  397. return cmOkRC;
  398. cmLex* p = _cmLexHandleToPtr(*hp);
  399. if( p != NULL )
  400. {
  401. if( p->mfp != NULL )
  402. {
  403. unsigned i = 0;
  404. // free the user token strings
  405. for(; i<p->mfi; ++i)
  406. if( p->mfp[i].tokenStr != NULL )
  407. cmMemPtrFree(&p->mfp[i].tokenStr);
  408. // free the matcher array
  409. cmMemPtrFree(&p->mfp);
  410. p->mfi = 0;
  411. p->mfn = 0;
  412. }
  413. cmMemPtrFree(&p->lineCmtStr);
  414. cmMemPtrFree(&p->blockBegCmtStr);
  415. cmMemPtrFree(&p->blockEndCmtStr);
  416. cmMemPtrFree(&p->textBuf);
  417. // free the lexer object
  418. cmMemPtrFree(&p);
  419. hp->h = NULL;
  420. }
  421. return kOkLexRC;
  422. }
  423. cmRC_t cmLexReset( cmLexH h )
  424. {
  425. cmLex* p = _cmLexHandleToPtr(h);
  426. return _cmLexReset(p);
  427. }
  428. bool cmLexIsValid( cmLexH h )
  429. { return h.h != NULL; }
  430. cmRC_t cmLexSetTextBuffer( cmLexH h, const cmChar_t* cp, unsigned cn )
  431. {
  432. cmLex* p = _cmLexHandleToPtr(h);
  433. return _cmLexSetTextBuffer(p,cp,cn);
  434. }
  435. cmRC_t cmLexSetFile( cmLexH h, const cmChar_t* fn )
  436. {
  437. cmRC_t rc = kOkLexRC;
  438. cmFileH_t fh = cmFileNullHandle;
  439. cmLex* p = _cmLexHandleToPtr(h);
  440. long n = 0;
  441. assert( fn != NULL && p != NULL );
  442. // open the file
  443. if( cmFileOpen(&fh,fn,kReadFileFl,p->err.rpt) != kOkFileRC )
  444. return kFileOpenErrLexRC;
  445. // seek to the end of the file
  446. if( cmFileSeek(fh,kEndFileFl,0) != kOkFileRC )
  447. return kFileSeekErrLexRC;
  448. // get the length of the file
  449. if( cmFileTell(fh,&n) != kOkFileRC )
  450. return kFileTellErrLexRC;
  451. // rewind to the beginning of the file
  452. if( cmFileSeek(fh,kBeginFileFl,0) != kOkFileRC )
  453. return kFileSeekErrLexRC;
  454. // allocate the text buffer
  455. if((p->textBuf = cmMemResizeZ( char, p->textBuf, n+1)) == NULL )
  456. {
  457. rc = _cmLexError(p,kMemAllocErrLexRC,"Unable to allocate the text file buffer for:'%s'.",fn);
  458. goto errLabel;
  459. }
  460. // read the file into the buffer
  461. if( cmFileRead(fh,p->textBuf,n) != kOkFileRC )
  462. return kFileReadErrLexRC;
  463. if((rc = _cmLexSetTextBuffer( p, p->textBuf, n )) != kOkLexRC )
  464. goto errLabel;
  465. errLabel:
  466. // close the file
  467. if( cmFileClose(&fh) != kOkFileRC )
  468. return kFileCloseErrLexRC;
  469. return rc;
  470. }
  471. cmLexMatcher* _cmLexFindUserToken( cmLex* p, unsigned id, const cmChar_t* tokenStr )
  472. {
  473. unsigned i = 0;
  474. for(; i<p->mfi; ++i)
  475. {
  476. if( id != cmInvalidId && p->mfp[i].typeId == id )
  477. return p->mfp + i;
  478. if( p->mfp[i].tokenStr != NULL && tokenStr != NULL && strcmp(p->mfp[i].tokenStr,tokenStr)==0 )
  479. return p->mfp + i;
  480. }
  481. return NULL;
  482. }
  483. cmRC_t cmLexRegisterToken( cmLexH h, unsigned id, const cmChar_t* tokenStr )
  484. {
  485. cmLex* p = _cmLexHandleToPtr(h);
  486. // prevent duplicate tokens
  487. if( _cmLexFindUserToken( p, id, tokenStr ) != NULL )
  488. return _cmLexError( p, kDuplicateTokenLexRC, "id:%i token:%s duplicates the token string or id", id, tokenStr );
  489. return _cmLexInstallMatcher( p, id, _cmLexExactStringMatcher, tokenStr, NULL );
  490. }
  491. cmRC_t cmLexRegisterMatcher( cmLexH h, unsigned id, cmLexUserMatcherPtr_t userPtr )
  492. {
  493. cmLex* p = _cmLexHandleToPtr(h);
  494. // prevent duplicate tokens
  495. if( _cmLexFindUserToken( p, id, NULL ) != NULL )
  496. return _cmLexError( p, kDuplicateTokenLexRC, "A token matching function has already been installed for token id: %i", id );
  497. return _cmLexInstallMatcher( p, id, NULL, NULL, userPtr );
  498. }
  499. cmRC_t cmLexEnableToken( cmLexH h, unsigned id, bool enableFl )
  500. {
  501. cmLex* p = _cmLexHandleToPtr(h);
  502. unsigned mi = 0;
  503. for(; mi<p->mfi; ++mi)
  504. if( p->mfp[mi].typeId == id )
  505. {
  506. p->mfp[mi].enableFl = enableFl;
  507. return cmOkRC;
  508. }
  509. return _cmLexError( p, kInvalidLexTIdLexRC, "%i is not a valid token type id.",id);
  510. }
  511. unsigned cmLexFilterFlags( cmLexH h )
  512. {
  513. cmLex* p = _cmLexHandleToPtr(h);
  514. return p->flags;
  515. }
  516. void cmLexSetFilterFlags( cmLexH h, unsigned flags )
  517. {
  518. cmLex* p = _cmLexHandleToPtr(h);
  519. p->flags = flags;
  520. }
  521. unsigned cmLexGetNextToken( cmLexH h )
  522. {
  523. cmLex* p = _cmLexHandleToPtr(h);
  524. if( cmErrLastRC(&p->err) != kOkLexRC )
  525. return kErrorLexTId;
  526. while( p->ci < p->cn )
  527. {
  528. unsigned i;
  529. unsigned mi = 0;
  530. unsigned maxCharCnt = 0;
  531. unsigned maxIdx = cmInvalidIdx;
  532. p->curTokenId = kErrorLexTId;
  533. p->curTokenCharIdx = cmInvalidIdx;
  534. p->curTokenCharCnt = 0;
  535. p->attrFlags = 0;
  536. // try each matcher
  537. for(; mi<p->mfi; ++mi)
  538. if( p->mfp[mi].enableFl )
  539. {
  540. unsigned charCnt = 0;
  541. if( p->mfp[mi].funcPtr != NULL )
  542. charCnt = p->mfp[mi].funcPtr(p, p->cp + p->ci, p->cn - p->ci, p->mfp[mi].tokenStr );
  543. else
  544. charCnt = p->mfp[mi].userPtr( p->cp + p->ci, p->cn - p->ci);
  545. // notice if the matcher set the error code
  546. if( cmErrLastRC(&p->err) != kOkLexRC )
  547. return kErrorLexTId;
  548. // if this matched token is longer then the prev. matched token or
  549. // if the prev matched token was an identifier and this matched token is an equal length user defined token
  550. if( (charCnt > maxCharCnt)
  551. || (charCnt>0 && charCnt==maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId )
  552. || (charCnt>0 && charCnt<maxCharCnt && p->mfp[maxIdx].typeId==kIdentLexTId && p->mfp[mi].typeId >=kUserLexTId && cmIsFlag(p->flags,kUserDefPriorityLexFl))
  553. )
  554. {
  555. maxCharCnt = charCnt;
  556. maxIdx = mi;
  557. }
  558. }
  559. // no token was matched
  560. if( maxIdx == cmInvalidIdx )
  561. {
  562. if( cmIsFlag(p->flags,kReturnUnknownLexFl) )
  563. {
  564. maxCharCnt = 1;
  565. }
  566. else
  567. {
  568. _cmLexError( p, kNoMatchLexRC, "Unable to recognize token:'%c'.",*(p->cp+p->ci));
  569. return kErrorLexTId;
  570. }
  571. }
  572. // update the current line and column position
  573. p->curLine = p->nextLine;
  574. p->curCol = p->nextCol;
  575. // find the next column and line position
  576. for(i=0; i<maxCharCnt; ++i)
  577. {
  578. if( _cmLexIsNewline(p->cp[ p->ci + i ]) )
  579. {
  580. p->nextLine++;
  581. p->nextCol = 1;
  582. }
  583. else
  584. p->nextCol++;
  585. }
  586. bool returnFl = true;
  587. if( maxIdx != cmInvalidIdx )
  588. {
  589. // check the space token filter
  590. if( (p->mfp[ maxIdx ].typeId == kSpaceLexTId) && (cmIsFlag(p->flags,kReturnSpaceLexFl)==0) )
  591. returnFl = false;
  592. // check the comment token filter
  593. if( _cmLexIsCommentTypeId(p->mfp[ maxIdx ].typeId) && (cmIsFlag(p->flags,kReturnCommentsLexFl)==0) )
  594. returnFl = false;
  595. }
  596. // update the lexer state
  597. p->curTokenId = maxIdx==cmInvalidIdx ? kUnknownLexTId : p->mfp[ maxIdx ].typeId;
  598. p->curTokenCharIdx = p->ci;
  599. p->curTokenCharCnt = maxCharCnt;
  600. // advance the text buffer
  601. p->ci += maxCharCnt;
  602. if( returnFl )
  603. return p->curTokenId;
  604. }
  605. cmErrSetRC(&p->err,kEofRC);
  606. return kEofLexTId;
  607. }
  608. unsigned cmLexTokenId( cmLexH h )
  609. {
  610. cmLex* p = _cmLexHandleToPtr(h);
  611. return p->curTokenId;
  612. }
  613. const cmChar_t* cmLexTokenText( cmLexH h )
  614. {
  615. cmLex* p = _cmLexHandleToPtr(h);
  616. if( p->curTokenCharIdx == cmInvalidIdx )
  617. return NULL;
  618. unsigned n = p->curTokenId == kQStrLexTId ? 1 : 0;
  619. return p->cp + p->curTokenCharIdx + n;
  620. }
  621. unsigned cmLexTokenCharCount( cmLexH h )
  622. {
  623. cmLex* p = _cmLexHandleToPtr(h);
  624. if( p->curTokenCharIdx == cmInvalidIdx )
  625. return 0;
  626. unsigned n = p->curTokenId == kQStrLexTId ? 2 : 0;
  627. return p->curTokenCharCnt - n;
  628. }
  629. int cmLexTokenInt( cmLexH h )
  630. { return strtol( cmLexTokenText(h),NULL,0 ); }
  631. unsigned cmLexTokenUInt( cmLexH h )
  632. { return strtol( cmLexTokenText(h),NULL,0 ); }
  633. float cmLexTokenFloat( cmLexH h )
  634. { return strtof( cmLexTokenText(h),NULL ); }
  635. double cmLexTokenDouble( cmLexH h )
  636. { return strtod( cmLexTokenText(h),NULL ); }
  637. bool cmLexTokenIsUnsigned( cmLexH h )
  638. {
  639. cmLex* p = _cmLexHandleToPtr(h);
  640. return p->curTokenId == kIntLexTId && cmIsFlag(p->attrFlags,kIntUnsignedLexFl);
  641. }
  642. bool cmLexTokenIsSinglePrecision( cmLexH h )
  643. {
  644. cmLex* p = _cmLexHandleToPtr(h);
  645. return p->curTokenId == kRealLexTId && cmIsFlag(p->attrFlags,kRealFloatLexFl);
  646. }
  647. unsigned cmLexCurrentLineNumber( cmLexH h )
  648. {
  649. cmLex* p = _cmLexHandleToPtr(h);
  650. return p->curLine + 1;
  651. }
  652. unsigned cmLexCurrentColumnNumber( cmLexH h )
  653. {
  654. cmLex* p = _cmLexHandleToPtr(h);
  655. return p->curCol + 1;
  656. }
  657. unsigned cmLexErrorRC( cmLexH h )
  658. {
  659. cmLex* p = _cmLexHandleToPtr(h);
  660. return cmErrLastRC(&p->err);
  661. }
  662. const cmChar_t* cmLexIdToLabel( cmLexH h, unsigned typeId )
  663. {
  664. cmLex* p = _cmLexHandleToPtr(h);
  665. switch( typeId )
  666. {
  667. case kErrorLexTId: return "<error>";
  668. case kEofLexTId: return "<EOF>";
  669. case kSpaceLexTId: return "<space>";
  670. case kRealLexTId: return "<real>";
  671. case kIntLexTId: return "<int>";
  672. case kHexLexTId: return "<hex>";
  673. case kIdentLexTId: return "<ident>";
  674. case kQStrLexTId: return "<qstr>";
  675. case kBlockCmtLexTId: return "<bcmt>";
  676. case kLineCmtLexTId: return "<lcmt>";
  677. default:
  678. {
  679. cmLexMatcher* mp;
  680. if((mp = _cmLexFindUserToken(p,typeId,NULL)) == NULL )
  681. return "<unknown>";
  682. return mp->tokenStr;
  683. }
  684. }
  685. return "<invalid>";
  686. }
  687. const cmChar_t* cmLexRcToMsg( unsigned rc )
  688. {
  689. unsigned i=0;
  690. for(i=0; cmLexErrorArray[i].code != kInvalidLexRC; ++i)
  691. if( cmLexErrorArray[i].code == rc )
  692. break;
  693. return cmLexErrorArray[i].msg;
  694. }
  695. //{ { label:cmLexEx }
  696. //(
  697. // cmLexTest() gives a simple cmLex example.
  698. //)
  699. //(
  700. void cmLexTest( cmRpt_t* rpt)
  701. {
  702. cmChar_t buf[] =
  703. "123ident0\n 123.456\nident0\n"
  704. "0xa12+.2\n"
  705. "// comment \n"
  706. "/* block \n"
  707. "comment */"
  708. "\"quoted string\""
  709. "ident1"
  710. "// last line comment";
  711. // initialize a lexer with a buffer of text
  712. cmLexH h = cmLexInit(buf,strlen(buf),
  713. kReturnSpaceLexFl | kReturnCommentsLexFl,rpt);
  714. // verify that the lexer initialization succeded.
  715. if( cmLexIsValid(h) == false )
  716. {
  717. cmRptPrintf(rpt,"Lexer initialization failed.");
  718. return;
  719. }
  720. // register some additional recoginizers
  721. cmLexRegisterToken(h,kUserLexTId+1,"+");
  722. cmLexRegisterToken(h,kUserLexTId+2,"-");
  723. unsigned tid;
  724. // ask for token id's
  725. while( (tid = cmLexGetNextToken(h)) != kEofLexTId )
  726. {
  727. // print information about each token
  728. cmRptPrintf(rpt,"%i %i %s '%.*s' (%i) ",
  729. cmLexCurrentLineNumber(h),
  730. cmLexCurrentColumnNumber(h),
  731. cmLexIdToLabel(h,tid),
  732. cmLexTokenCharCount(h),
  733. cmLexTokenText(h) ,
  734. cmLexTokenCharCount(h));
  735. // if the token is a number ...
  736. if( tid==kIntLexTId || tid==kRealLexTId || tid==kHexLexTId )
  737. {
  738. // ... then request the numbers value
  739. int iv = cmLexTokenInt(h);
  740. double dv = cmLexTokenDouble(h);
  741. cmRptPrintf(rpt,"%i %f",iv,dv);
  742. }
  743. cmRptPrintf(rpt,"\n");
  744. // handle errors
  745. if( tid == kErrorLexTId )
  746. {
  747. cmRptPrintf(rpt,"Error:%i\n", cmLexErrorRC(h));
  748. break;
  749. }
  750. }
  751. // finalize the lexer
  752. cmLexFinal(&h);
  753. }
  754. //)
  755. //}