One Level Up Top Level

src/lj_lex.c - luajit-2.0-src

Global variables defined

Functions defined

Macros defined

Source code

  1. /*
  2. ** Lexical analyzer.
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. **
  5. ** Major portions taken verbatim or adapted from the Lua interpreter.
  6. ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
  7. */

  9. ‌#define lj_lex_c
  10. ‌#define LUA_CORE

  12. #include "lj_obj.h"
  13. #include "lj_gc.h"
  14. #include "lj_err.h"
  15. #include "lj_buf.h"
  16. #include "lj_str.h"
  17. #if LJ_HASFFI
  18. #include "lj_tab.h"
  19. #include "lj_ctype.h"
  20. #include "lj_cdata.h"
  21. #include "lualib.h"
  22. #endif
  23. #include "lj_state.h"
  24. #include "lj_lex.h"
  25. #include "lj_parse.h"
  26. #include "lj_char.h"
  27. #include "lj_strscan.h"
  28. #include "lj_strfmt.h"

  30. /* Lua lexer token names. */
  31. ‌static const char *const tokennames[] = {
  32. ‌#define TKSTR1(name)                #name,
  33. ‌#define TKSTR2(name, sym)        #sym,
  34. TKDEF(TKSTR1, TKSTR2)
  35. ‌#undef TKSTR1
  36. ‌#undef TKSTR2
  37.   NULL
  38. };

  40. /* -- Buffer handling ----------------------------------------------------- */

  42. ‌#define LEX_EOF                        (-1)
  43. ‌#define lex_iseol(ls)                (ls->c == '\n' || ls->c == '\r')

  45. /* Get more input from reader. */
  46. ‌static LJ_NOINLINE LexChar lex_more(LexState *ls)
  47. {
  48.   size_t sz;
  49.   const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
  50.   if (p == NULL || sz == 0) return LEX_EOF;
  51.   ls->pe = p + sz;
  52.   ls->p = p + 1;
  53.   return (LexChar)(uint8_t)p[0];
  54. }

  56. /* Get next character. */
  57. ‌static LJ_AINLINE LexChar lex_next(LexState *ls)
  58. {
  59.   return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
  60. }

  62. /* Save character. */
  63. ‌static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
  64. {
  65.   lj_buf_putb(&ls->sb, c);
  66. }

  68. /* Save previous character and get next character. */
  69. ‌static LJ_AINLINE LexChar lex_savenext(LexState *ls)
  70. {
  71.   lex_save(ls, ls->c);
  72.   return lex_next(ls);
  73. }

  75. /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
  76. ‌static void lex_newline(LexState *ls)
  77. {
  78.   LexChar old = ls->c;
  79.   lua_assert(lex_iseol(ls));
  80.   lex_next(ls);  /* Skip "\n" or "\r". */
  81.   if (lex_iseol(ls) && ls->c != old) lex_next(ls);  /* Skip "\n\r" or "\r\n". */
  82.   if (++ls->linenumber >= LJ_MAX_LINE)
  83.     lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
  84. }

  86. /* -- Scanner for terminals ----------------------------------------------- */

  88. /* Parse a number literal. */
  89. ‌static void lex_number(LexState *ls, TValue *tv)
  90. {
  91.   StrScanFmt fmt;
  92.   LexChar c, xp = 'e';
  93.   lua_assert(lj_char_isdigit(ls->c));
  94.   if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
  95.     xp = 'p';
  96.   while (lj_char_isident(ls->c) || ls->c == '.' ||
  97.          ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
  98.     c = ls->c;
  99.     lex_savenext(ls);
  100.   }
  101.   lex_save(ls, '\0');
  102.   fmt = lj_strscan_scan((const uint8_t *)sbufB(&ls->sb), tv,
  103.           (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
  104.           (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
  105.   if (LJ_DUALNUM && fmt == STRSCAN_INT) {
  106.     setitype(tv, LJ_TISNUM);
  107.   } else if (fmt == STRSCAN_NUM) {
  108.     /* Already in correct format. */
  109. #if LJ_HASFFI
  110.   } else if (fmt != STRSCAN_ERROR) {
  111.     lua_State *L = ls->L;
  112.     GCcdata *cd;
  113.     lua_assert(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG);
  114.     if (!ctype_ctsG(G(L))) {
  115.       ptrdiff_t oldtop = savestack(L, L->top);
  116.       luaopen_ffi(L);  /* Load FFI library on-demand. */
  117.       L->top = restorestack(L, oldtop);
  118.     }
  119.     if (fmt == STRSCAN_IMAG) {
  120.       cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
  121.       ((double *)cdataptr(cd))[0] = 0;
  122.       ((double *)cdataptr(cd))[1] = numV(tv);
  123.     } else {
  124.       cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
  125.       *(uint64_t *)cdataptr(cd) = tv->u64;
  126.     }
  127.     lj_parse_keepcdata(ls, tv, cd);
  128. #endif
  129.   } else {
  130.     lua_assert(fmt == STRSCAN_ERROR);
  131.     lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
  132.   }
  133. }

  135. /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
  136. ‌static int lex_skipeq(LexState *ls)
  137. {
  138.   int count = 0;
  139.   LexChar s = ls->c;
  140.   lua_assert(s == '[' || s == ']');
  141.   while (lex_savenext(ls) == '=')
  142.     count++;
  143.   return (ls->c == s) ? count : (-count) - 1;
  144. }

  146. /* Parse a long string or long comment (tv set to NULL). */
  147. ‌static void lex_longstring(LexState *ls, TValue *tv, int sep)
  148. {
  149.   lex_savenext(ls);  /* Skip second '['. */
  150.   if (lex_iseol(ls))  /* Skip initial newline. */
  151.     lex_newline(ls);
  152.   for (;;) {
  153.     switch (ls->c) {
  154.     case LEX_EOF:
  155.       lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
  156.       break;
  157.     case ']':
  158.       if (lex_skipeq(ls) == sep) {
  159.         lex_savenext(ls);  /* Skip second ']'. */
  160.         goto endloop;
  161.       }
  162.       break;
  163.     case '\n':
  164.     case '\r':
  165.       lex_save(ls, '\n');
  166.       lex_newline(ls);
  167.       if (!tv) lj_buf_reset(&ls->sb);  /* Don't waste space for comments. */
  168.       break;
  169.     default:
  170.       lex_savenext(ls);
  171.       break;
  172.     }
  173.   } endloop:
  174.   if (tv) {
  175.     GCstr *str = lj_parse_keepstr(ls, sbufB(&ls->sb) + (2 + (MSize)sep),
  176.                                       sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
  177.     setstrV(ls->L, tv, str);
  178.   }
  179. }

  181. /* Parse a string. */
  182. ‌static void lex_string(LexState *ls, TValue *tv)
  183. {
  184.   LexChar delim = ls->c;  /* Delimiter is '\'' or '"'. */
  185.   lex_savenext(ls);
  186.   while (ls->c != delim) {
  187.     switch (ls->c) {
  188.     case LEX_EOF:
  189.       lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
  190.       continue;
  191.     case '\n':
  192.     case '\r':
  193.       lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
  194.       continue;
  195.     case '\\': {
  196.       LexChar c = lex_next(ls);  /* Skip the '\\'. */
  197.       switch (c) {
  198.       case 'a': c = '\a'; break;
  199.       case 'b': c = '\b'; break;
  200.       case 'f': c = '\f'; break;
  201.       case 'n': c = '\n'; break;
  202.       case 'r': c = '\r'; break;
  203.       case 't': c = '\t'; break;
  204.       case 'v': c = '\v'; break;
  205.       case 'x':  /* Hexadecimal escape '\xXX'. */
  206.         c = (lex_next(ls) & 15u) << 4;
  207.         if (!lj_char_isdigit(ls->c)) {
  208.           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
  209.           c += 9 << 4;
  210.         }
  211.         c += (lex_next(ls) & 15u);
  212.         if (!lj_char_isdigit(ls->c)) {
  213.           if (!lj_char_isxdigit(ls->c)) goto err_xesc;
  214.           c += 9;
  215.         }
  216.         break;
  217.       case 'z':  /* Skip whitespace. */
  218.         lex_next(ls);
  219.         while (lj_char_isspace(ls->c))
  220.           if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
  221.         continue;
  222.       case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
  223.       case '\\': case '\"': case '\'': break;
  224.       case LEX_EOF: continue;
  225.       default:
  226.         if (!lj_char_isdigit(c))
  227.           goto err_xesc;
  228.         c -= '0';  /* Decimal escape '\ddd'. */
  229.         if (lj_char_isdigit(lex_next(ls))) {
  230.           c = c*10 + (ls->c - '0');
  231.           if (lj_char_isdigit(lex_next(ls))) {
  232.             c = c*10 + (ls->c - '0');
  233.             if (c > 255) {
  234.             err_xesc:
  235.               lj_lex_error(ls, TK_string, LJ_ERR_XESC);
  236.             }
  237.             lex_next(ls);
  238.           }
  239.         }
  240.         lex_save(ls, c);
  241.         continue;
  242.       }
  243.       lex_save(ls, c);
  244.       lex_next(ls);
  245.       continue;
  246.       }
  247.     default:
  248.       lex_savenext(ls);
  249.       break;
  250.     }
  251.   }
  252.   lex_savenext(ls);  /* Skip trailing delimiter. */
  253.   setstrV(ls->L, tv,
  254.           lj_parse_keepstr(ls, sbufB(&ls->sb)+1, sbuflen(&ls->sb)-2));
  255. }

  257. /* -- Main lexical scanner ------------------------------------------------ */

  259. /* Get next lexical token. */
  260. ‌static LexToken lex_scan(LexState *ls, TValue *tv)
  261. {
  262.   lj_buf_reset(&ls->sb);
  263.   for (;;) {
  264.     if (lj_char_isident(ls->c)) {
  265.       GCstr *s;
  266.       if (lj_char_isdigit(ls->c)) {  /* Numeric literal. */
  267.         lex_number(ls, tv);
  268.         return TK_number;
  269.       }
  270.       /* Identifier or reserved word. */
  271.       do {
  272.         lex_savenext(ls);
  273.       } while (lj_char_isident(ls->c));
  274.       s = lj_parse_keepstr(ls, sbufB(&ls->sb), sbuflen(&ls->sb));
  275.       setstrV(ls->L, tv, s);
  276.       if (s->reserved > 0)  /* Reserved word? */
  277.         return TK_OFS + s->reserved;
  278.       return TK_name;
  279.     }
  280.     switch (ls->c) {
  281.     case '\n':
  282.     case '\r':
  283.       lex_newline(ls);
  284.       continue;
  285.     case ' ':
  286.     case '\t':
  287.     case '\v':
  288.     case '\f':
  289.       lex_next(ls);
  290.       continue;
  291.     case '-':
  292.       lex_next(ls);
  293.       if (ls->c != '-') return '-';
  294.       lex_next(ls);
  295.       if (ls->c == '[') {  /* Long comment "--[=*[...]=*]". */
  296.         int sep = lex_skipeq(ls);
  297.         lj_buf_reset(&ls->sb);  /* `lex_skipeq' may dirty the buffer */
  298.         if (sep >= 0) {
  299.           lex_longstring(ls, NULL, sep);
  300.           lj_buf_reset(&ls->sb);
  301.           continue;
  302.         }
  303.       }
  304.       /* Short comment "--.*\n". */
  305.       while (!lex_iseol(ls) && ls->c != LEX_EOF)
  306.         lex_next(ls);
  307.       continue;
  308.     case '[': {
  309.       int sep = lex_skipeq(ls);
  310.       if (sep >= 0) {
  311.         lex_longstring(ls, tv, sep);
  312.         return TK_string;
  313.       } else if (sep == -1) {
  314.         return '[';
  315.       } else {
  316.         lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
  317.         continue;
  318.       }
  319.       }
  320.     case '=':
  321.       lex_next(ls);
  322.       if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
  323.     case '<':
  324.       lex_next(ls);
  325.       if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
  326.     case '>':
  327.       lex_next(ls);
  328.       if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
  329.     case '~':
  330.       lex_next(ls);
  331.       if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
  332.     case ':':
  333.       lex_next(ls);
  334.       if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
  335.     case '"':
  336.     case '\'':
  337.       lex_string(ls, tv);
  338.       return TK_string;
  339.     case '.':
  340.       if (lex_savenext(ls) == '.') {
  341.         lex_next(ls);
  342.         if (ls->c == '.') {
  343.           lex_next(ls);
  344.           return TK_dots;   /* ... */
  345.         }
  346.         return TK_concat;   /* .. */
  347.       } else if (!lj_char_isdigit(ls->c)) {
  348.         return '.';
  349.       } else {
  350.         lex_number(ls, tv);
  351.         return TK_number;
  352.       }
  353.     case LEX_EOF:
  354.       return TK_eof;
  355.     default: {
  356.       LexChar c = ls->c;
  357.       lex_next(ls);
  358.       return c;  /* Single-char tokens (+ - / ...). */
  359.     }
  360.     }
  361.   }
  362. }

  364. /* -- Lexer API ----------------------------------------------------------- */

  366. /* Setup lexer state. */
  367. ‌int lj_lex_setup(lua_State *L, LexState *ls)
  368. {
  369.   int header = 0;
  370.   ls->L = L;
  371.   ls->fs = NULL;
  372.   ls->pe = ls->p = NULL;
  373.   ls->vstack = NULL;
  374.   ls->sizevstack = 0;
  375.   ls->vtop = 0;
  376.   ls->bcstack = NULL;
  377.   ls->sizebcstack = 0;
  378.   ls->tok = 0;
  379.   ls->lookahead = TK_eof;  /* No look-ahead token. */
  380.   ls->linenumber = 1;
  381.   ls->lastline = 1;
  382.   lex_next(ls);  /* Read-ahead first char. */
  383.   if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
  384.       (uint8_t)ls->p[1] == 0xbf) {  /* Skip UTF-8 BOM (if buffered). */
  385.     ls->p += 2;
  386.     lex_next(ls);
  387.     header = 1;
  388.   }
  389.   if (ls->c == '#') {  /* Skip POSIX #! header line. */
  390.     do {
  391.       lex_next(ls);
  392.       if (ls->c == LEX_EOF) return 0;
  393.     } while (!lex_iseol(ls));
  394.     lex_newline(ls);
  395.     header = 1;
  396.   }
  397.   if (ls->c == LUA_SIGNATURE[0]) {  /* Bytecode dump. */
  398.     if (header) {
  399.       /*
  400.       ** Loading bytecode with an extra header is disabled for security
  401.       ** reasons. This may circumvent the usual check for bytecode vs.
  402.       ** Lua code by looking at the first char. Since this is a potential
  403.       ** security violation no attempt is made to echo the chunkname either.
  404.       */
  405.       setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
  406.       lj_err_throw(L, LUA_ERRSYNTAX);
  407.     }
  408.     return 1;
  409.   }
  410.   return 0;
  411. }

  413. /* Cleanup lexer state. */
  414. ‌void lj_lex_cleanup(lua_State *L, LexState *ls)
  415. {
  416.   global_State *g = G(L);
  417.   lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
  418.   lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
  419.   lj_buf_free(g, &ls->sb);
  420. }

  422. /* Return next lexical token. */
  423. ‌void lj_lex_next(LexState *ls)
  424. {
  425.   ls->lastline = ls->linenumber;
  426.   if (LJ_LIKELY(ls->lookahead == TK_eof)) {  /* No lookahead token? */
  427.     ls->tok = lex_scan(ls, &ls->tokval);  /* Get next token. */
  428.   } else {  /* Otherwise return lookahead token. */
  429.     ls->tok = ls->lookahead;
  430.     ls->lookahead = TK_eof;
  431.     ls->tokval = ls->lookaheadval;
  432.   }
  433. }

  435. /* Look ahead for the next token. */
  436. ‌LexToken lj_lex_lookahead(LexState *ls)
  437. {
  438.   lua_assert(ls->lookahead == TK_eof);
  439.   ls->lookahead = lex_scan(ls, &ls->lookaheadval);
  440.   return ls->lookahead;
  441. }

  443. /* Convert token to string. */
  444. ‌const char *lj_lex_token2str(LexState *ls, LexToken tok)
  445. {
  446.   if (tok > TK_OFS)
  447.     return tokennames[tok-TK_OFS-1];
  448.   else if (!lj_char_iscntrl(tok))
  449.     return lj_strfmt_pushf(ls->L, "%c", tok);
  450.   else
  451.     return lj_strfmt_pushf(ls->L, "char(%d)", tok);
  452. }

  454. /* Lexer error. */
  455. ‌void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
  456. {
  457.   const char *tokstr;
  458.   va_list argp;
  459.   if (tok == 0) {
  460.     tokstr = NULL;
  461.   } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
  462.     lex_save(ls, '\0');
  463.     tokstr = sbufB(&ls->sb);
  464.   } else {
  465.     tokstr = lj_lex_token2str(ls, tok);
  466.   }
  467.   va_start(argp, em);
  468.   lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
  469.   va_end(argp);
  470. }

  472. /* Initialize strings for reserved words. */
  473. ‌void lj_lex_init(lua_State *L)
  474. {
  475.   uint32_t i;
  476.   for (i = 0; i < TK_RESERVED; i++) {
  477.     GCstr *s = lj_str_newz(L, tokennames[i]);
  478.     fixstring(s);  /* Reserved words are never collected. */
  479.     s->reserved = (uint8_t)(i+1);
  480.   }
  481. }

One Level Up Top Level