One Level Up Top Level

src/lj_trace.c - luajit-2.0-src

Data types defined

Functions defined

Macros defined

Source code

  1. /*
  2. ** Trace management.
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. */

  6. ‌#define lj_trace_c
  7. ‌#define LUA_CORE

  9. #include "lj_obj.h"

  11. #if LJ_HASJIT

  13. #include "lj_gc.h"
  14. #include "lj_err.h"
  15. #include "lj_debug.h"
  16. #include "lj_str.h"
  17. #include "lj_frame.h"
  18. #include "lj_state.h"
  19. #include "lj_bc.h"
  20. #include "lj_ir.h"
  21. #include "lj_jit.h"
  22. #include "lj_iropt.h"
  23. #include "lj_mcode.h"
  24. #include "lj_trace.h"
  25. #include "lj_snap.h"
  26. #include "lj_gdbjit.h"
  27. #include "lj_record.h"
  28. #include "lj_asm.h"
  29. #include "lj_dispatch.h"
  30. #include "lj_vm.h"
  31. #include "lj_vmevent.h"
  32. #include "lj_target.h"

  34. /* -- Error handling ------------------------------------------------------ */

  36. /* Synchronous abort with error message. */
  37. ‌void lj_trace_err(jit_State *J, TraceError e)
  38. {
  39.   setnilV(&J->errinfo);  /* No error info. */
  40.   setintV(J->L->top++, (int32_t)e);
  41.   lj_err_throw(J->L, LUA_ERRRUN);
  42. }

  44. /* Synchronous abort with error message and error info. */
  45. ‌void lj_trace_err_info(jit_State *J, TraceError e)
  46. {
  47.   setintV(J->L->top++, (int32_t)e);
  48.   lj_err_throw(J->L, LUA_ERRRUN);
  49. }

  51. /* -- Trace management ---------------------------------------------------- */

  53. /* The current trace is first assembled in J->cur. The variable length
  54. ** arrays point to shared, growable buffers (J->irbuf etc.). When trace
  55. ** recording ends successfully, the current trace and its data structures
  56. ** are copied to a new (compact) GCtrace object.
  57. */

  59. /* Find a free trace number. */
  60. ‌static TraceNo trace_findfree(jit_State *J)
  61. {
  62.   MSize osz, lim;
  63.   if (J->freetrace == 0)
  64.     J->freetrace = 1;
  65.   for (; J->freetrace < J->sizetrace; J->freetrace++)
  66.     if (traceref(J, J->freetrace) == NULL)
  67.       return J->freetrace++;
  68.   /* Need to grow trace array. */
  69.   lim = (MSize)J->param[JIT_P_maxtrace] + 1;
  70.   if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
  71.   osz = J->sizetrace;
  72.   if (osz >= lim)
  73.     return 0;  /* Too many traces. */
  74.   lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
  75.   for (; osz < J->sizetrace; osz++)
  76.     setgcrefnull(J->trace[osz]);
  77.   return J->freetrace;
  78. }

  80. ‌#define TRACE_APPENDVEC(field, szfield, tp) \
  81.   T->field = (tp *)p; \
  82.   memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
  83.   p += J->cur.szfield*sizeof(tp);

  85. #ifdef LUAJIT_USE_PERFTOOLS
  86. /*
  87. ** Create symbol table of JIT-compiled code. For use with Linux perf tools.
  88. ** Example usage:
  89. **   perf record -f -e cycles luajit test.lua
  90. **   perf report -s symbol
  91. **   rm perf.data /tmp/perf-*.map
  92. */
  93. #include <stdio.h>
  94. #include <unistd.h>

  96. ‌static void perftools_addtrace(GCtrace *T)
  97. {
  98.   static FILE *fp;
  99.   GCproto *pt = &gcref(T->startpt)->pt;
  100.   const BCIns *startpc = mref(T->startpc, const BCIns);
  101.   const char *name = proto_chunknamestr(pt);
  102.   BCLine lineno;
  103.   if (name[0] == '@' || name[0] == '=')
  104.     name++;
  105.   else
  106.     name = "(string)";
  107.   lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
  108.   lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
  109.   if (!fp) {
  110.     char fname[40];
  111.     sprintf(fname, "/tmp/perf-%d.map", getpid());
  112.     if (!(fp = fopen(fname, "w"))) return;
  113.     setlinebuf(fp);
  114.   }
  115.   fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
  116.           (long)T->mcode, T->szmcode, T->traceno, name, lineno);
  117. }
  118. #endif

  120. /* Save current trace by copying and compacting it. */
  121. ‌static void trace_save(jit_State *J)
  122. {
  123.   size_t sztr = ((sizeof(GCtrace)+7)&~7);
  124.   size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
  125.   size_t sz = sztr + szins +
  126.               J->cur.nsnap*sizeof(SnapShot) +
  127.               J->cur.nsnapmap*sizeof(SnapEntry);
  128.   GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace);
  129.   char *p = (char *)T + sztr;
  130.   memcpy(T, &J->cur, sizeof(GCtrace));
  131.   setgcrefr(T->nextgc, J2G(J)->gc.root);
  132.   setgcrefp(J2G(J)->gc.root, T);
  133.   newwhite(J2G(J), T);
  134.   T->gct = ~LJ_TTRACE;
  135.   T->ir = (IRIns *)p - J->cur.nk;
  136.   memcpy(p, J->cur.ir+J->cur.nk, szins);
  137.   p += szins;
  138.   TRACE_APPENDVEC(snap, nsnap, SnapShot)
  139.   TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
  140.   J->cur.traceno = 0;
  141.   setgcrefp(J->trace[T->traceno], T);
  142.   lj_gc_barriertrace(J2G(J), T->traceno);
  143.   lj_gdbjit_addtrace(J, T);
  144. #ifdef LUAJIT_USE_PERFTOOLS
  145.   perftools_addtrace(T);
  146. #endif
  147. }

  149. ‌void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
  150. {
  151.   jit_State *J = G2J(g);
  152.   if (T->traceno) {
  153.     lj_gdbjit_deltrace(J, T);
  154.     if (T->traceno < J->freetrace)
  155.       J->freetrace = T->traceno;
  156.     setgcrefnull(J->trace[T->traceno]);
  157.   }
  158.   lj_mem_free(g, T,
  159.     ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
  160.     T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
  161. }

  163. /* Re-enable compiling a prototype by unpatching any modified bytecode. */
  164. ‌void lj_trace_reenableproto(GCproto *pt)
  165. {
  166.   if ((pt->flags & PROTO_ILOOP)) {
  167.     BCIns *bc = proto_bc(pt);
  168.     BCPos i, sizebc = pt->sizebc;;
  169.     pt->flags &= ~PROTO_ILOOP;
  170.     if (bc_op(bc[0]) == BC_IFUNCF)
  171.       setbc_op(&bc[0], BC_FUNCF);
  172.     for (i = 1; i < sizebc; i++) {
  173.       BCOp op = bc_op(bc[i]);
  174.       if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
  175.         setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
  176.     }
  177.   }
  178. }

  180. /* Unpatch the bytecode modified by a root trace. */
  181. ‌static void trace_unpatch(jit_State *J, GCtrace *T)
  182. {
  183.   BCOp op = bc_op(T->startins);
  184.   BCIns *pc = mref(T->startpc, BCIns);
  185.   UNUSED(J);
  186.   if (op == BC_JMP)
  187.     return;  /* No need to unpatch branches in parent traces (yet). */
  188.   switch (bc_op(*pc)) {
  189.   case BC_JFORL:
  190.     lua_assert(traceref(J, bc_d(*pc)) == T);
  191.     *pc = T->startins;
  192.     pc += bc_j(T->startins);
  193.     lua_assert(bc_op(*pc) == BC_JFORI);
  194.     setbc_op(pc, BC_FORI);
  195.     break;
  196.   case BC_JITERL:
  197.   case BC_JLOOP:
  198.     lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
  199.     *pc = T->startins;
  200.     break;
  201.   case BC_JMP:
  202.     lua_assert(op == BC_ITERL);
  203.     pc += bc_j(*pc)+2;
  204.     if (bc_op(*pc) == BC_JITERL) {
  205.       lua_assert(traceref(J, bc_d(*pc)) == T);
  206.       *pc = T->startins;
  207.     }
  208.     break;
  209.   case BC_JFUNCF:
  210.     lua_assert(op == BC_FUNCF);
  211.     *pc = T->startins;
  212.     break;
  213.   default:  /* Already unpatched. */
  214.     break;
  215.   }
  216. }

  218. /* Flush a root trace. */
  219. ‌static void trace_flushroot(jit_State *J, GCtrace *T)
  220. {
  221.   GCproto *pt = &gcref(T->startpt)->pt;
  222.   lua_assert(T->root == 0 && pt != NULL);
  223.   /* First unpatch any modified bytecode. */
  224.   trace_unpatch(J, T);
  225.   /* Unlink root trace from chain anchored in prototype. */
  226.   if (pt->trace == T->traceno) {  /* Trace is first in chain. Easy. */
  227.     pt->trace = T->nextroot;
  228.   } else if (pt->trace) {  /* Otherwise search in chain of root traces. */
  229.     GCtrace *T2 = traceref(J, pt->trace);
  230.     if (T2) {
  231.       for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
  232.         if (T2->nextroot == T->traceno) {
  233.           T2->nextroot = T->nextroot;  /* Unlink from chain. */
  234.           break;
  235.         }
  236.     }
  237.   }
  238. }

  240. /* Flush a trace. Only root traces are considered. */
  241. ‌void lj_trace_flush(jit_State *J, TraceNo traceno)
  242. {
  243.   if (traceno > 0 && traceno < J->sizetrace) {
  244.     GCtrace *T = traceref(J, traceno);
  245.     if (T && T->root == 0)
  246.       trace_flushroot(J, T);
  247.   }
  248. }

  250. /* Flush all traces associated with a prototype. */
  251. ‌void lj_trace_flushproto(global_State *g, GCproto *pt)
  252. {
  253.   while (pt->trace != 0)
  254.     trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
  255. }

  257. /* Flush all traces. */
  258. ‌int lj_trace_flushall(lua_State *L)
  259. {
  260.   jit_State *J = L2J(L);
  261.   ptrdiff_t i;
  262.   if ((J2G(J)->hookmask & HOOK_GC))
  263.     return 1;
  264.   for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
  265.     GCtrace *T = traceref(J, i);
  266.     if (T) {
  267.       if (T->root == 0)
  268.         trace_flushroot(J, T);
  269.       lj_gdbjit_deltrace(J, T);
  270.       T->traceno = 0;
  271.       setgcrefnull(J->trace[i]);
  272.     }
  273.   }
  274.   J->cur.traceno = 0;
  275.   J->freetrace = 0;
  276.   /* Clear penalty cache. */
  277.   memset(J->penalty, 0, sizeof(J->penalty));
  278.   /* Free the whole machine code and invalidate all exit stub groups. */
  279.   lj_mcode_free(J);
  280.   memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
  281.   lj_vmevent_send(L, TRACE,
  282.     setstrV(L, L->top++, lj_str_newlit(L, "flush"));
  283.   );
  284.   return 0;
  285. }

  287. /* Initialize JIT compiler state. */
  288. ‌void lj_trace_initstate(global_State *g)
  289. {
  290.   jit_State *J = G2J(g);
  291.   TValue *tv;
  292.   /* Initialize SIMD constants. */
  293.   tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
  294.   tv[0].u64 = U64x(7fffffff,ffffffff);
  295.   tv[1].u64 = U64x(7fffffff,ffffffff);
  296.   tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
  297.   tv[0].u64 = U64x(80000000,00000000);
  298.   tv[1].u64 = U64x(80000000,00000000);
  299. }

  301. /* Free everything associated with the JIT compiler state. */
  302. ‌void lj_trace_freestate(global_State *g)
  303. {
  304.   jit_State *J = G2J(g);
  305. #ifdef LUA_USE_ASSERT
  306.   {  /* This assumes all traces have already been freed. */
  307.     ptrdiff_t i;
  308.     for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
  309.       lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
  310.   }
  311. #endif
  312.   lj_mcode_free(J);
  313.   lj_ir_k64_freeall(J);
  314.   lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
  315.   lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
  316.   lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
  317.   lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
  318. }

  320. /* -- Penalties and blacklisting ------------------------------------------ */

  322. /* Blacklist a bytecode instruction. */
  323. ‌static void blacklist_pc(GCproto *pt, BCIns *pc)
  324. {
  325.   setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
  326.   pt->flags |= PROTO_ILOOP;
  327. }

  329. /* Penalize a bytecode instruction. */
  330. ‌static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
  331. {
  332.   uint32_t i, val = PENALTY_MIN;
  333.   for (i = 0; i < PENALTY_SLOTS; i++)
  334.     if (mref(J->penalty[i].pc, const BCIns) == pc) {  /* Cache slot found? */
  335.       /* First try to bump its hotcount several times. */
  336.       val = ((uint32_t)J->penalty[i].val << 1) +
  337.             LJ_PRNG_BITS(J, PENALTY_RNDBITS);
  338.       if (val > PENALTY_MAX) {
  339.         blacklist_pc(pt, pc);  /* Blacklist it, if that didn't help. */
  340.         return;
  341.       }
  342.       goto setpenalty;
  343.     }
  344.   /* Assign a new penalty cache slot. */
  345.   i = J->penaltyslot;
  346.   J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
  347.   setmref(J->penalty[i].pc, pc);
  348. setpenalty:
  349.   J->penalty[i].val = (uint16_t)val;
  350.   J->penalty[i].reason = e;
  351.   hotcount_set(J2GG(J), pc+1, val);
  352. }

  354. /* -- Trace compiler state machine ---------------------------------------- */

  356. /* Start tracing. */
  357. ‌static void trace_start(jit_State *J)
  358. {
  359.   lua_State *L;
  360.   TraceNo traceno;

  362.   if ((J->pt->flags & PROTO_NOJIT)) {  /* JIT disabled for this proto? */
  363.     if (J->parent == 0 && J->exitno == 0) {
  364.       /* Lazy bytecode patching to disable hotcount events. */
  365.       lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
  366.                  bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
  367.       setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
  368.       J->pt->flags |= PROTO_ILOOP;
  369.     }
  370.     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
  371.     return;
  372.   }

  374.   /* Get a new trace number. */
  375.   traceno = trace_findfree(J);
  376.   if (LJ_UNLIKELY(traceno == 0)) {  /* No free trace? */
  377.     lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
  378.     lj_trace_flushall(J->L);
  379.     J->state = LJ_TRACE_IDLE;  /* Silently ignored. */
  380.     return;
  381.   }
  382.   setgcrefp(J->trace[traceno], &J->cur);

  384.   /* Setup enough of the current trace to be able to send the vmevent. */
  385.   memset(&J->cur, 0, sizeof(GCtrace));
  386.   J->cur.traceno = traceno;
  387.   J->cur.nins = J->cur.nk = REF_BASE;
  388.   J->cur.ir = J->irbuf;
  389.   J->cur.snap = J->snapbuf;
  390.   J->cur.snapmap = J->snapmapbuf;
  391.   J->mergesnap = 0;
  392.   J->needsnap = 0;
  393.   J->bcskip = 0;
  394.   J->guardemit.irt = 0;
  395.   J->postproc = LJ_POST_NONE;
  396.   lj_resetsplit(J);
  397.   setgcref(J->cur.startpt, obj2gco(J->pt));

  399.   L = J->L;
  400.   lj_vmevent_send(L, TRACE,
  401.     setstrV(L, L->top++, lj_str_newlit(L, "start"));
  402.     setintV(L->top++, traceno);
  403.     setfuncV(L, L->top++, J->fn);
  404.     setintV(L->top++, proto_bcpos(J->pt, J->pc));
  405.     if (J->parent) {
  406.       setintV(L->top++, J->parent);
  407.       setintV(L->top++, J->exitno);
  408.     }
  409.   );
  410.   lj_record_setup(J);
  411. }

  413. /* Stop tracing. */
  414. ‌static void trace_stop(jit_State *J)
  415. {
  416.   BCIns *pc = mref(J->cur.startpc, BCIns);
  417.   BCOp op = bc_op(J->cur.startins);
  418.   GCproto *pt = &gcref(J->cur.startpt)->pt;
  419.   TraceNo traceno = J->cur.traceno;
  420.   lua_State *L;

  422.   switch (op) {
  423.   case BC_FORL:
  424.     setbc_op(pc+bc_j(J->cur.startins), BC_JFORI);  /* Patch FORI, too. */
  425.     /* fallthrough */
  426.   case BC_LOOP:
  427.   case BC_ITERL:
  428.   case BC_FUNCF:
  429.     /* Patch bytecode of starting instruction in root trace. */
  430.     setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
  431.     setbc_d(pc, traceno);
  432.   addroot:
  433.     /* Add to root trace chain in prototype. */
  434.     J->cur.nextroot = pt->trace;
  435.     pt->trace = (TraceNo1)traceno;
  436.     break;
  437.   case BC_RET:
  438.   case BC_RET0:
  439.   case BC_RET1:
  440.     *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
  441.     goto addroot;
  442.   case BC_JMP:
  443.     /* Patch exit branch in parent to side trace entry. */
  444.     lua_assert(J->parent != 0 && J->cur.root != 0);
  445.     lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
  446.     /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
  447.     traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
  448.     /* Add to side trace chain in root trace. */
  449.     {
  450.       GCtrace *root = traceref(J, J->cur.root);
  451.       root->nchild++;
  452.       J->cur.nextside = root->nextside;
  453.       root->nextside = (TraceNo1)traceno;
  454.     }
  455.     break;
  456.   case BC_CALLM:
  457.   case BC_CALL:
  458.   case BC_ITERC:
  459.     /* Trace stitching: patch link of previous trace. */
  460.     traceref(J, J->exitno)->link = traceno;
  461.     break;
  462.   default:
  463.     lua_assert(0);
  464.     break;
  465.   }

  467.   /* Commit new mcode only after all patching is done. */
  468.   lj_mcode_commit(J, J->cur.mcode);
  469.   J->postproc = LJ_POST_NONE;
  470.   trace_save(J);

  472.   L = J->L;
  473.   lj_vmevent_send(L, TRACE,
  474.     setstrV(L, L->top++, lj_str_newlit(L, "stop"));
  475.     setintV(L->top++, traceno);
  476.     setfuncV(L, L->top++, J->fn);
  477.   );
  478. }

  480. /* Start a new root trace for down-recursion. */
  481. ‌static int trace_downrec(jit_State *J)
  482. {
  483.   /* Restart recording at the return instruction. */
  484.   lua_assert(J->pt != NULL);
  485.   lua_assert(bc_isret(bc_op(*J->pc)));
  486.   if (bc_op(*J->pc) == BC_RETM)
  487.     return 0;  /* NYI: down-recursion with RETM. */
  488.   J->parent = 0;
  489.   J->exitno = 0;
  490.   J->state = LJ_TRACE_RECORD;
  491.   trace_start(J);
  492.   return 1;
  493. }

  495. /* Abort tracing. */
  496. ‌static int trace_abort(jit_State *J)
  497. {
  498.   lua_State *L = J->L;
  499.   TraceError e = LJ_TRERR_RECERR;
  500.   TraceNo traceno;

  502.   J->postproc = LJ_POST_NONE;
  503.   lj_mcode_abort(J);
  504.   if (tvisnumber(L->top-1))
  505.     e = (TraceError)numberVint(L->top-1);
  506.   if (e == LJ_TRERR_MCODELM) {
  507.     L->top--;  /* Remove error object */
  508.     J->state = LJ_TRACE_ASM;
  509.     return 1;  /* Retry ASM with new MCode area. */
  510.   }
  511.   /* Penalize or blacklist starting bytecode instruction. */
  512.   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
  513.     if (J->exitno == 0)
  514.       penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e);
  515.     else
  516.       traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
  517.   }

  519.   /* Is there anything to abort? */
  520.   traceno = J->cur.traceno;
  521.   if (traceno) {
  522.     ptrdiff_t errobj = savestack(L, L->top-1);  /* Stack may be resized. */
  523.     J->cur.link = 0;
  524.     J->cur.linktype = LJ_TRLINK_NONE;
  525.     lj_vmevent_send(L, TRACE,
  526.       TValue *frame;
  527.       const BCIns *pc;
  528.       GCfunc *fn;
  529.       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
  530.       setintV(L->top++, traceno);
  531.       /* Find original Lua function call to generate a better error message. */
  532.       frame = J->L->base-1;
  533.       pc = J->pc;
  534.       while (!isluafunc(frame_func(frame))) {
  535.         pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
  536.         frame = frame_prev(frame);
  537.       }
  538.       fn = frame_func(frame);
  539.       setfuncV(L, L->top++, fn);
  540.       setintV(L->top++, proto_bcpos(funcproto(fn), pc));
  541.       copyTV(L, L->top++, restorestack(L, errobj));
  542.       copyTV(L, L->top++, &J->errinfo);
  543.     );
  544.     /* Drop aborted trace after the vmevent (which may still access it). */
  545.     setgcrefnull(J->trace[traceno]);
  546.     if (traceno < J->freetrace)
  547.       J->freetrace = traceno;
  548.     J->cur.traceno = 0;
  549.   }
  550.   L->top--;  /* Remove error object */
  551.   if (e == LJ_TRERR_DOWNREC)
  552.     return trace_downrec(J);
  553.   else if (e == LJ_TRERR_MCODEAL)
  554.     lj_trace_flushall(L);
  555.   return 0;
  556. }

  558. /* Perform pending re-patch of a bytecode instruction. */
  559. ‌static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
  560. {
  561.   if (LJ_UNLIKELY(J->patchpc)) {
  562.     if (force || J->bcskip == 0) {
  563.       *J->patchpc = J->patchins;
  564.       J->patchpc = NULL;
  565.     } else {
  566.       J->bcskip = 0;
  567.     }
  568.   }
  569. }

  571. /* State machine for the trace compiler. Protected callback. */
  572. ‌static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
  573. {
  574.   jit_State *J = (jit_State *)ud;
  575.   UNUSED(dummy);
  576.   do {
  577.   retry:
  578.     switch (J->state) {
  579.     case LJ_TRACE_START:
  580.       J->state = LJ_TRACE_RECORD;  /* trace_start() may change state. */
  581.       trace_start(J);
  582.       lj_dispatch_update(J2G(J));
  583.       break;

  585.     case LJ_TRACE_RECORD:
  586.       trace_pendpatch(J, 0);
  587.       setvmstate(J2G(J), RECORD);
  588.       lj_vmevent_send_(L, RECORD,
  589.         /* Save/restore tmptv state for trace recorder. */
  590.         TValue savetv = J2G(J)->tmptv;
  591.         TValue savetv2 = J2G(J)->tmptv2;
  592.         setintV(L->top++, J->cur.traceno);
  593.         setfuncV(L, L->top++, J->fn);
  594.         setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
  595.         setintV(L->top++, J->framedepth);
  596.       ,
  597.         J2G(J)->tmptv = savetv;
  598.         J2G(J)->tmptv2 = savetv2;
  599.       );
  600.       lj_record_ins(J);
  601.       break;

  603.     case LJ_TRACE_END:
  604.       trace_pendpatch(J, 1);
  605.       J->loopref = 0;
  606.       if ((J->flags & JIT_F_OPT_LOOP) &&
  607.           J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
  608.         setvmstate(J2G(J), OPT);
  609.         lj_opt_dce(J);
  610.         if (lj_opt_loop(J)) {  /* Loop optimization failed? */
  611.           J->cur.link = 0;
  612.           J->cur.linktype = LJ_TRLINK_NONE;
  613.           J->loopref = J->cur.nins;
  614.           J->state = LJ_TRACE_RECORD;  /* Try to continue recording. */
  615.           break;
  616.         }
  617.         J->loopref = J->chain[IR_LOOP];  /* Needed by assembler. */
  618.       }
  619.       lj_opt_split(J);
  620.       lj_opt_sink(J);
  621.       if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
  622.       J->state = LJ_TRACE_ASM;
  623.       break;

  625.     case LJ_TRACE_ASM:
  626.       setvmstate(J2G(J), ASM);
  627.       lj_asm_trace(J, &J->cur);
  628.       trace_stop(J);
  629.       setvmstate(J2G(J), INTERP);
  630.       J->state = LJ_TRACE_IDLE;
  631.       lj_dispatch_update(J2G(J));
  632.       return NULL;

  634.     default:  /* Trace aborted asynchronously. */
  635.       setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
  636.       /* fallthrough */
  637.     case LJ_TRACE_ERR:
  638.       trace_pendpatch(J, 1);
  639.       if (trace_abort(J))
  640.         goto retry;
  641.       setvmstate(J2G(J), INTERP);
  642.       J->state = LJ_TRACE_IDLE;
  643.       lj_dispatch_update(J2G(J));
  644.       return NULL;
  645.     }
  646.   } while (J->state > LJ_TRACE_RECORD);
  647.   return NULL;
  648. }

  650. /* -- Event handling ------------------------------------------------------ */

  652. /* A bytecode instruction is about to be executed. Record it. */
  653. ‌void lj_trace_ins(jit_State *J, const BCIns *pc)
  654. {
  655.   /* Note: J->L must already be set. pc is the true bytecode PC here. */
  656.   J->pc = pc;
  657.   J->fn = curr_func(J->L);
  658.   J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
  659.   while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
  660.     J->state = LJ_TRACE_ERR;
  661. }

  663. /* A hotcount triggered. Start recording a root trace. */
  664. ‌void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
  665. {
  666.   /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
  667.   ERRNO_SAVE
  668.   /* Reset hotcount. */
  669.   hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
  670.   /* Only start a new trace if not recording or inside __gc call or vmevent. */
  671.   if (J->state == LJ_TRACE_IDLE &&
  672.       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
  673.     J->parent = 0;  /* Root trace. */
  674.     J->exitno = 0;
  675.     J->state = LJ_TRACE_START;
  676.     lj_trace_ins(J, pc-1);
  677.   }
  678.   ERRNO_RESTORE
  679. }

  681. /* Check for a hot side exit. If yes, start recording a side trace. */
  682. ‌static void trace_hotside(jit_State *J, const BCIns *pc)
  683. {
  684.   SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
  685.   if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
  686.       isluafunc(curr_func(J->L)) &&
  687.       snap->count != SNAPCOUNT_DONE &&
  688.       ++snap->count >= J->param[JIT_P_hotexit]) {
  689.     lua_assert(J->state == LJ_TRACE_IDLE);
  690.     /* J->parent is non-zero for a side trace. */
  691.     J->state = LJ_TRACE_START;
  692.     lj_trace_ins(J, pc);
  693.   }
  694. }

  696. /* Stitch a new trace to the previous trace. */
  697. ‌void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
  698. {
  699.   /* Only start a new trace if not recording or inside __gc call or vmevent. */
  700.   if (J->state == LJ_TRACE_IDLE &&
  701.       !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
  702.     J->parent = 0;  /* Have to treat it like a root trace. */
  703.     /* J->exitno is set to the invoking trace. */
  704.     J->state = LJ_TRACE_START;
  705.     lj_trace_ins(J, pc);
  706.   }
  707. }


  710. /* Tiny struct to pass data to protected call. */
  711. ‌typedef struct ExitDataCP {
  712.   jit_State *J;
  713.   void *exptr;                /* Pointer to exit state. */
  714.   const BCIns *pc;        /* Restart interpreter at this PC. */
  715. ‌} ExitDataCP;

  717. /* Need to protect lj_snap_restore because it may throw. */
  718. ‌static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
  719. {
  720.   ExitDataCP *exd = (ExitDataCP *)ud;
  721.   cframe_errfunc(L->cframe) = -1;  /* Inherit error function. */
  722.   exd->pc = lj_snap_restore(exd->J, exd->exptr);
  723.   UNUSED(dummy);
  724.   return NULL;
  725. }

  727. #ifndef LUAJIT_DISABLE_VMEVENT
  728. /* Push all registers from exit state. */
  729. ‌static void trace_exit_regs(lua_State *L, ExitState *ex)
  730. {
  731.   int32_t i;
  732.   setintV(L->top++, RID_NUM_GPR);
  733.   setintV(L->top++, RID_NUM_FPR);
  734.   for (i = 0; i < RID_NUM_GPR; i++) {
  735.     if (sizeof(ex->gpr[i]) == sizeof(int32_t))
  736.       setintV(L->top++, (int32_t)ex->gpr[i]);
  737.     else
  738.       setnumV(L->top++, (lua_Number)ex->gpr[i]);
  739.   }
  740. #if !LJ_SOFTFP
  741.   for (i = 0; i < RID_NUM_FPR; i++) {
  742.     setnumV(L->top, ex->fpr[i]);
  743.     if (LJ_UNLIKELY(tvisnan(L->top)))
  744.       setnanV(L->top);
  745.     L->top++;
  746.   }
  747. #endif
  748. }
  749. #endif

  751. #ifdef EXITSTATE_PCREG
  752. /* Determine trace number from pc of exit instruction. */
  753. ‌static TraceNo trace_exit_find(jit_State *J, MCode *pc)
  754. {
  755.   TraceNo traceno;
  756.   for (traceno = 1; traceno < J->sizetrace; traceno++) {
  757.     GCtrace *T = traceref(J, traceno);
  758.     if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
  759.       return traceno;
  760.   }
  761.   lua_assert(0);
  762.   return 0;
  763. }
  764. #endif

  766. /* A trace exited. Restore interpreter state. */
  767. ‌int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
  768. {
  769.   ERRNO_SAVE
  770.   lua_State *L = J->L;
  771.   ExitState *ex = (ExitState *)exptr;
  772.   ExitDataCP exd;
  773.   int errcode;
  774.   const BCIns *pc;
  775.   void *cf;
  776.   GCtrace *T;
  777. #ifdef EXITSTATE_PCREG
  778.   J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
  779. #endif
  780.   T = traceref(J, J->parent); UNUSED(T);
  781. #ifdef EXITSTATE_CHECKEXIT
  782.   if (J->exitno == T->nsnap) {  /* Treat stack check like a parent exit. */
  783.     lua_assert(T->root != 0);
  784.     J->exitno = T->ir[REF_BASE].op2;
  785.     J->parent = T->ir[REF_BASE].op1;
  786.     T = traceref(J, J->parent);
  787.   }
  788. #endif
  789.   lua_assert(T != NULL && J->exitno < T->nsnap);
  790.   exd.J = J;
  791.   exd.exptr = exptr;
  792.   errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
  793.   if (errcode)
  794.     return -errcode;  /* Return negated error code. */

  796.   if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
  797.     lj_vmevent_send(L, TEXIT,
  798.       lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
  799.       setintV(L->top++, J->parent);
  800.       setintV(L->top++, J->exitno);
  801.       trace_exit_regs(L, ex);
  802.     );

  804.   pc = exd.pc;
  805.   cf = cframe_raw(L->cframe);
  806.   setcframe_pc(cf, pc);
  807.   if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
  808.     /* Just exit to interpreter. */
  809.   } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
  810.     if (!(G(L)->hookmask & HOOK_GC))
  811.       lj_gc_step(L);  /* Exited because of GC: drive GC forward. */
  812.   } else {
  813.     trace_hotside(J, pc);
  814.   }
  815.   if (bc_op(*pc) == BC_JLOOP) {
  816.     BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
  817.     if (bc_isret(bc_op(*retpc))) {
  818.       if (J->state == LJ_TRACE_RECORD) {
  819.         J->patchins = *pc;
  820.         J->patchpc = (BCIns *)pc;
  821.         *J->patchpc = *retpc;
  822.         J->bcskip = 1;
  823.       } else {
  824.         pc = retpc;
  825.         setcframe_pc(cf, pc);
  826.       }
  827.     }
  828.   }
  829.   /* Return MULTRES or 0. */
  830.   ERRNO_RESTORE
  831.   switch (bc_op(*pc)) {
  832.   case BC_CALLM: case BC_CALLMT:
  833.     return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) + LJ_FR2);
  834.   case BC_RETM:
  835.     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
  836.   case BC_TSETM:
  837.     return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
  838.   default:
  839.     if (bc_op(*pc) >= BC_FUNCF)
  840.       return (int)((BCReg)(L->top - L->base) + 1);
  841.     return 0;
  842.   }
  843. }

  845. #endif

One Level Up Top Level