src/lj_dispatch.c - luajit-2.0-src

Global variables defined

Functions defined

Macros defined

Source code

  1. /*
  2. ** Instruction dispatch handling.
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. */

  5. #define lj_dispatch_c
  6. #define LUA_CORE

  7. #include "lj_obj.h"
  8. #include "lj_err.h"
  9. #include "lj_buf.h"
  10. #include "lj_func.h"
  11. #include "lj_str.h"
  12. #include "lj_tab.h"
  13. #include "lj_meta.h"
  14. #include "lj_debug.h"
  15. #include "lj_state.h"
  16. #include "lj_frame.h"
  17. #include "lj_bc.h"
  18. #include "lj_ff.h"
  19. #include "lj_strfmt.h"
  20. #if LJ_HASJIT
  21. #include "lj_jit.h"
  22. #endif
  23. #if LJ_HASFFI
  24. #include "lj_ccallback.h"
  25. #endif
  26. #include "lj_trace.h"
  27. #include "lj_dispatch.h"
  28. #if LJ_HASPROFILE
  29. #include "lj_profile.h"
  30. #endif
  31. #include "lj_vm.h"
  32. #include "luajit.h"

  33. /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
  34. LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);

  35. /* -- Dispatch table management ------------------------------------------- */

  36. #if LJ_TARGET_MIPS
  37. #include <math.h>
  38. LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
  39.                                                           lua_State *co);
  40. #if !LJ_HASJIT
  41. #define lj_dispatch_stitch        lj_dispatch_ins
  42. #endif
  43. #if !LJ_HASPROFILE
  44. #define lj_dispatch_profile        lj_dispatch_ins
  45. #endif

  46. #define GOTFUNC(name)        (ASMFunction)name,
  47. static const ASMFunction dispatch_got[] = {
  48.   GOTDEF(GOTFUNC)
  49. };
  50. #undef GOTFUNC
  51. #endif

  52. /* Initialize instruction dispatch table and hot counters. */
  53. void lj_dispatch_init(GG_State *GG)
  54. {
  55.   uint32_t i;
  56.   ASMFunction *disp = GG->dispatch;
  57.   for (i = 0; i < GG_LEN_SDISP; i++)
  58.     disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  59.   for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  60.     disp[i] = makeasmfunc(lj_bc_ofs[i]);
  61.   /* The JIT engine is off by default. luaopen_jit() turns it on. */
  62.   disp[BC_FORL] = disp[BC_IFORL];
  63.   disp[BC_ITERL] = disp[BC_IITERL];
  64.   disp[BC_LOOP] = disp[BC_ILOOP];
  65.   disp[BC_FUNCF] = disp[BC_IFUNCF];
  66.   disp[BC_FUNCV] = disp[BC_IFUNCV];
  67.   GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  68.   for (i = 0; i < GG_NUM_ASMFF; i++)
  69.     GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
  70. #if LJ_TARGET_MIPS
  71.   memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
  72. #endif
  73. }

  74. #if LJ_HASJIT
  75. /* Initialize hotcount table. */
  76. void lj_dispatch_init_hotcount(global_State *g)
  77. {
  78.   int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  79.   HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  80.   HotCount *hotcount = G2GG(g)->hotcount;
  81.   uint32_t i;
  82.   for (i = 0; i < HOTCOUNT_SIZE; i++)
  83.     hotcount[i] = start;
  84. }
  85. #endif

  86. /* Internal dispatch mode bits. */
  87. #define DISPMODE_CALL        0x01        /* Override call dispatch. */
  88. #define DISPMODE_RET        0x02        /* Override return dispatch. */
  89. #define DISPMODE_INS        0x04        /* Override instruction dispatch. */
  90. #define DISPMODE_JIT        0x10        /* JIT compiler on. */
  91. #define DISPMODE_REC        0x20        /* Recording active. */
  92. #define DISPMODE_PROF        0x40        /* Profiling active. */

  93. /* Update dispatch table depending on various flags. */
  94. void lj_dispatch_update(global_State *g)
  95. {
  96.   uint8_t oldmode = g->dispatchmode;
  97.   uint8_t mode = 0;
  98. #if LJ_HASJIT
  99.   mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
  100.   mode |= G2J(g)->state != LJ_TRACE_IDLE ?
  101.             (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
  102. #endif
  103. #if LJ_HASPROFILE
  104.   mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
  105. #endif
  106.   mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
  107.   mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
  108.   mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
  109.   if (oldmode != mode) {  /* Mode changed? */
  110.     ASMFunction *disp = G2GG(g)->dispatch;
  111.     ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
  112.     g->dispatchmode = mode;

  113.     /* Hotcount if JIT is on, but not while recording. */
  114.     if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
  115.       f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
  116.       f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
  117.       f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
  118.       f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
  119.       f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
  120.     } else/* Otherwise use the non-hotcounting instructions. */
  121.       f_forl = disp[GG_LEN_DDISP+BC_IFORL];
  122.       f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
  123.       f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
  124.       f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
  125.       f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
  126.     }
  127.     /* Init static counting instruction dispatch first (may be copied below). */
  128.     disp[GG_LEN_DDISP+BC_FORL] = f_forl;
  129.     disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
  130.     disp[GG_LEN_DDISP+BC_LOOP] = f_loop;

  131.     /* Set dynamic instruction dispatch. */
  132.     if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
  133.       /* Need to update the whole table. */
  134.       if (!(mode & DISPMODE_INS)) {  /* No ins dispatch? */
  135.         /* Copy static dispatch table to dynamic dispatch table. */
  136.         memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
  137.         /* Overwrite with dynamic return dispatch. */
  138.         if ((mode & DISPMODE_RET)) {
  139.           disp[BC_RETM] = lj_vm_rethook;
  140.           disp[BC_RET] = lj_vm_rethook;
  141.           disp[BC_RET0] = lj_vm_rethook;
  142.           disp[BC_RET1] = lj_vm_rethook;
  143.         }
  144.       } else {
  145.         /* The recording dispatch also checks for hooks. */
  146.         ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
  147.                         (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
  148.         uint32_t i;
  149.         for (i = 0; i < GG_LEN_SDISP; i++)
  150.           disp[i] = f;
  151.       }
  152.     } else if (!(mode & DISPMODE_INS)) {
  153.       /* Otherwise set dynamic counting ins. */
  154.       disp[BC_FORL] = f_forl;
  155.       disp[BC_ITERL] = f_iterl;
  156.       disp[BC_LOOP] = f_loop;
  157.       /* Set dynamic return dispatch. */
  158.       if ((mode & DISPMODE_RET)) {
  159.         disp[BC_RETM] = lj_vm_rethook;
  160.         disp[BC_RET] = lj_vm_rethook;
  161.         disp[BC_RET0] = lj_vm_rethook;
  162.         disp[BC_RET1] = lj_vm_rethook;
  163.       } else {
  164.         disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
  165.         disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
  166.         disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
  167.         disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
  168.       }
  169.     }

  170.     /* Set dynamic call dispatch. */
  171.     if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
  172.       uint32_t i;
  173.       if ((mode & DISPMODE_CALL) == 0) {  /* No call hooks? */
  174.         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  175.           disp[i] = makeasmfunc(lj_bc_ofs[i]);
  176.       } else {
  177.         for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
  178.           disp[i] = lj_vm_callhook;
  179.       }
  180.     }
  181.     if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
  182.       disp[BC_FUNCF] = f_funcf;
  183.       disp[BC_FUNCV] = f_funcv;
  184.     }

  185. #if LJ_HASJIT
  186.     /* Reset hotcounts for JIT off to on transition. */
  187.     if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
  188.       lj_dispatch_init_hotcount(g);
  189. #endif
  190.   }
  191. }

  192. /* -- JIT mode setting ---------------------------------------------------- */

  193. #if LJ_HASJIT
  194. /* Set JIT mode for a single prototype. */
  195. static void setptmode(global_State *g, GCproto *pt, int mode)
  196. {
  197.   if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
  198.     pt->flags &= ~PROTO_NOJIT;
  199.     lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
  200.   } else/* Flush and/or disable JIT compilation. */
  201.     if (!(mode & LUAJIT_MODE_FLUSH))
  202.       pt->flags |= PROTO_NOJIT;
  203.     lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
  204.   }
  205. }

  206. /* Recursively set the JIT mode for all children of a prototype. */
  207. static void setptmode_all(global_State *g, GCproto *pt, int mode)
  208. {
  209.   ptrdiff_t i;
  210.   if (!(pt->flags & PROTO_CHILD)) return;
  211.   for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
  212.     GCobj *o = proto_kgc(pt, i);
  213.     if (o->gch.gct == ~LJ_TPROTO) {
  214.       setptmode(g, gco2pt(o), mode);
  215.       setptmode_all(g, gco2pt(o), mode);
  216.     }
  217.   }
  218. }
  219. #endif

  220. /* Public API function: control the JIT engine. */
  221. int luaJIT_setmode(lua_State *L, int idx, int mode)
  222. {
  223.   global_State *g = G(L);
  224.   int mm = mode & LUAJIT_MODE_MASK;
  225.   lj_trace_abort(g);  /* Abort recording on any state change. */
  226.   /* Avoid pulling the rug from under our own feet. */
  227.   if ((g->hookmask & HOOK_GC))
  228.     lj_err_caller(L, LJ_ERR_NOGCMM);
  229.   switch (mm) {
  230. #if LJ_HASJIT
  231.   case LUAJIT_MODE_ENGINE:
  232.     if ((mode & LUAJIT_MODE_FLUSH)) {
  233.       lj_trace_flushall(L);
  234.     } else {
  235.       if (!(mode & LUAJIT_MODE_ON))
  236.         G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
  237. #if LJ_TARGET_X86ORX64
  238.       else if ((G2J(g)->flags & JIT_F_SSE2))
  239.         G2J(g)->flags |= (uint32_t)JIT_F_ON;
  240.       else
  241.         return 0/* Don't turn on JIT compiler without SSE2 support. */
  242. #else
  243.       else
  244.         G2J(g)->flags |= (uint32_t)JIT_F_ON;
  245. #endif
  246.       lj_dispatch_update(g);
  247.     }
  248.     break;
  249.   case LUAJIT_MODE_FUNC:
  250.   case LUAJIT_MODE_ALLFUNC:
  251.   case LUAJIT_MODE_ALLSUBFUNC: {
  252.     cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
  253.                   idx > 0 ? L->base + (idx-1) : L->top + idx;
  254.     GCproto *pt;
  255.     if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
  256.       pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
  257.     else if (tvisproto(tv))
  258.       pt = protoV(tv);
  259.     else
  260.       return 0/* Failed. */
  261.     if (mm != LUAJIT_MODE_ALLSUBFUNC)
  262.       setptmode(g, pt, mode);
  263.     if (mm != LUAJIT_MODE_FUNC)
  264.       setptmode_all(g, pt, mode);
  265.     break;
  266.     }
  267.   case LUAJIT_MODE_TRACE:
  268.     if (!(mode & LUAJIT_MODE_FLUSH))
  269.       return 0/* Failed. */
  270.     lj_trace_flush(G2J(g), idx);
  271.     break;
  272. #else
  273.   case LUAJIT_MODE_ENGINE:
  274.   case LUAJIT_MODE_FUNC:
  275.   case LUAJIT_MODE_ALLFUNC:
  276.   case LUAJIT_MODE_ALLSUBFUNC:
  277.     UNUSED(idx);
  278.     if ((mode & LUAJIT_MODE_ON))
  279.       return 0/* Failed. */
  280.     break;
  281. #endif
  282.   case LUAJIT_MODE_WRAPCFUNC:
  283.     if ((mode & LUAJIT_MODE_ON)) {
  284.       if (idx != 0) {
  285.         cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
  286.         if (tvislightud(tv))
  287.           g->wrapf = (lua_CFunction)lightudV(tv);
  288.         else
  289.           return 0/* Failed. */
  290.       } else {
  291.         return 0/* Failed. */
  292.       }
  293.       g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
  294.     } else {
  295.       g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
  296.     }
  297.     break;
  298.   default:
  299.     return 0/* Failed. */
  300.   }
  301.   return 1/* OK. */
  302. }

  303. /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
  304. LUA_API void LUAJIT_VERSION_SYM(void)
  305. {
  306. }

  307. /* -- Hooks --------------------------------------------------------------- */

  308. /* This function can be called asynchronously (e.g. during a signal). */
  309. LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
  310. {
  311.   global_State *g = G(L);
  312.   mask &= HOOK_EVENTMASK;
  313.   if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
  314.   g->hookf = func;
  315.   g->hookcount = g->hookcstart = (int32_t)count;
  316.   g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
  317.   lj_trace_abort(g);  /* Abort recording on any hook change. */
  318.   lj_dispatch_update(g);
  319.   return 1;
  320. }

  321. LUA_API lua_Hook lua_gethook(lua_State *L)
  322. {
  323.   return G(L)->hookf;
  324. }

  325. LUA_API int lua_gethookmask(lua_State *L)
  326. {
  327.   return G(L)->hookmask & HOOK_EVENTMASK;
  328. }

  329. LUA_API int lua_gethookcount(lua_State *L)
  330. {
  331.   return (int)G(L)->hookcstart;
  332. }

  333. /* Call a hook. */
  334. static void callhook(lua_State *L, int event, BCLine line)
  335. {
  336.   global_State *g = G(L);
  337.   lua_Hook hookf = g->hookf;
  338.   if (hookf && !hook_active(g)) {
  339.     lua_Debug ar;
  340.     lj_trace_abort(g);  /* Abort recording on any hook call. */
  341.     ar.event = event;
  342.     ar.currentline = line;
  343.     /* Top frame, nextframe = NULL. */
  344.     ar.i_ci = (int)((L->base-1) - tvref(L->stack));
  345.     lj_state_checkstack(L, 1+LUA_MINSTACK);
  346. #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
  347.     lj_profile_hook_enter(g);
  348. #else
  349.     hook_enter(g);
  350. #endif
  351.     hookf(L, &ar);
  352.     lua_assert(hook_active(g));
  353.     setgcref(g->cur_L, obj2gco(L));
  354. #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
  355.     lj_profile_hook_leave(g);
  356. #else
  357.     hook_leave(g);
  358. #endif
  359.   }
  360. }

  361. /* -- Dispatch callbacks -------------------------------------------------- */

  362. /* Calculate number of used stack slots in the current frame. */
  363. static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
  364. {
  365.   BCIns ins = pc[-1];
  366.   if (bc_op(ins) == BC_UCLO)
  367.     ins = pc[bc_j(ins)];
  368.   switch (bc_op(ins)) {
  369.   case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
  370.   case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
  371.   case BC_TSETM: return bc_a(ins) + nres-1;
  372.   default: return pt->framesize;
  373.   }
  374. }

  375. /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
  376. void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
  377. {
  378.   ERRNO_SAVE
  379.   GCfunc *fn = curr_func(L);
  380.   GCproto *pt = funcproto(fn);
  381.   void *cf = cframe_raw(L->cframe);
  382.   const BCIns *oldpc = cframe_pc(cf);
  383.   global_State *g = G(L);
  384.   BCReg slots;
  385.   setcframe_pc(cf, pc);
  386.   slots = cur_topslot(pt, pc, cframe_multres_n(cf));
  387.   L->top = L->base + slots;  /* Fix top. */
  388. #if LJ_HASJIT
  389.   {
  390.     jit_State *J = G2J(g);
  391.     if (J->state != LJ_TRACE_IDLE) {
  392. #ifdef LUA_USE_ASSERT
  393.       ptrdiff_t delta = L->top - L->base;
  394. #endif
  395.       J->L = L;
  396.       lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
  397.       lua_assert(L->top - L->base == delta);
  398.     }
  399.   }
  400. #endif
  401.   if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
  402.     g->hookcount = g->hookcstart;
  403.     callhook(L, LUA_HOOKCOUNT, -1);
  404.     L->top = L->base + slots;  /* Fix top again. */
  405.   }
  406.   if ((g->hookmask & LUA_MASKLINE)) {
  407.     BCPos npc = proto_bcpos(pt, pc) - 1;
  408.     BCPos opc = proto_bcpos(pt, oldpc) - 1;
  409.     BCLine line = lj_debug_line(pt, npc);
  410.     if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
  411.       callhook(L, LUA_HOOKLINE, line);
  412.       L->top = L->base + slots;  /* Fix top again. */
  413.     }
  414.   }
  415.   if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
  416.     callhook(L, LUA_HOOKRET, -1);
  417.   ERRNO_RESTORE
  418. }

  419. /* Initialize call. Ensure stack space and return # of missing parameters. */
  420. static int call_init(lua_State *L, GCfunc *fn)
  421. {
  422.   if (isluafunc(fn)) {
  423.     GCproto *pt = funcproto(fn);
  424.     int numparams = pt->numparams;
  425.     int gotparams = (int)(L->top - L->base);
  426.     int need = pt->framesize;
  427.     if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
  428.     lj_state_checkstack(L, (MSize)need);
  429.     numparams -= gotparams;
  430.     return numparams >= 0 ? numparams : 0;
  431.   } else {
  432.     lj_state_checkstack(L, LUA_MINSTACK);
  433.     return 0;
  434.   }
  435. }

  436. /* Call dispatch. Used by call hooks, hot calls or when recording. */
  437. ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
  438. {
  439.   ERRNO_SAVE
  440.   GCfunc *fn = curr_func(L);
  441.   BCOp op;
  442.   global_State *g = G(L);
  443. #if LJ_HASJIT
  444.   jit_State *J = G2J(g);
  445. #endif
  446.   int missing = call_init(L, fn);
  447. #if LJ_HASJIT
  448.   J->L = L;
  449.   if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
  450. #ifdef LUA_USE_ASSERT
  451.     ptrdiff_t delta = L->top - L->base;
  452. #endif
  453.     pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
  454.     lj_trace_hot(J, pc);
  455.     lua_assert(L->top - L->base == delta);
  456.     goto out;
  457.   } else if (J->state != LJ_TRACE_IDLE &&
  458.              !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
  459. #ifdef LUA_USE_ASSERT
  460.     ptrdiff_t delta = L->top - L->base;
  461. #endif
  462.     /* Record the FUNC* bytecodes, too. */
  463.     lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
  464.     lua_assert(L->top - L->base == delta);
  465.   }
  466. #endif
  467.   if ((g->hookmask & LUA_MASKCALL)) {
  468.     int i;
  469.     for (i = 0; i < missing; i++)  /* Add missing parameters. */
  470.       setnilV(L->top++);
  471.     callhook(L, LUA_HOOKCALL, -1);
  472.     /* Preserve modifications of missing parameters by lua_setlocal(). */
  473.     while (missing-- > 0 && tvisnil(L->top - 1))
  474.       L->top--;
  475.   }
  476. #if LJ_HASJIT
  477. out:
  478. #endif
  479.   op = bc_op(pc[-1]);  /* Get FUNC* op. */
  480. #if LJ_HASJIT
  481.   /* Use the non-hotcounting variants if JIT is off or while recording. */
  482.   if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
  483.       (op == BC_FUNCF || op == BC_FUNCV))
  484.     op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
  485. #endif
  486.   ERRNO_RESTORE
  487.   return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
  488. }

  489. #if LJ_HASJIT
  490. /* Stitch a new trace. */
  491. void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
  492. {
  493.   ERRNO_SAVE
  494.   lua_State *L = J->L;
  495.   void *cf = cframe_raw(L->cframe);
  496.   const BCIns *oldpc = cframe_pc(cf);
  497.   setcframe_pc(cf, pc);
  498.   /* Before dispatch, have to bias PC by 1. */
  499.   L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
  500.   lj_trace_stitch(J, pc-1);  /* Point to the CALL instruction. */
  501.   setcframe_pc(cf, oldpc);
  502.   ERRNO_RESTORE
  503. }
  504. #endif

  505. #if LJ_HASPROFILE
  506. /* Profile dispatch. */
  507. void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
  508. {
  509.   ERRNO_SAVE
  510.   GCfunc *fn = curr_func(L);
  511.   GCproto *pt = funcproto(fn);
  512.   void *cf = cframe_raw(L->cframe);
  513.   const BCIns *oldpc = cframe_pc(cf);
  514.   global_State *g;
  515.   setcframe_pc(cf, pc);
  516.   L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
  517.   lj_profile_interpreter(L);
  518.   setcframe_pc(cf, oldpc);
  519.   g = G(L);
  520.   setgcref(g->cur_L, obj2gco(L));
  521.   setvmstate(g, INTERP);
  522.   ERRNO_RESTORE
  523. }
  524. #endif