src/lj_record.c - luajit-2.0-src

Data types defined

Functions defined

Macros defined

Source code

  1. /*
  2. ** Trace recorder (bytecode -> SSA IR).
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. */

  5. #define lj_record_c
  6. #define LUA_CORE

  7. #include "lj_obj.h"

  8. #if LJ_HASJIT

  9. #include "lj_err.h"
  10. #include "lj_str.h"
  11. #include "lj_tab.h"
  12. #include "lj_meta.h"
  13. #include "lj_frame.h"
  14. #if LJ_HASFFI
  15. #include "lj_ctype.h"
  16. #endif
  17. #include "lj_bc.h"
  18. #include "lj_ff.h"
  19. #if LJ_HASPROFILE
  20. #include "lj_debug.h"
  21. #endif
  22. #include "lj_ir.h"
  23. #include "lj_jit.h"
  24. #include "lj_ircall.h"
  25. #include "lj_iropt.h"
  26. #include "lj_trace.h"
  27. #include "lj_record.h"
  28. #include "lj_ffrecord.h"
  29. #include "lj_snap.h"
  30. #include "lj_dispatch.h"
  31. #include "lj_vm.h"

  32. /* Some local macros to save typing. Undef'd at the end. */
  33. #define IR(ref)                        (&J->cur.ir[(ref)])

  34. /* Pass IR on to next optimization in chain (FOLD). */
  35. #define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))

  36. /* Emit raw IR without passing through optimizations. */
  37. #define emitir_raw(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))

  38. /* -- Sanity checks ------------------------------------------------------- */

  39. #ifdef LUA_USE_ASSERT
  40. /* Sanity check the whole IR -- sloooow. */
  41. static void rec_check_ir(jit_State *J)
  42. {
  43.   IRRef i, nins = J->cur.nins, nk = J->cur.nk;
  44.   lua_assert(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536);
  45.   for (i = nins-1; i >= nk; i--) {
  46.     IRIns *ir = IR(i);
  47.     uint32_t mode = lj_ir_mode[ir->o];
  48.     IRRef op1 = ir->op1;
  49.     IRRef op2 = ir->op2;
  50.     switch (irm_op1(mode)) {
  51.     case IRMnone: lua_assert(op1 == 0); break;
  52.     case IRMref: lua_assert(op1 >= nk);
  53.       lua_assert(i >= REF_BIAS ? op1 < i : op1 > i); break;
  54.     case IRMlit: break;
  55.     case IRMcst: lua_assert(i < REF_BIAS); continue;
  56.     }
  57.     switch (irm_op2(mode)) {
  58.     case IRMnone: lua_assert(op2 == 0); break;
  59.     case IRMref: lua_assert(op2 >= nk);
  60.       lua_assert(i >= REF_BIAS ? op2 < i : op2 > i); break;
  61.     case IRMlit: break;
  62.     case IRMcst: lua_assert(0); break;
  63.     }
  64.     if (ir->prev) {
  65.       lua_assert(ir->prev >= nk);
  66.       lua_assert(i >= REF_BIAS ? ir->prev < i : ir->prev > i);
  67.       lua_assert(ir->o == IR_NOP || IR(ir->prev)->o == ir->o);
  68.     }
  69.   }
  70. }

  71. /* Compare stack slots and frames of the recorder and the VM. */
  72. static void rec_check_slots(jit_State *J)
  73. {
  74.   BCReg s, nslots = J->baseslot + J->maxslot;
  75.   int32_t depth = 0;
  76.   cTValue *base = J->L->base - J->baseslot;
  77.   lua_assert(J->baseslot >= 1 && J->baseslot < LJ_MAX_JSLOTS);
  78.   lua_assert(J->baseslot == 1 || (J->slot[J->baseslot-1] & TREF_FRAME));
  79.   lua_assert(nslots < LJ_MAX_JSLOTS);
  80.   for (s = 0; s < nslots; s++) {
  81.     TRef tr = J->slot[s];
  82.     if (tr) {
  83.       cTValue *tv = &base[s];
  84.       IRRef ref = tref_ref(tr);
  85.       IRIns *ir;
  86.       lua_assert(ref >= J->cur.nk && ref < J->cur.nins);
  87.       ir = IR(ref);
  88.       lua_assert(irt_t(ir->t) == tref_t(tr));
  89.       if (s == 0) {
  90.         lua_assert(tref_isfunc(tr));
  91.       } else if ((tr & TREF_FRAME)) {
  92.         GCfunc *fn = gco2func(frame_gc(tv));
  93.         BCReg delta = (BCReg)(tv - frame_prev(tv));
  94.         lua_assert(tref_isfunc(tr));
  95.         if (tref_isk(tr)) lua_assert(fn == ir_kfunc(ir));
  96.         lua_assert(s > delta ? (J->slot[s-delta] & TREF_FRAME) : (s == delta));
  97.         depth++;
  98.       } else if ((tr & TREF_CONT)) {
  99.         lua_assert(ir_kptr(ir) == gcrefp(tv->gcr, void));
  100.         lua_assert((J->slot[s+1] & TREF_FRAME));
  101.         depth++;
  102.       } else {
  103.         if (tvisnumber(tv))
  104.           lua_assert(tref_isnumber(tr));  /* Could be IRT_INT etc., too. */
  105.         else
  106.           lua_assert(itype2irt(tv) == tref_type(tr));
  107.         if (tref_isk(tr)) {  /* Compare constants. */
  108.           TValue tvk;
  109.           lj_ir_kvalue(J->L, &tvk, ir);
  110.           if (!(tvisnum(&tvk) && tvisnan(&tvk)))
  111.             lua_assert(lj_obj_equal(tv, &tvk));
  112.           else
  113.             lua_assert(tvisnum(tv) && tvisnan(tv));
  114.         }
  115.       }
  116.     }
  117.   }
  118.   lua_assert(J->framedepth == depth);
  119. }
  120. #endif

  121. /* -- Type handling and specialization ------------------------------------ */

  122. /* Note: these functions return tagged references (TRef). */

  123. /* Specialize a slot to a specific type. Note: slot can be negative! */
  124. static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
  125. {
  126.   /* Caller may set IRT_GUARD in t. */
  127.   TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
  128.   J->base[slot] = ref;
  129.   return ref;
  130. }

  131. /* Specialize a slot to the runtime type. Note: slot can be negative! */
  132. static TRef sload(jit_State *J, int32_t slot)
  133. {
  134.   IRType t = itype2irt(&J->L->base[slot]);
  135.   TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot,
  136.                         IRSLOAD_TYPECHECK);
  137.   if (irtype_ispri(t)) ref = TREF_PRI(t);  /* Canonicalize primitive refs. */
  138.   J->base[slot] = ref;
  139.   return ref;
  140. }

  141. /* Get TRef from slot. Load slot and specialize if not done already. */
  142. #define getslot(J, s)        (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))

  143. /* Get TRef for current function. */
  144. static TRef getcurrf(jit_State *J)
  145. {
  146.   if (J->base[-1])
  147.     return J->base[-1];
  148.   lua_assert(J->baseslot == 1);
  149.   return sloadt(J, -1, IRT_FUNC, IRSLOAD_READONLY);
  150. }

  151. /* Compare for raw object equality.
  152. ** Returns 0 if the objects are the same.
  153. ** Returns 1 if they are different, but the same type.
  154. ** Returns 2 for two different types.
  155. ** Comparisons between primitives always return 1 -- no caller cares about it.
  156. */
  157. int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
  158. {
  159.   int diff = !lj_obj_equal(av, bv);
  160.   if (!tref_isk2(a, b)) {  /* Shortcut, also handles primitives. */
  161.     IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a);
  162.     IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b);
  163.     if (ta != tb) {
  164.       /* Widen mixed number/int comparisons to number/number comparison. */
  165.       if (ta == IRT_INT && tb == IRT_NUM) {
  166.         a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT);
  167.         ta = IRT_NUM;
  168.       } else if (ta == IRT_NUM && tb == IRT_INT) {
  169.         b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT);
  170.       } else {
  171.         return 2/* Two different types are never equal. */
  172.       }
  173.     }
  174.     emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
  175.   }
  176.   return diff;
  177. }

  178. /* Constify a value. Returns 0 for non-representable object types. */
  179. TRef lj_record_constify(jit_State *J, cTValue *o)
  180. {
  181.   if (tvisgcv(o))
  182.     return lj_ir_kgc(J, gcV(o), itype2irt(o));
  183.   else if (tvisint(o))
  184.     return lj_ir_kint(J, intV(o));
  185.   else if (tvisnum(o))
  186.     return lj_ir_knumint(J, numV(o));
  187.   else if (tvisbool(o))
  188.     return TREF_PRI(itype2irt(o));
  189.   else
  190.     return 0/* Can't represent lightuserdata (pointless). */
  191. }

  192. /* -- Record loop ops ----------------------------------------------------- */

  193. /* Loop event. */
  194. typedef enum {
  195.   LOOPEV_LEAVE,                /* Loop is left or not entered. */
  196.   LOOPEV_ENTERLO,        /* Loop is entered with a low iteration count left. */
  197.   LOOPEV_ENTER                /* Loop is entered. */
  198. } LoopEvent;

  199. /* Canonicalize slots: convert integers to numbers. */
  200. static void canonicalize_slots(jit_State *J)
  201. {
  202.   BCReg s;
  203.   if (LJ_DUALNUM) return;
  204.   for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
  205.     TRef tr = J->slot[s];
  206.     if (tref_isinteger(tr)) {
  207.       IRIns *ir = IR(tref_ref(tr));
  208.       if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY)))
  209.         J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
  210.     }
  211.   }
  212. }

  213. /* Stop recording. */
  214. void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
  215. {
  216.   lj_trace_end(J);
  217.   J->cur.linktype = (uint8_t)linktype;
  218.   J->cur.link = (uint16_t)lnk;
  219.   /* Looping back at the same stack level? */
  220.   if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) {
  221.     if ((J->flags & JIT_F_OPT_LOOP))  /* Shall we try to create a loop? */
  222.       goto nocanon;  /* Do not canonicalize or we lose the narrowing. */
  223.     if (J->cur.root)  /* Otherwise ensure we always link to the root trace. */
  224.       J->cur.link = J->cur.root;
  225.   }
  226.   canonicalize_slots(J);
  227. nocanon:
  228.   /* Note: all loop ops must set J->pc to the following instruction! */
  229.   lj_snap_add(J);  /* Add loop snapshot. */
  230.   J->needsnap = 0;
  231.   J->mergesnap = 1/* In case recording continues. */
  232. }

  233. /* Search bytecode backwards for a int/num constant slot initializer. */
  234. static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t)
  235. {
  236.   /* This algorithm is rather simplistic and assumes quite a bit about
  237.   ** how the bytecode is generated. It works fine for FORI initializers,
  238.   ** but it won't necessarily work in other cases (e.g. iterator arguments).
  239.   ** It doesn't do anything fancy, either (like backpropagating MOVs).
  240.   */
  241.   const BCIns *pc, *startpc = proto_bc(J->pt);
  242.   for (pc = endpc-1; pc > startpc; pc--) {
  243.     BCIns ins = *pc;
  244.     BCOp op = bc_op(ins);
  245.     /* First try to find the last instruction that stores to this slot. */
  246.     if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) {
  247.       return 0/* Multiple results, e.g. from a CALL or KNIL. */
  248.     } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) {
  249.       if (op == BC_KSHORT || op == BC_KNUM) {  /* Found const. initializer. */
  250.         /* Now try to verify there's no forward jump across it. */
  251.         const BCIns *kpc = pc;
  252.         for (; pc > startpc; pc--)
  253.           if (bc_op(*pc) == BC_JMP) {
  254.             const BCIns *target = pc+bc_j(*pc)+1;
  255.             if (target > kpc && target <= endpc)
  256.               return 0/* Conditional assignment. */
  257.           }
  258.         if (op == BC_KSHORT) {
  259.           int32_t k = (int32_t)(int16_t)bc_d(ins);
  260.           return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k);
  261.         } else {
  262.           cTValue *tv = proto_knumtv(J->pt, bc_d(ins));
  263.           if (t == IRT_INT) {
  264.             int32_t k = numberVint(tv);
  265.             if (tvisint(tv) || numV(tv) == (lua_Number)k)  /* -0 is ok here. */
  266.               return lj_ir_kint(J, k);
  267.             return 0/* Type mismatch. */
  268.           } else {
  269.             return lj_ir_knum(J, numberVnum(tv));
  270.           }
  271.         }
  272.       }
  273.       return 0/* Non-constant initializer. */
  274.     }
  275.   }
  276.   return 0/* No assignment to this slot found? */
  277. }

  278. /* Load and optionally convert a FORI argument from a slot. */
  279. static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode)
  280. {
  281.   int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0;
  282.   return sloadt(J, (int32_t)slot,
  283.                 t + (((mode & IRSLOAD_TYPECHECK) ||
  284.                       (conv && t == IRT_INT && !(mode >> 16))) ?
  285.                      IRT_GUARD : 0),
  286.                 mode + conv);
  287. }

  288. /* Peek before FORI to find a const initializer. Otherwise load from slot. */
  289. static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot,
  290.                      IRType t, int mode)
  291. {
  292.   TRef tr = J->base[slot];
  293.   if (!tr) {
  294.     tr = find_kinit(J, fori, slot, t);
  295.     if (!tr)
  296.       tr = fori_load(J, slot, t, mode);
  297.   }
  298.   return tr;
  299. }

  300. /* Return the direction of the FOR loop iterator.
  301. ** It's important to exactly reproduce the semantics of the interpreter.
  302. */
  303. static int rec_for_direction(cTValue *o)
  304. {
  305.   return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0;
  306. }

  307. /* Simulate the runtime behavior of the FOR loop iterator. */
  308. static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl)
  309. {
  310.   lua_Number stopv = numberVnum(&o[FORL_STOP]);
  311.   lua_Number idxv = numberVnum(&o[FORL_IDX]);
  312.   lua_Number stepv = numberVnum(&o[FORL_STEP]);
  313.   if (isforl)
  314.     idxv += stepv;
  315.   if (rec_for_direction(&o[FORL_STEP])) {
  316.     if (idxv <= stopv) {
  317.       *op = IR_LE;
  318.       return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
  319.     }
  320.     *op = IR_GT; return LOOPEV_LEAVE;
  321.   } else {
  322.     if (stopv <= idxv) {
  323.       *op = IR_GE;
  324.       return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
  325.     }
  326.     *op = IR_LT; return LOOPEV_LEAVE;
  327.   }
  328. }

  329. /* Record checks for FOR loop overflow and step direction. */
  330. static void rec_for_check(jit_State *J, IRType t, int dir,
  331.                           TRef stop, TRef step, int init)
  332. {
  333.   if (!tref_isk(step)) {
  334.     /* Non-constant step: need a guard for the direction. */
  335.     TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
  336.     emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
  337.     /* Add hoistable overflow checks for a narrowed FORL index. */
  338.     if (init && t == IRT_INT) {
  339.       if (tref_isk(stop)) {
  340.         /* Constant stop: optimize check away or to a range check for step. */
  341.         int32_t k = IR(tref_ref(stop))->i;
  342.         if (dir) {
  343.           if (k > 0)
  344.             emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
  345.         } else {
  346.           if (k < 0)
  347.             emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
  348.         }
  349.       } else {
  350.         /* Stop+step variable: need full overflow check. */
  351.         TRef tr = emitir(IRTGI(IR_ADDOV), step, stop);
  352.         emitir(IRTI(IR_USE), tr, 0);  /* ADDOV is weak. Avoid dead result. */
  353.       }
  354.     }
  355.   } else if (init && t == IRT_INT && !tref_isk(stop)) {
  356.     /* Constant step: optimize overflow check to a range check for stop. */
  357.     int32_t k = IR(tref_ref(step))->i;
  358.     k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
  359.     emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
  360.   }
  361. }

  362. /* Record a FORL instruction. */
  363. static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
  364.                          int init)
  365. {
  366.   BCReg ra = bc_a(*fori);
  367.   cTValue *tv = &J->L->base[ra];
  368.   TRef idx = J->base[ra+FORL_IDX];
  369.   IRType t = idx ? tref_type(idx) :
  370.              (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM;
  371.   int mode = IRSLOAD_INHERIT +
  372.     ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0);
  373.   TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
  374.   TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
  375.   int tc, dir = rec_for_direction(&tv[FORL_STEP]);
  376.   lua_assert(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI);
  377.   scev->t.irt = t;
  378.   scev->dir = dir;
  379.   scev->stop = tref_ref(stop);
  380.   scev->step = tref_ref(step);
  381.   rec_for_check(J, t, dir, stop, step, init);
  382.   scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT));
  383.   tc = (LJ_DUALNUM &&
  384.         !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) &&
  385.           tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ?
  386.         IRSLOAD_TYPECHECK : 0;
  387.   if (tc) {
  388.     J->base[ra+FORL_STOP] = stop;
  389.     J->base[ra+FORL_STEP] = step;
  390.   }
  391.   if (!idx)
  392.     idx = fori_load(J, ra+FORL_IDX, t,
  393.                     IRSLOAD_INHERIT + tc + (J->scev.start << 16));
  394.   if (!init)
  395.     J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
  396.   J->base[ra+FORL_EXT] = idx;
  397.   scev->idx = tref_ref(idx);
  398.   setmref(scev->pc, fori);
  399.   J->maxslot = ra+FORL_EXT+1;
  400. }

  401. /* Record FORL/JFORL or FORI/JFORI. */
  402. static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
  403. {
  404.   BCReg ra = bc_a(*fori);
  405.   TValue *tv = &J->L->base[ra];
  406.   TRef *tr = &J->base[ra];
  407.   IROp op;
  408.   LoopEvent ev;
  409.   TRef stop;
  410.   IRType t;
  411.   if (isforl) {  /* Handle FORL/JFORL opcodes. */
  412.     TRef idx = tr[FORL_IDX];
  413.     if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) {
  414.       t = J->scev.t.irt;
  415.       stop = J->scev.stop;
  416.       idx = emitir(IRT(IR_ADD, t), idx, J->scev.step);
  417.       tr[FORL_EXT] = tr[FORL_IDX] = idx;
  418.     } else {
  419.       ScEvEntry scev;
  420.       rec_for_loop(J, fori, &scev, 0);
  421.       t = scev.t.irt;
  422.       stop = scev.stop;
  423.     }
  424.   } else/* Handle FORI/JFORI opcodes. */
  425.     BCReg i;
  426.     lj_meta_for(J->L, tv);
  427.     t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) :
  428.                                                    IRT_NUM;
  429.     for (i = FORL_IDX; i <= FORL_STEP; i++) {
  430.       if (!tr[i]) sload(J, ra+i);
  431.       lua_assert(tref_isnumber_str(tr[i]));
  432.       if (tref_isstr(tr[i]))
  433.         tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
  434.       if (t == IRT_INT) {
  435.         if (!tref_isinteger(tr[i]))
  436.           tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK);
  437.       } else {
  438.         if (!tref_isnum(tr[i]))
  439.           tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT);
  440.       }
  441.     }
  442.     tr[FORL_EXT] = tr[FORL_IDX];
  443.     stop = tr[FORL_STOP];
  444.     rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]),
  445.                   stop, tr[FORL_STEP], 1);
  446.   }

  447.   ev = rec_for_iter(&op, tv, isforl);
  448.   if (ev == LOOPEV_LEAVE) {
  449.     J->maxslot = ra+FORL_EXT+1;
  450.     J->pc = fori+1;
  451.   } else {
  452.     J->maxslot = ra;
  453.     J->pc = fori+bc_j(*fori)+1;
  454.   }
  455.   lj_snap_add(J);

  456.   emitir(IRTG(op, t), tr[FORL_IDX], stop);

  457.   if (ev == LOOPEV_LEAVE) {
  458.     J->maxslot = ra;
  459.     J->pc = fori+bc_j(*fori)+1;
  460.   } else {
  461.     J->maxslot = ra+FORL_EXT+1;
  462.     J->pc = fori+1;
  463.   }
  464.   J->needsnap = 1;
  465.   return ev;
  466. }

  467. /* Record ITERL/JITERL. */
  468. static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
  469. {
  470.   BCReg ra = bc_a(iterins);
  471.   lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  472.   if (!tref_isnil(getslot(J, ra))) {  /* Looping back? */
  473.     J->base[ra-1] = J->base[ra];  /* Copy result of ITERC to control var. */
  474.     J->maxslot = ra-1+bc_b(J->pc[-1]);
  475.     J->pc += bc_j(iterins)+1;
  476.     return LOOPEV_ENTER;
  477.   } else {
  478.     J->maxslot = ra-3;
  479.     J->pc++;
  480.     return LOOPEV_LEAVE;
  481.   }
  482. }

  483. /* Record LOOP/JLOOP. Now, that was easy. */
  484. static LoopEvent rec_loop(jit_State *J, BCReg ra)
  485. {
  486.   if (ra < J->maxslot) J->maxslot = ra;
  487.   J->pc++;
  488.   return LOOPEV_ENTER;
  489. }

  490. /* Check if a loop repeatedly failed to trace because it didn't loop back. */
  491. static int innerloopleft(jit_State *J, const BCIns *pc)
  492. {
  493.   ptrdiff_t i;
  494.   for (i = 0; i < PENALTY_SLOTS; i++)
  495.     if (mref(J->penalty[i].pc, const BCIns) == pc) {
  496.       if ((J->penalty[i].reason == LJ_TRERR_LLEAVE ||
  497.            J->penalty[i].reason == LJ_TRERR_LINNER) &&
  498.           J->penalty[i].val >= 2*PENALTY_MIN)
  499.         return 1;
  500.       break;
  501.     }
  502.   return 0;
  503. }

  504. /* Handle the case when an interpreted loop op is hit. */
  505. static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
  506. {
  507.   if (J->parent == 0 && J->exitno == 0) {
  508.     if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
  509.       /* Same loop? */
  510.       if (ev == LOOPEV_LEAVE)  /* Must loop back to form a root trace. */
  511.         lj_trace_err(J, LJ_TRERR_LLEAVE);
  512.       lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Looping trace. */
  513.     } else if (ev != LOOPEV_LEAVE) {  /* Entering inner loop? */
  514.       /* It's usually better to abort here and wait until the inner loop
  515.       ** is traced. But if the inner loop repeatedly didn't loop back,
  516.       ** this indicates a low trip count. In this case try unrolling
  517.       ** an inner loop even in a root trace. But it's better to be a bit
  518.       ** more conservative here and only do it for very short loops.
  519.       */
  520.       if (bc_j(*pc) != -1 && !innerloopleft(J, pc))
  521.         lj_trace_err(J, LJ_TRERR_LINNER);  /* Root trace hit an inner loop. */
  522.       if ((ev != LOOPEV_ENTERLO &&
  523.            J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0)
  524.         lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
  525.       J->loopref = J->cur.nins;
  526.     }
  527.   } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters an inner loop. */
  528.     J->loopref = J->cur.nins;
  529.     if (--J->loopunroll < 0)
  530.       lj_trace_err(J, LJ_TRERR_LUNROLL);  /* Limit loop unrolling. */
  531.   }  /* Side trace continues across a loop that's left or not entered. */
  532. }

  533. /* Handle the case when an already compiled loop op is hit. */
  534. static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
  535. {
  536.   if (J->parent == 0 && J->exitno == 0) {  /* Root trace hit an inner loop. */
  537.     /* Better let the inner loop spawn a side trace back here. */
  538.     lj_trace_err(J, LJ_TRERR_LINNER);
  539.   } else if (ev != LOOPEV_LEAVE) {  /* Side trace enters a compiled loop. */
  540.     J->instunroll = 0/* Cannot continue across a compiled loop op. */
  541.     if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
  542.       lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno);  /* Form extra loop. */
  543.     else
  544.       lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the loop. */
  545.   }  /* Side trace continues across a loop that's left or not entered. */
  546. }

  547. /* -- Record profiler hook checks ----------------------------------------- */

  548. #if LJ_HASPROFILE

  549. /* Need to insert profiler hook check? */
  550. static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
  551. {
  552.   GCproto *ppt;
  553.   lua_assert(J->prof_mode == 'f' || J->prof_mode == 'l');
  554.   if (!pt)
  555.     return 0;
  556.   ppt = J->prev_pt;
  557.   J->prev_pt = pt;
  558.   if (pt != ppt && ppt) {
  559.     J->prev_line = -1;
  560.     return 1;
  561.   }
  562.   if (J->prof_mode == 'l') {
  563.     BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
  564.     BCLine pline = J->prev_line;
  565.     J->prev_line = line;
  566.     if (pline != line)
  567.       return 1;
  568.   }
  569.   return 0;
  570. }

  571. static void rec_profile_ins(jit_State *J, const BCIns *pc)
  572. {
  573.   if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
  574.     emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
  575.     lj_snap_add(J);
  576.   }
  577. }

  578. static void rec_profile_ret(jit_State *J)
  579. {
  580.   if (J->prof_mode == 'f') {
  581.     emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
  582.     J->prev_pt = NULL;
  583.     lj_snap_add(J);
  584.   }
  585. }

  586. #endif

  587. /* -- Record calls and returns -------------------------------------------- */

  588. /* Specialize to the runtime value of the called function or its prototype. */
  589. static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
  590. {
  591.   TRef kfunc;
  592.   if (isluafunc(fn)) {
  593.     GCproto *pt = funcproto(fn);
  594.     /* Too many closures created? Probably not a monomorphic function. */
  595.     if (pt->flags >= PROTO_CLC_POLY) {  /* Specialize to prototype instead. */
  596.       TRef trpt = emitir(IRT(IR_FLOAD, IRT_P32), tr, IRFL_FUNC_PC);
  597.       emitir(IRTG(IR_EQ, IRT_P32), trpt, lj_ir_kptr(J, proto_bc(pt)));
  598.       (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);  /* Prevent GC of proto. */
  599.       return tr;
  600.     }
  601.   } else {
  602.     /* Don't specialize to non-monomorphic builtins. */
  603.     switch (fn->c.ffid) {
  604.     case FF_coroutine_wrap_aux:
  605.     case FF_string_gmatch_aux:
  606.       /* NYI: io_file_iter doesn't have an ffid, yet. */
  607.       {  /* Specialize to the ffid. */
  608.         TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
  609.         emitir(IRTG(IR_EQ, IRT_INT), trid, lj_ir_kint(J, fn->c.ffid));
  610.       }
  611.       return tr;
  612.     default:
  613.       /* NYI: don't specialize to non-monomorphic C functions. */
  614.       break;
  615.     }
  616.   }
  617.   /* Otherwise specialize to the function (closure) value itself. */
  618.   kfunc = lj_ir_kfunc(J, fn);
  619.   emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc);
  620.   return kfunc;
  621. }

  622. /* Record call setup. */
  623. static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
  624. {
  625.   RecordIndex ix;
  626.   TValue *functv = &J->L->base[func];
  627.   TRef *fbase = &J->base[func];
  628.   ptrdiff_t i;
  629.   lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  630.   for (i = 0; i <= nargs; i++)
  631.     (void)getslot(J, func+i);  /* Ensure func and all args have a reference. */
  632.   if (!tref_isfunc(fbase[0])) {  /* Resolve __call metamethod. */
  633.     ix.tab = fbase[0];
  634.     copyTV(J->L, &ix.tabv, functv);
  635.     if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
  636.       lj_trace_err(J, LJ_TRERR_NOMM);
  637.     for (i = ++nargs; i > 0; i--)  /* Shift arguments up. */
  638.       fbase[i] = fbase[i-1];
  639.     fbase[0] = ix.mobj;  /* Replace function. */
  640.     functv = &ix.mobjv;
  641.   }
  642.   fbase[0] = TREF_FRAME | rec_call_specialize(J, funcV(functv), fbase[0]);
  643.   J->maxslot = (BCReg)nargs;
  644. }

  645. /* Record call. */
  646. void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
  647. {
  648.   rec_call_setup(J, func, nargs);
  649.   /* Bump frame. */
  650.   J->framedepth++;
  651.   J->base += func+1;
  652.   J->baseslot += func+1;
  653. }

  654. /* Record tail call. */
  655. void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
  656. {
  657.   rec_call_setup(J, func, nargs);
  658.   if (frame_isvarg(J->L->base - 1)) {
  659.     BCReg cbase = (BCReg)frame_delta(J->L->base - 1);
  660.     if (--J->framedepth < 0)
  661.       lj_trace_err(J, LJ_TRERR_NYIRETL);
  662.     J->baseslot -= (BCReg)cbase;
  663.     J->base -= cbase;
  664.     func += cbase;
  665.   }
  666.   /* Move func + args down. */
  667.   memmove(&J->base[-1], &J->base[func], sizeof(TRef)*(J->maxslot+1));
  668.   /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
  669.   /* Tailcalls can form a loop, so count towards the loop unroll limit. */
  670.   if (++J->tailcalled > J->loopunroll)
  671.     lj_trace_err(J, LJ_TRERR_LUNROLL);
  672. }

  673. /* Check unroll limits for down-recursion. */
  674. static int check_downrec_unroll(jit_State *J, GCproto *pt)
  675. {
  676.   IRRef ptref;
  677.   for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev)
  678.     if (ir_kgc(IR(ptref)) == obj2gco(pt)) {
  679.       int count = 0;
  680.       IRRef ref;
  681.       for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev)
  682.         if (IR(ref)->op1 == ptref)
  683.           count++;
  684.       if (count) {
  685.         if (J->pc == J->startpc) {
  686.           if (count + J->tailcalled > J->param[JIT_P_recunroll])
  687.             return 1;
  688.         } else {
  689.           lj_trace_err(J, LJ_TRERR_DOWNREC);
  690.         }
  691.       }
  692.     }
  693.   return 0;
  694. }

  695. static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);

  696. /* Record return. */
  697. void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
  698. {
  699.   TValue *frame = J->L->base - 1;
  700.   ptrdiff_t i;
  701.   for (i = 0; i < gotresults; i++)
  702.     (void)getslot(J, rbase+i);  /* Ensure all results have a reference. */
  703.   while (frame_ispcall(frame)) {  /* Immediately resolve pcall() returns. */
  704.     BCReg cbase = (BCReg)frame_delta(frame);
  705.     if (--J->framedepth < 0)
  706.       lj_trace_err(J, LJ_TRERR_NYIRETL);
  707.     lua_assert(J->baseslot > 1);
  708.     gotresults++;
  709.     rbase += cbase;
  710.     J->baseslot -= (BCReg)cbase;
  711.     J->base -= cbase;
  712.     J->base[--rbase] = TREF_TRUE/* Prepend true to results. */
  713.     frame = frame_prevd(frame);
  714.   }
  715.   /* Return to lower frame via interpreter for unhandled cases. */
  716.   if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
  717.        (!frame_islua(frame) ||
  718.         (J->parent == 0 && J->exitno == 0 &&
  719.          !bc_isret(bc_op(J->cur.startins))))) {
  720.     /* NYI: specialize to frame type and return directly, not via RET*. */
  721.     for (i = 0; i < (ptrdiff_t)rbase; i++)
  722.       J->base[i] = 0/* Purge dead slots. */
  723.     J->maxslot = rbase + (BCReg)gotresults;
  724.     lj_record_stop(J, LJ_TRLINK_RETURN, 0);  /* Return to interpreter. */
  725.     return;
  726.   }
  727.   if (frame_isvarg(frame)) {
  728.     BCReg cbase = (BCReg)frame_delta(frame);
  729.     if (--J->framedepth < 0/* NYI: return of vararg func to lower frame. */
  730.       lj_trace_err(J, LJ_TRERR_NYIRETL);
  731.     lua_assert(J->baseslot > 1);
  732.     rbase += cbase;
  733.     J->baseslot -= (BCReg)cbase;
  734.     J->base -= cbase;
  735.     frame = frame_prevd(frame);
  736.   }
  737.   if (frame_islua(frame)) {  /* Return to Lua frame. */
  738.     BCIns callins = *(frame_pc(frame)-1);
  739.     ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
  740.     BCReg cbase = bc_a(callins);
  741.     GCproto *pt = funcproto(frame_func(frame - (cbase+1-LJ_FR2)));
  742.     lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame teardown. */
  743.     if ((pt->flags & PROTO_NOJIT))
  744.       lj_trace_err(J, LJ_TRERR_CJITOFF);
  745.     if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
  746.       if (check_downrec_unroll(J, pt)) {
  747.         J->maxslot = (BCReg)(rbase + gotresults);
  748.         lj_snap_purge(J);
  749.         lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno);  /* Down-rec. */
  750.         return;
  751.       }
  752.       lj_snap_add(J);
  753.     }
  754.     for (i = 0; i < nresults; i++)  /* Adjust results. */
  755.       J->base[i-1] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
  756.     J->maxslot = cbase+(BCReg)nresults;
  757.     if (J->framedepth > 0) {  /* Return to a frame that is part of the trace. */
  758.       J->framedepth--;
  759.       lua_assert(J->baseslot > cbase+1);
  760.       J->baseslot -= cbase+1;
  761.       J->base -= cbase+1;
  762.     } else if (J->parent == 0 && J->exitno == 0 &&
  763.                !bc_isret(bc_op(J->cur.startins))) {
  764.       /* Return to lower frame would leave the loop in a root trace. */
  765.       lj_trace_err(J, LJ_TRERR_LLEAVE);
  766.     } else if (J->needsnap) {  /* Tailcalled to ff with side-effects. */
  767.       lj_trace_err(J, LJ_TRERR_NYIRETL);  /* No way to insert snapshot here. */
  768.     } else/* Return to lower frame. Guard for the target we return to. */
  769.       TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
  770.       TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
  771.       emitir(IRTG(IR_RETF, IRT_P32), trpt, trpc);
  772.       J->retdepth++;
  773.       J->needsnap = 1;
  774.       lua_assert(J->baseslot == 1);
  775.       /* Shift result slots up and clear the slots of the new frame below. */
  776.       memmove(J->base + cbase, J->base-1, sizeof(TRef)*nresults);
  777.       memset(J->base-1, 0, sizeof(TRef)*(cbase+1));
  778.     }
  779.   } else if (frame_iscont(frame)) {  /* Return to continuation frame. */
  780.     ASMFunction cont = frame_contf(frame);
  781.     BCReg cbase = (BCReg)frame_delta(frame);
  782.     if ((J->framedepth -= 2) < 0)
  783.       lj_trace_err(J, LJ_TRERR_NYIRETL);
  784.     J->baseslot -= (BCReg)cbase;
  785.     J->base -= cbase;
  786.     J->maxslot = cbase-2;
  787.     if (cont == lj_cont_ra) {
  788.       /* Copy result to destination slot. */
  789.       BCReg dst = bc_a(*(frame_contpc(frame)-1));
  790.       J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
  791.       if (dst >= J->maxslot) J->maxslot = dst+1;
  792.     } else if (cont == lj_cont_nop) {
  793.       /* Nothing to do here. */
  794.     } else if (cont == lj_cont_cat) {
  795.       BCReg bslot = bc_b(*(frame_contpc(frame)-1));
  796.       TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
  797.       if (bslot != cbase-2) {  /* Concatenate the remainder. */
  798.         TValue *b = J->L->base, save/* Simulate lower frame and result. */
  799.         J->base[cbase-2] = tr;
  800.         copyTV(J->L, &save, b-2);
  801.         if (gotresults) copyTV(J->L, b-2, b+rbase); else setnilV(b-2);
  802.         J->L->base = b - cbase;
  803.         tr = rec_cat(J, bslot, cbase-2);
  804.         b = J->L->base + cbase;  /* Undo. */
  805.         J->L->base = b;
  806.         copyTV(J->L, b-2, &save);
  807.       }
  808.       if (tr) {  /* Store final result. */
  809.         BCReg dst = bc_a(*(frame_contpc(frame)-1));
  810.         J->base[dst] = tr;
  811.         if (dst >= J->maxslot) J->maxslot = dst+1;
  812.       }  /* Otherwise continue with another __concat call. */
  813.     } else {
  814.       /* Result type already specialized. */
  815.       lua_assert(cont == lj_cont_condf || cont == lj_cont_condt);
  816.     }
  817.   } else {
  818.     lj_trace_err(J, LJ_TRERR_NYIRETL);  /* NYI: handle return to C frame. */
  819.   }
  820.   lua_assert(J->baseslot >= 1);
  821. }

  822. /* -- Metamethod handling ------------------------------------------------- */

  823. /* Prepare to record call to metamethod. */
  824. static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
  825. {
  826.   BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
  827. #if LJ_64
  828.   TRef trcont = lj_ir_kptr(J, (void *)((int64_t)cont-(int64_t)lj_vm_asm_begin));
  829. #else
  830.   TRef trcont = lj_ir_kptr(J, (void *)cont);
  831. #endif
  832.   J->base[top] = trcont | TREF_CONT;
  833.   J->framedepth++;
  834.   for (s = J->maxslot; s < top; s++)
  835.     J->base[s] = 0/* Clear frame gap to avoid resurrecting previous refs. */
  836.   return top+1;
  837. }

  838. /* Record metamethod lookup. */
  839. int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
  840. {
  841.   RecordIndex mix;
  842.   GCtab *mt;
  843.   if (tref_istab(ix->tab)) {
  844.     mt = tabref(tabV(&ix->tabv)->metatable);
  845.     mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
  846.   } else if (tref_isudata(ix->tab)) {
  847.     int udtype = udataV(&ix->tabv)->udtype;
  848.     mt = tabref(udataV(&ix->tabv)->metatable);
  849.     /* The metatables of special userdata objects are treated as immutable. */
  850.     if (udtype != UDTYPE_USERDATA) {
  851.       cTValue *mo;
  852.       if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
  853.         /* Specialize to the C library namespace object. */
  854.         emitir(IRTG(IR_EQ, IRT_P32), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
  855.       } else {
  856.         /* Specialize to the type of userdata. */
  857.         TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
  858.         emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype));
  859.       }
  860.   immutable_mt:
  861.       mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm));
  862.       if (!mo || tvisnil(mo))
  863.         return 0/* No metamethod. */
  864.       /* Treat metamethod or index table as immutable, too. */
  865.       if (!(tvisfunc(mo) || tvistab(mo)))
  866.         lj_trace_err(J, LJ_TRERR_BADTYPE);
  867.       copyTV(J->L, &ix->mobjv, mo);
  868.       ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB);
  869.       ix->mtv = mt;
  870.       ix->mt = TREF_NIL/* Dummy value for comparison semantics. */
  871.       return 1/* Got metamethod or index table. */
  872.     }
  873.     mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
  874.   } else {
  875.     /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
  876.     mt = tabref(basemt_obj(J2G(J), &ix->tabv));
  877.     if (mt == NULL) {
  878.       ix->mt = TREF_NIL;
  879.       return 0/* No metamethod. */
  880.     }
  881.     /* The cdata metatable is treated as immutable. */
  882.     if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt;
  883.     ix->mt = mix.tab = lj_ir_ktab(J, mt);
  884.     goto nocheck;
  885.   }
  886.   ix->mt = mt ? mix.tab : TREF_NIL;
  887.   emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
  888. nocheck:
  889.   if (mt) {
  890.     GCstr *mmstr = mmname_str(J2G(J), mm);
  891.     cTValue *mo = lj_tab_getstr(mt, mmstr);
  892.     if (mo && !tvisnil(mo))
  893.       copyTV(J->L, &ix->mobjv, mo);
  894.     ix->mtv = mt;
  895.     settabV(J->L, &mix.tabv, mt);
  896.     setstrV(J->L, &mix.keyv, mmstr);
  897.     mix.key = lj_ir_kstr(J, mmstr);
  898.     mix.val = 0;
  899.     mix.idxchain = 0;
  900.     ix->mobj = lj_record_idx(J, &mix);
  901.     return !tref_isnil(ix->mobj);  /* 1 if metamethod found, 0 if not. */
  902.   }
  903.   return 0/* No metamethod. */
  904. }

  905. /* Record call to arithmetic metamethod. */
  906. static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
  907. {
  908.   /* Set up metamethod call first to save ix->tab and ix->tabv. */
  909.   BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
  910.   TRef *base = J->base + func;
  911.   TValue *basev = J->L->base + func;
  912.   base[1] = ix->tab; base[2] = ix->key;
  913.   copyTV(J->L, basev+1, &ix->tabv);
  914.   copyTV(J->L, basev+2, &ix->keyv);
  915.   if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
  916.     if (mm != MM_unm) {
  917.       ix->tab = ix->key;
  918.       copyTV(J->L, &ix->tabv, &ix->keyv);
  919.       if (lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
  920.         goto ok;
  921.     }
  922.     lj_trace_err(J, LJ_TRERR_NOMM);
  923.   }
  924. ok:
  925.   lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  926.   base[0] = ix->mobj;
  927.   copyTV(J->L, basev+0, &ix->mobjv);
  928.   lj_record_call(J, func, 2);
  929.   return 0/* No result yet. */
  930. }

  931. /* Record call to __len metamethod. */
  932. static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
  933. {
  934.   RecordIndex ix;
  935.   ix.tab = tr;
  936.   copyTV(J->L, &ix.tabv, tv);
  937.   if (lj_record_mm_lookup(J, &ix, MM_len)) {
  938.     BCReg func = rec_mm_prep(J, lj_cont_ra);
  939.     TRef *base = J->base + func;
  940.     TValue *basev = J->L->base + func;
  941.     lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  942.     base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
  943.     base[1] = tr; copyTV(J->L, basev+1, tv);
  944. #if LJ_52
  945.     base[2] = tr; copyTV(J->L, basev+2, tv);
  946. #else
  947.     base[2] = TREF_NIL; setnilV(basev+2);
  948. #endif
  949.     lj_record_call(J, func, 2);
  950.   } else {
  951.     if (LJ_52 && tref_istab(tr))
  952.       return lj_ir_call(J, IRCALL_lj_tab_len, tr);
  953.     lj_trace_err(J, LJ_TRERR_NOMM);
  954.   }
  955.   return 0/* No result yet. */
  956. }

  957. /* Call a comparison metamethod. */
  958. static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
  959. {
  960.   BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
  961.   TRef *base = J->base + func;
  962.   TValue *tv = J->L->base + func;
  963.   lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  964.   base[0] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
  965.   copyTV(J->L, tv+0, &ix->mobjv);
  966.   copyTV(J->L, tv+1, &ix->valv);
  967.   copyTV(J->L, tv+2, &ix->keyv);
  968.   lj_record_call(J, func, 2);
  969. }

  970. /* Record call to equality comparison metamethod (for tab and udata only). */
  971. static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
  972. {
  973.   ix->tab = ix->val;
  974.   copyTV(J->L, &ix->tabv, &ix->valv);
  975.   if (lj_record_mm_lookup(J, ix, MM_eq)) {  /* Lookup mm on 1st operand. */
  976.     cTValue *bv;
  977.     TRef mo1 = ix->mobj;
  978.     TValue mo1v;
  979.     copyTV(J->L, &mo1v, &ix->mobjv);
  980.     /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
  981.     bv = &ix->keyv;
  982.     if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
  983.       TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
  984.       emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
  985.     } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
  986.       TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
  987.       emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
  988.     } else/* Lookup metamethod on 2nd operand and compare both. */
  989.       ix->tab = ix->key;
  990.       copyTV(J->L, &ix->tabv, bv);
  991.       if (!lj_record_mm_lookup(J, ix, MM_eq) ||
  992.           lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
  993.         return;
  994.     }
  995.     rec_mm_callcomp(J, ix, op);
  996.   }
  997. }

  998. /* Record call to ordered comparison metamethods (for arbitrary objects). */
  999. static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
  1000. {
  1001.   ix->tab = ix->val;
  1002.   copyTV(J->L, &ix->tabv, &ix->valv);
  1003.   while (1) {
  1004.     MMS mm = (op & 2) ? MM_le : MM_lt;  /* Try __le + __lt or only __lt. */
  1005. #if LJ_52
  1006.     if (!lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
  1007.       ix->tab = ix->key;
  1008.       copyTV(J->L, &ix->tabv, &ix->keyv);
  1009.       if (!lj_record_mm_lookup(J, ix, mm))  /* Lookup mm on 2nd operand. */
  1010.         goto nomatch;
  1011.     }
  1012.     rec_mm_callcomp(J, ix, op);
  1013.     return;
  1014. #else
  1015.     if (lj_record_mm_lookup(J, ix, mm)) {  /* Lookup mm on 1st operand. */
  1016.       cTValue *bv;
  1017.       TRef mo1 = ix->mobj;
  1018.       TValue mo1v;
  1019.       copyTV(J->L, &mo1v, &ix->mobjv);
  1020.       /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
  1021.       bv = &ix->keyv;
  1022.       if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
  1023.         TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
  1024.         emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
  1025.       } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
  1026.         TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
  1027.         emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
  1028.       } else/* Lookup metamethod on 2nd operand and compare both. */
  1029.         ix->tab = ix->key;
  1030.         copyTV(J->L, &ix->tabv, bv);
  1031.         if (!lj_record_mm_lookup(J, ix, mm) ||
  1032.             lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
  1033.           goto nomatch;
  1034.       }
  1035.       rec_mm_callcomp(J, ix, op);
  1036.       return;
  1037.     }
  1038. #endif
  1039.   nomatch:
  1040.     /* Lookup failed. Retry with  __lt and swapped operands. */
  1041.     if (!(op & 2)) break/* Already at __lt. Interpreter will throw. */
  1042.     ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
  1043.     copyTV(J->L, &ix->tabv, &ix->keyv);
  1044.     copyTV(J->L, &ix->keyv, &ix->valv);
  1045.     copyTV(J->L, &ix->valv, &ix->tabv);
  1046.     op ^= 3;
  1047.   }
  1048. }

  1049. #if LJ_HASFFI
  1050. /* Setup call to cdata comparison metamethod. */
  1051. static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
  1052. {
  1053.   lj_snap_add(J);
  1054.   if (tref_iscdata(ix->val)) {
  1055.     ix->tab = ix->val;
  1056.     copyTV(J->L, &ix->tabv, &ix->valv);
  1057.   } else {
  1058.     lua_assert(tref_iscdata(ix->key));
  1059.     ix->tab = ix->key;
  1060.     copyTV(J->L, &ix->tabv, &ix->keyv);
  1061.   }
  1062.   lj_record_mm_lookup(J, ix, mm);
  1063.   rec_mm_callcomp(J, ix, op);
  1064. }
  1065. #endif

  1066. /* -- Indexed access ------------------------------------------------------ */

  1067. /* Record bounds-check. */
  1068. static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
  1069. {
  1070.   /* Try to emit invariant bounds checks. */
  1071.   if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) ==
  1072.       (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) {
  1073.     IRRef ref = tref_ref(ikey);
  1074.     IRIns *ir = IR(ref);
  1075.     int32_t ofs = 0;
  1076.     IRRef ofsref = 0;
  1077.     /* Handle constant offsets. */
  1078.     if (ir->o == IR_ADD && irref_isk(ir->op2)) {
  1079.       ofsref = ir->op2;
  1080.       ofs = IR(ofsref)->i;
  1081.       ref = ir->op1;
  1082.       ir = IR(ref);
  1083.     }
  1084.     /* Got scalar evolution analysis results for this reference? */
  1085.     if (ref == J->scev.idx) {
  1086.       int32_t stop;
  1087.       lua_assert(irt_isint(J->scev.t) && ir->o == IR_SLOAD);
  1088.       stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
  1089.       /* Runtime value for stop of loop is within bounds? */
  1090.       if ((uint64_t)stop + ofs < (uint64_t)asize) {
  1091.         /* Emit invariant bounds check for stop. */
  1092.         emitir(IRTG(IR_ABC, IRT_P32), asizeref, ofs == 0 ? J->scev.stop :
  1093.                emitir(IRTI(IR_ADD), J->scev.stop, ofsref));
  1094.         /* Emit invariant bounds check for start, if not const or negative. */
  1095.         if (!(J->scev.dir && J->scev.start &&
  1096.               (int64_t)IR(J->scev.start)->i + ofs >= 0))
  1097.           emitir(IRTG(IR_ABC, IRT_P32), asizeref, ikey);
  1098.         return;
  1099.       }
  1100.     }
  1101.   }
  1102.   emitir(IRTGI(IR_ABC), asizeref, ikey);  /* Emit regular bounds check. */
  1103. }

  1104. /* Record indexed key lookup. */
  1105. static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref)
  1106. {
  1107.   TRef key;
  1108.   GCtab *t = tabV(&ix->tabv);
  1109.   ix->oldv = lj_tab_get(J->L, t, &ix->keyv);  /* Lookup previous value. */
  1110.   *rbref = 0;

  1111.   /* Integer keys are looked up in the array part first. */
  1112.   key = ix->key;
  1113.   if (tref_isnumber(key)) {
  1114.     int32_t k = numberVint(&ix->keyv);
  1115.     if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k)
  1116.       k = LJ_MAX_ASIZE;
  1117.     if ((MSize)k < LJ_MAX_ASIZE) {  /* Potential array key? */
  1118.       TRef ikey = lj_opt_narrow_index(J, key);
  1119.       TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
  1120.       if ((MSize)k < t->asize) {  /* Currently an array key? */
  1121.         TRef arrayref;
  1122.         rec_idx_abc(J, asizeref, ikey, t->asize);
  1123.         arrayref = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_ARRAY);
  1124.         return emitir(IRT(IR_AREF, IRT_P32), arrayref, ikey);
  1125.       } else/* Currently not in array (may be an array extension)? */
  1126.         emitir(IRTGI(IR_ULE), asizeref, ikey);  /* Inv. bounds check. */
  1127.         if (k == 0 && tref_isk(key))
  1128.           key = lj_ir_knum_zero(J);  /* Canonicalize 0 or +-0.0 to +0.0. */
  1129.         /* And continue with the hash lookup. */
  1130.       }
  1131.     } else if (!tref_isk(key)) {
  1132.       /* We can rule out const numbers which failed the integerness test
  1133.       ** above. But all other numbers are potential array keys.
  1134.       */
  1135.       if (t->asize == 0) {  /* True sparse tables have an empty array part. */
  1136.         /* Guard that the array part stays empty. */
  1137.         TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
  1138.         emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
  1139.       } else {
  1140.         lj_trace_err(J, LJ_TRERR_NYITMIX);
  1141.       }
  1142.     }
  1143.   }

  1144.   /* Otherwise the key is located in the hash part. */
  1145.   if (t->hmask == 0) {  /* Shortcut for empty hash part. */
  1146.     /* Guard that the hash part stays empty. */
  1147.     TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
  1148.     emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
  1149.     return lj_ir_kkptr(J, niltvg(J2G(J)));
  1150.   }
  1151.   if (tref_isinteger(key))  /* Hash keys are based on numbers, not ints. */
  1152.     key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
  1153.   if (tref_isk(key)) {
  1154.     /* Optimize lookup of constant hash keys. */
  1155.     MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val);
  1156.     if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) &&
  1157.         hslot <= 65535*(MSize)sizeof(Node)) {
  1158.       TRef node, kslot, hm;
  1159.       *rbref = J->cur.nins;  /* Mark possible rollback point. */
  1160.       hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
  1161.       emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
  1162.       node = emitir(IRT(IR_FLOAD, IRT_P32), ix->tab, IRFL_TAB_NODE);
  1163.       kslot = lj_ir_kslot(J, key, hslot / sizeof(Node));
  1164.       return emitir(IRTG(IR_HREFK, IRT_P32), node, kslot);
  1165.     }
  1166.   }
  1167.   /* Fall back to a regular hash lookup. */
  1168.   return emitir(IRT(IR_HREF, IRT_P32), ix->tab, key);
  1169. }

  1170. /* Determine whether a key is NOT one of the fast metamethod names. */
  1171. static int nommstr(jit_State *J, TRef key)
  1172. {
  1173.   if (tref_isstr(key)) {
  1174.     if (tref_isk(key)) {
  1175.       GCstr *str = ir_kstr(IR(tref_ref(key)));
  1176.       uint32_t mm;
  1177.       for (mm = 0; mm <= MM_FAST; mm++)
  1178.         if (mmname_str(J2G(J), mm) == str)
  1179.           return 0/* MUST be one the fast metamethod names. */
  1180.     } else {
  1181.       return 0/* Variable string key MAY be a metamethod name. */
  1182.     }
  1183.   }
  1184.   return 1/* CANNOT be a metamethod name. */
  1185. }

  1186. /* Record indexed load/store. */
  1187. TRef lj_record_idx(jit_State *J, RecordIndex *ix)
  1188. {
  1189.   TRef xref;
  1190.   IROp xrefop, loadop;
  1191.   IRRef rbref;
  1192.   cTValue *oldv;

  1193.   while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
  1194.     /* Never call raw lj_record_idx() on non-table. */
  1195.     lua_assert(ix->idxchain != 0);
  1196.     if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
  1197.       lj_trace_err(J, LJ_TRERR_NOMM);
  1198.   handlemm:
  1199.     if (tref_isfunc(ix->mobj)) {  /* Handle metamethod call. */
  1200.       BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
  1201.       TRef *base = J->base + func;
  1202.       TValue *tv = J->L->base + func;
  1203.       lua_assert(!LJ_FR2);  /* TODO_FR2: handle different frame setup. */
  1204.       base[0] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
  1205.       setfuncV(J->L, tv+0, funcV(&ix->mobjv));
  1206.       copyTV(J->L, tv+1, &ix->tabv);
  1207.       copyTV(J->L, tv+2, &ix->keyv);
  1208.       if (ix->val) {
  1209.         base[3] = ix->val;
  1210.         copyTV(J->L, tv+3, &ix->valv);
  1211.         lj_record_call(J, func, 3);  /* mobj(tab, key, val) */
  1212.         return 0;
  1213.       } else {
  1214.         lj_record_call(J, func, 2);  /* res = mobj(tab, key) */
  1215.         return 0/* No result yet. */
  1216.       }
  1217.     }
  1218.     /* Otherwise retry lookup with metaobject. */
  1219.     ix->tab = ix->mobj;
  1220.     copyTV(J->L, &ix->tabv, &ix->mobjv);
  1221.     if (--ix->idxchain == 0)
  1222.       lj_trace_err(J, LJ_TRERR_IDXLOOP);
  1223.   }

  1224.   /* First catch nil and NaN keys for tables. */
  1225.   if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
  1226.     if (ix->val)  /* Better fail early. */
  1227.       lj_trace_err(J, LJ_TRERR_STORENN);
  1228.     if (tref_isk(ix->key)) {
  1229.       if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
  1230.         goto handlemm;
  1231.       return TREF_NIL;
  1232.     }
  1233.   }

  1234.   /* Record the key lookup. */
  1235.   xref = rec_idx_key(J, ix, &rbref);
  1236.   xrefop = IR(tref_ref(xref))->o;
  1237.   loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
  1238.   /* The lj_meta_tset() inconsistency is gone, but better play safe. */
  1239.   oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv;

  1240.   if (ix->val == 0) {  /* Indexed load */
  1241.     IRType t = itype2irt(oldv);
  1242.     TRef res;
  1243.     if (oldv == niltvg(J2G(J))) {
  1244.       emitir(IRTG(IR_EQ, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
  1245.       res = TREF_NIL;
  1246.     } else {
  1247.       res = emitir(IRTG(loadop, t), xref, 0);
  1248.     }
  1249.     if (tref_ref(res) < rbref)  /* HREFK + load forwarded? */
  1250.       lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
  1251.     if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
  1252.       goto handlemm;
  1253.     if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitives. */
  1254.     return res;
  1255.   } else/* Indexed store. */
  1256.     GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
  1257.     int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
  1258.     if (tref_ref(xref) < rbref)  /* HREFK forwarded? */
  1259.       lj_ir_rollback(J, rbref);  /* Rollback to eliminate hmask guard. */
  1260.     if (tvisnil(oldv)) {  /* Previous value was nil? */
  1261.       /* Need to duplicate the hasmm check for the early guards. */
  1262.       int hasmm = 0;
  1263.       if (ix->idxchain && mt) {
  1264.         cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex));
  1265.         hasmm = mo && !tvisnil(mo);
  1266.       }
  1267.       if (hasmm)
  1268.         emitir(IRTG(loadop, IRT_NIL), xref, 0);  /* Guard for nil value. */
  1269.       else if (xrefop == IR_HREF)
  1270.         emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_P32),
  1271.                xref, lj_ir_kkptr(J, niltvg(J2G(J))));
  1272.       if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
  1273.         lua_assert(hasmm);
  1274.         goto handlemm;
  1275.       }
  1276.       lua_assert(!hasmm);
  1277.       if (oldv == niltvg(J2G(J))) {  /* Need to insert a new key. */
  1278.         TRef key = ix->key;
  1279.         if (tref_isinteger(key))  /* NEWREF needs a TValue as a key. */
  1280.           key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
  1281.         xref = emitir(IRT(IR_NEWREF, IRT_P32), ix->tab, key);
  1282.         keybarrier = 0/* NEWREF already takes care of the key barrier. */
  1283.       }
  1284.     } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
  1285.       /* Cannot derive that the previous value was non-nil, must do checks. */
  1286.       if (xrefop == IR_HREF)  /* Guard against store to niltv. */
  1287.         emitir(IRTG(IR_NE, IRT_P32), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
  1288.       if (ix->idxchain) {  /* Metamethod lookup required? */
  1289.         /* A check for NULL metatable is cheaper (hoistable) than a load. */
  1290.         if (!mt) {
  1291.           TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
  1292.           emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
  1293.         } else {
  1294.           IRType t = itype2irt(oldv);
  1295.           emitir(IRTG(loadop, t), xref, 0);  /* Guard for non-nil value. */
  1296.         }
  1297.       }
  1298.     } else {
  1299.       keybarrier = 0/* Previous non-nil value kept the key alive. */
  1300.     }
  1301.     /* Convert int to number before storing. */
  1302.     if (!LJ_DUALNUM && tref_isinteger(ix->val))
  1303.       ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT);
  1304.     emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
  1305.     if (keybarrier || tref_isgcv(ix->val))
  1306.       emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
  1307.     /* Invalidate neg. metamethod cache for stores with certain string keys. */
  1308.     if (!nommstr(J, ix->key)) {
  1309.       TRef fref = emitir(IRT(IR_FREF, IRT_P32), ix->tab, IRFL_TAB_NOMM);
  1310.       emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
  1311.     }
  1312.     J->needsnap = 1;
  1313.     return 0;
  1314.   }
  1315. }

  1316. static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
  1317. {
  1318.   RecordIndex ix;
  1319.   cTValue *basev = J->L->base;
  1320.   copyTV(J->L, &ix.tabv, &basev[ra-1]);
  1321.   ix.tab = getslot(J, ra-1);
  1322.   ix.idxchain = 0;
  1323.   for (; ra < rn; i++, ra++) {
  1324.     setintV(&ix.keyv, i);
  1325.     ix.key = lj_ir_kint(J, i);
  1326.     copyTV(J->L, &ix.valv, &basev[ra]);
  1327.     ix.val = getslot(J, ra);
  1328.     lj_record_idx(J, &ix);
  1329.   }
  1330. }

  1331. /* -- Upvalue access ------------------------------------------------------ */

  1332. /* Check whether upvalue is immutable and ok to constify. */
  1333. static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
  1334. {
  1335.   if (uvp->immutable) {
  1336.     cTValue *o = uvval(uvp);
  1337.     /* Don't constify objects that may retain large amounts of memory. */
  1338. #if LJ_HASFFI
  1339.     if (tviscdata(o)) {
  1340.       GCcdata *cd = cdataV(o);
  1341.       if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
  1342.         CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
  1343.         if (!ctype_hassize(ct->info) || ct->size <= 16)
  1344.           return 1;
  1345.       }
  1346.       return 0;
  1347.     }
  1348. #else
  1349.     UNUSED(J);
  1350. #endif
  1351.     if (!(tvistab(o) || tvisudata(o) || tvisthread(o)))
  1352.       return 1;
  1353.   }
  1354.   return 0;
  1355. }

  1356. /* Record upvalue load/store. */
  1357. static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
  1358. {
  1359.   GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
  1360.   TRef fn = getcurrf(J);
  1361.   IRRef uref;
  1362.   int needbarrier = 0;
  1363.   if (rec_upvalue_constify(J, uvp)) {  /* Try to constify immutable upvalue. */
  1364.     TRef tr, kfunc;
  1365.     lua_assert(val == 0);
  1366.     if (!tref_isk(fn)) {  /* Late specialization of current function. */
  1367.       if (J->pt->flags >= PROTO_CLC_POLY)
  1368.         goto noconstify;
  1369.       kfunc = lj_ir_kfunc(J, J->fn);
  1370.       emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
  1371.       J->base[-1] = TREF_FRAME | kfunc;
  1372.       fn = kfunc;
  1373.     }
  1374.     tr = lj_record_constify(J, uvval(uvp));
  1375.     if (tr)
  1376.       return tr;
  1377.   }
  1378. noconstify:
  1379.   /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
  1380.   uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
  1381.   if (!uvp->closed) {
  1382.     /* In current stack? */
  1383.     if (uvval(uvp) >= tvref(J->L->stack) &&
  1384.         uvval(uvp) < tvref(J->L->maxstack)) {
  1385.       int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
  1386.       if (slot >= 0) {  /* Aliases an SSA slot? */
  1387.         slot -= (int32_t)J->baseslot;  /* Note: slot number may be negative! */
  1388.         /* NYI: add IR to guard that it's still aliasing the same slot. */
  1389.         if (val == 0) {
  1390.           return getslot(J, slot);
  1391.         } else {
  1392.           J->base[slot] = val;
  1393.           if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
  1394.           return 0;
  1395.         }
  1396.       }
  1397.     }
  1398.     uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_P32), fn, uv));
  1399.   } else {
  1400.     needbarrier = 1;
  1401.     uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_P32), fn, uv));
  1402.   }
  1403.   if (val == 0) {  /* Upvalue load */
  1404.     IRType t = itype2irt(uvval(uvp));
  1405.     TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
  1406.     if (irtype_ispri(t)) res = TREF_PRI(t);  /* Canonicalize primitive refs. */
  1407.     return res;
  1408.   } else/* Upvalue store. */
  1409.     /* Convert int to number before storing. */
  1410.     if (!LJ_DUALNUM && tref_isinteger(val))
  1411.       val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
  1412.     emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
  1413.     if (needbarrier && tref_isgcv(val))
  1414.       emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
  1415.     J->needsnap = 1;
  1416.     return 0;
  1417.   }
  1418. }

  1419. /* -- Record calls to Lua functions --------------------------------------- */

  1420. /* Check unroll limits for calls. */
  1421. static void check_call_unroll(jit_State *J, TraceNo lnk)
  1422. {
  1423.   cTValue *frame = J->L->base - 1;
  1424.   void *pc = mref(frame_func(frame)->l.pc, void);
  1425.   int32_t depth = J->framedepth;
  1426.   int32_t count = 0;
  1427.   if ((J->pt->flags & PROTO_VARARG)) depth--;  /* Vararg frame still missing. */
  1428.   for (; depth > 0; depth--) {  /* Count frames with same prototype. */
  1429.     if (frame_iscont(frame)) depth--;
  1430.     frame = frame_prev(frame);
  1431.     if (mref(frame_func(frame)->l.pc, void) == pc)
  1432.       count++;
  1433.   }
  1434.   if (J->pc == J->startpc) {
  1435.     if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
  1436.       J->pc++;
  1437.       if (J->framedepth + J->retdepth == 0)
  1438.         lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Tail-rec. */
  1439.       else
  1440.         lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno);  /* Up-recursion. */
  1441.     }
  1442.   } else {
  1443.     if (count > J->param[JIT_P_callunroll]) {
  1444.       if (lnk) {  /* Possible tail- or up-recursion. */
  1445.         lj_trace_flush(J, lnk);  /* Flush trace that only returns. */
  1446.         /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
  1447.         hotcount_set(J2GG(J), J->pc+1, LJ_PRNG_BITS(J, 4));
  1448.       }
  1449.       lj_trace_err(J, LJ_TRERR_CUNROLL);
  1450.     }
  1451.   }
  1452. }

  1453. /* Record Lua function setup. */
  1454. static void rec_func_setup(jit_State *J)
  1455. {
  1456.   GCproto *pt = J->pt;
  1457.   BCReg s, numparams = pt->numparams;
  1458.   if ((pt->flags & PROTO_NOJIT))
  1459.     lj_trace_err(J, LJ_TRERR_CJITOFF);
  1460.   if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
  1461.     lj_trace_err(J, LJ_TRERR_STACKOV);
  1462.   /* Fill up missing parameters with nil. */
  1463.   for (s = J->maxslot; s < numparams; s++)
  1464.     J->base[s] = TREF_NIL;
  1465.   /* The remaining slots should never be read before they are written. */
  1466.   J->maxslot = numparams;
  1467. }

  1468. /* Record Lua vararg function setup. */
  1469. static void rec_func_vararg(jit_State *J)
  1470. {
  1471.   GCproto *pt = J->pt;
  1472.   BCReg s, fixargs, vframe = J->maxslot+1;
  1473.   lua_assert((pt->flags & PROTO_VARARG));
  1474.   if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
  1475.     lj_trace_err(J, LJ_TRERR_STACKOV);
  1476.   J->base[vframe-1] = J->base[-1];  /* Copy function up. */
  1477.   /* Copy fixarg slots up and set their original slots to nil. */
  1478.   fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
  1479.   for (s = 0; s < fixargs; s++) {
  1480.     J->base[vframe+s] = J->base[s];
  1481.     J->base[s] = TREF_NIL;
  1482.   }
  1483.   J->maxslot = fixargs;
  1484.   J->framedepth++;
  1485.   J->base += vframe;
  1486.   J->baseslot += vframe;
  1487. }

  1488. /* Record entry to a Lua function. */
  1489. static void rec_func_lua(jit_State *J)
  1490. {
  1491.   rec_func_setup(J);
  1492.   check_call_unroll(J, 0);
  1493. }

  1494. /* Record entry to an already compiled function. */
  1495. static void rec_func_jit(jit_State *J, TraceNo lnk)
  1496. {
  1497.   GCtrace *T;
  1498.   rec_func_setup(J);
  1499.   T = traceref(J, lnk);
  1500.   if (T->linktype == LJ_TRLINK_RETURN) {  /* Trace returns to interpreter? */
  1501.     check_call_unroll(J, lnk);
  1502.     /* Temporarily unpatch JFUNC* to continue recording across function. */
  1503.     J->patchins = *J->pc;
  1504.     J->patchpc = (BCIns *)J->pc;
  1505.     *J->patchpc = T->startins;
  1506.     return;
  1507.   }
  1508.   J->instunroll = 0/* Cannot continue across a compiled function. */
  1509.   if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
  1510.     lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno);  /* Extra tail-rec. */
  1511.   else
  1512.     lj_record_stop(J, LJ_TRLINK_ROOT, lnk);  /* Link to the function. */
  1513. }

  1514. /* -- Vararg handling ----------------------------------------------------- */

  1515. /* Detect y = select(x, ...) idiom. */
  1516. static int select_detect(jit_State *J)
  1517. {
  1518.   BCIns ins = J->pc[1];
  1519.   if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) {
  1520.     cTValue *func = &J->L->base[bc_a(ins)];
  1521.     if (tvisfunc(func) && funcV(func)->c.ffid == FF_select)
  1522.       return 1;
  1523.   }
  1524.   return 0;
  1525. }

  1526. /* Record vararg instruction. */
  1527. static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
  1528. {
  1529.   int32_t numparams = J->pt->numparams;
  1530.   ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1;
  1531.   lua_assert(frame_isvarg(J->L->base-1));
  1532.   if (J->framedepth > 0) {  /* Simple case: varargs defined on-trace. */
  1533.     ptrdiff_t i;
  1534.     if (nvararg < 0) nvararg = 0;
  1535.     if (nresults == -1) {
  1536.       nresults = nvararg;
  1537.       J->maxslot = dst + (BCReg)nvararg;
  1538.     } else if (dst + nresults > J->maxslot) {
  1539.       J->maxslot = dst + (BCReg)nresults;
  1540.     }
  1541.     for (i = 0; i < nresults; i++)
  1542.       J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1) : TREF_NIL;
  1543.   } else/* Unknown number of varargs passed to trace. */
  1544.     TRef fr = emitir(IRTI(IR_SLOAD), 0, IRSLOAD_READONLY|IRSLOAD_FRAME);
  1545.     int32_t frofs = 8*(1+numparams)+FRAME_VARG;
  1546.     if (nresults >= 0) {  /* Known fixed number of results. */
  1547.       ptrdiff_t i;
  1548.       if (nvararg > 0) {
  1549.         ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg;
  1550.         TRef vbase;
  1551.         if (nvararg >= nresults)
  1552.           emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
  1553.         else
  1554.           emitir(IRTGI(IR_EQ), fr,
  1555.                  lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
  1556.         vbase = emitir(IRTI(IR_SUB), REF_BASE, fr);
  1557.         vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8));
  1558.         for (i = 0; i < nload; i++) {
  1559.           IRType t = itype2irt(&J->L->base[i-1-nvararg]);
  1560.           TRef aref = emitir(IRT(IR_AREF, IRT_P32),
  1561.                              vbase, lj_ir_kint(J, (int32_t)i));
  1562.           TRef tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
  1563.           if (irtype_ispri(t)) tr = TREF_PRI(t);  /* Canonicalize primitives. */
  1564.           J->base[dst+i] = tr;
  1565.         }
  1566.       } else {
  1567.         emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
  1568.         nvararg = 0;
  1569.       }
  1570.       for (i = nvararg; i < nresults; i++)
  1571.         J->base[dst+i] = TREF_NIL;
  1572.       if (dst + (BCReg)nresults > J->maxslot)
  1573.         J->maxslot = dst + (BCReg)nresults;
  1574.     } else if (select_detect(J)) {  /* y = select(x, ...) */
  1575.       TRef tridx = J->base[dst-1];
  1576.       TRef tr = TREF_NIL;
  1577.       ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]);
  1578.       if (idx < 0) goto nyivarg;
  1579.       if (idx != 0 && !tref_isinteger(tridx))
  1580.         tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX);
  1581.       if (idx != 0 && tref_isk(tridx)) {
  1582.         emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT),
  1583.                fr, lj_ir_kint(J, frofs+8*(int32_t)idx));
  1584.         frofs -= 8/* Bias for 1-based index. */
  1585.       } else if (idx <= nvararg) {  /* Compute size. */
  1586.         TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs));
  1587.         if (numparams)
  1588.           emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0));
  1589.         tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3));
  1590.         if (idx != 0) {
  1591.           tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1));
  1592.           rec_idx_abc(J, tr, tridx, (uint32_t)nvararg);
  1593.         }
  1594.       } else {
  1595.         TRef tmp = lj_ir_kint(J, frofs);
  1596.         if (idx != 0) {
  1597.           TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3));
  1598.           tmp = emitir(IRTI(IR_ADD), tmp2, tmp);
  1599.         } else {
  1600.           tr = lj_ir_kint(J, 0);
  1601.         }
  1602.         emitir(IRTGI(IR_LT), fr, tmp);
  1603.       }
  1604.       if (idx != 0 && idx <= nvararg) {
  1605.         IRType t;
  1606.         TRef aref, vbase = emitir(IRTI(IR_SUB), REF_BASE, fr);
  1607.         vbase = emitir(IRT(IR_ADD, IRT_P32), vbase, lj_ir_kint(J, frofs-8));
  1608.         t = itype2irt(&J->L->base[idx-2-nvararg]);
  1609.         aref = emitir(IRT(IR_AREF, IRT_P32), vbase, tridx);
  1610.         tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
  1611.         if (irtype_ispri(t)) tr = TREF_PRI(t);  /* Canonicalize primitives. */
  1612.       }
  1613.       J->base[dst-2] = tr;
  1614.       J->maxslot = dst-1;
  1615.       J->bcskip = 2/* Skip CALLM + select. */
  1616.     } else {
  1617.     nyivarg:
  1618.       setintV(&J->errinfo, BC_VARG);
  1619.       lj_trace_err_info(J, LJ_TRERR_NYIBC);
  1620.     }
  1621.   }
  1622. }

  1623. /* -- Record allocations -------------------------------------------------- */

  1624. static TRef rec_tnew(jit_State *J, uint32_t ah)
  1625. {
  1626.   uint32_t asize = ah & 0x7ff;
  1627.   uint32_t hbits = ah >> 11;
  1628.   if (asize == 0x7ff) asize = 0x801;
  1629.   return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
  1630. }

  1631. /* -- Concatenation ------------------------------------------------------- */

  1632. static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
  1633. {
  1634.   TRef *top = &J->base[topslot];
  1635.   TValue savetv[5];
  1636.   BCReg s;
  1637.   RecordIndex ix;
  1638.   lua_assert(baseslot < topslot);
  1639.   for (s = baseslot; s <= topslot; s++)
  1640.     (void)getslot(J, s);  /* Ensure all arguments have a reference. */
  1641.   if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
  1642.     TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
  1643.     /* First convert numbers to strings. */
  1644.     for (trp = top; trp >= base; trp--) {
  1645.       if (tref_isnumber(*trp))
  1646.         *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
  1647.                       tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
  1648.       else if (!tref_isstr(*trp))
  1649.         break;
  1650.     }
  1651.     xbase = ++trp;
  1652.     tr = hdr = emitir(IRT(IR_BUFHDR, IRT_P32),
  1653.                       lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
  1654.     do {
  1655.       tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, *trp++);
  1656.     } while (trp <= top);
  1657.     tr = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr);
  1658.     J->maxslot = (BCReg)(xbase - J->base);
  1659.     if (xbase == base) return tr;  /* Return simple concatenation result. */
  1660.     /* Pass partial result. */
  1661.     topslot = J->maxslot--;
  1662.     *xbase = tr;
  1663.     top = xbase;
  1664.     setstrV(J->L, &ix.keyv, &J2G(J)->strempty);  /* Simulate string result. */
  1665.   } else {
  1666.     J->maxslot = topslot-1;
  1667.     copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
  1668.   }
  1669.   copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
  1670.   ix.tab = top[-1];
  1671.   ix.key = top[0];
  1672.   memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv));  /* Save slots. */
  1673.   rec_mm_arith(J, &ix, MM_concat);  /* Call __concat metamethod. */
  1674.   memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv));  /* Restore slots. */
  1675.   return 0/* No result yet. */
  1676. }

  1677. /* -- Record bytecode ops ------------------------------------------------- */

  1678. /* Prepare for comparison. */
  1679. static void rec_comp_prep(jit_State *J)
  1680. {
  1681.   /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
  1682.   if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins)
  1683.     emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
  1684.   lj_snap_add(J);
  1685. }

  1686. /* Fixup comparison. */
  1687. static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
  1688. {
  1689.   BCIns jmpins = pc[1];
  1690.   const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
  1691.   SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
  1692.   /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
  1693.   J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
  1694.   J->needsnap = 1;
  1695.   if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
  1696.   lj_snap_shrink(J);  /* Shrink last snapshot if possible. */
  1697. }

  1698. /* Record the next bytecode instruction (_before_ it's executed). */
  1699. void lj_record_ins(jit_State *J)
  1700. {
  1701.   cTValue *lbase;
  1702.   RecordIndex ix;
  1703.   const BCIns *pc;
  1704.   BCIns ins;
  1705.   BCOp op;
  1706.   TRef ra, rb, rc;

  1707.   /* Perform post-processing action before recording the next instruction. */
  1708.   if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
  1709.     switch (J->postproc) {
  1710.     case LJ_POST_FIXCOMP:  /* Fixup comparison. */
  1711.       pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
  1712.       rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
  1713.       /* fallthrough */
  1714.     case LJ_POST_FIXGUARD:  /* Fixup and emit pending guard. */
  1715.     case LJ_POST_FIXGUARDSNAP:  /* Fixup and emit pending guard and snapshot. */
  1716.       if (!tvistruecond(&J2G(J)->tmptv2)) {
  1717.         J->fold.ins.o ^= 1/* Flip guard to opposite. */
  1718.         if (J->postproc == LJ_POST_FIXGUARDSNAP) {
  1719.           SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
  1720.           J->cur.snapmap[snap->mapofs+snap->nent-1]--;  /* False -> true. */
  1721.         }
  1722.       }
  1723.       lj_opt_fold(J);  /* Emit pending guard. */
  1724.       /* fallthrough */
  1725.     case LJ_POST_FIXBOOL:
  1726.       if (!tvistruecond(&J2G(J)->tmptv2)) {
  1727.         BCReg s;
  1728.         TValue *tv = J->L->base;
  1729.         for (s = 0; s < J->maxslot; s++)  /* Fixup stack slot (if any). */
  1730.           if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) {
  1731.             J->base[s] = TREF_FALSE;
  1732.             break;
  1733.           }
  1734.       }
  1735.       break;
  1736.     case LJ_POST_FIXCONST:
  1737.       {
  1738.         BCReg s;
  1739.         TValue *tv = J->L->base;
  1740.         for (s = 0; s < J->maxslot; s++)  /* Constify stack slots (if any). */
  1741.           if (J->base[s] == TREF_NIL && !tvisnil(&tv[s]))
  1742.             J->base[s] = lj_record_constify(J, &tv[s]);
  1743.       }
  1744.       break;
  1745.     case LJ_POST_FFRETRY:  /* Suppress recording of retried fast function. */
  1746.       if (bc_op(*J->pc) >= BC__MAX)
  1747.         return;
  1748.       break;
  1749.     default: lua_assert(0); break;
  1750.     }
  1751.     J->postproc = LJ_POST_NONE;
  1752.   }

  1753.   /* Need snapshot before recording next bytecode (e.g. after a store). */
  1754.   if (J->needsnap) {
  1755.     J->needsnap = 0;
  1756.     lj_snap_purge(J);
  1757.     lj_snap_add(J);
  1758.     J->mergesnap = 1;
  1759.   }

  1760.   /* Skip some bytecodes. */
  1761.   if (LJ_UNLIKELY(J->bcskip > 0)) {
  1762.     J->bcskip--;
  1763.     return;
  1764.   }

  1765.   /* Record only closed loops for root traces. */
  1766.   pc = J->pc;
  1767.   if (J->framedepth == 0 &&
  1768.      (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
  1769.     lj_trace_err(J, LJ_TRERR_LLEAVE);

  1770. #ifdef LUA_USE_ASSERT
  1771.   rec_check_slots(J);
  1772.   rec_check_ir(J);
  1773. #endif

  1774. #if LJ_HASPROFILE
  1775.   rec_profile_ins(J, pc);
  1776. #endif

  1777.   /* Keep a copy of the runtime values of var/num/str operands. */
  1778. #define rav        (&ix.valv)
  1779. #define rbv        (&ix.tabv)
  1780. #define rcv        (&ix.keyv)

  1781.   lbase = J->L->base;
  1782.   ins = *pc;
  1783.   op = bc_op(ins);
  1784.   ra = bc_a(ins);
  1785.   ix.val = 0;
  1786.   switch (bcmode_a(op)) {
  1787.   case BCMvar:
  1788.     copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
  1789.   default: break/* Handled later. */
  1790.   }
  1791.   rb = bc_b(ins);
  1792.   rc = bc_c(ins);
  1793.   switch (bcmode_b(op)) {
  1794.   case BCMnone: rb = 0; rc = bc_d(ins); break/* Upgrade rc to 'rd'. */
  1795.   case BCMvar:
  1796.     copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
  1797.   default: break/* Handled later. */
  1798.   }
  1799.   switch (bcmode_c(op)) {
  1800.   case BCMvar:
  1801.     copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
  1802.   case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
  1803.   case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
  1804.     copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
  1805.     lj_ir_knumint(J, numV(tv)); } break;
  1806.   case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
  1807.     setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
  1808.   default: break/* Handled later. */
  1809.   }

  1810.   switch (op) {

  1811.   /* -- Comparison ops ---------------------------------------------------- */

  1812.   case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
  1813. #if LJ_HASFFI
  1814.     if (tref_iscdata(ra) || tref_iscdata(rc)) {
  1815.       rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
  1816.       break;
  1817.     }
  1818. #endif
  1819.     /* Emit nothing for two numeric or string consts. */
  1820.     if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
  1821.       IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
  1822.       IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc);
  1823.       int irop;
  1824.       if (ta != tc) {
  1825.         /* Widen mixed number/int comparisons to number/number comparison. */
  1826.         if (ta == IRT_INT && tc == IRT_NUM) {
  1827.           ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT);
  1828.           ta = IRT_NUM;
  1829.         } else if (ta == IRT_NUM && tc == IRT_INT) {
  1830.           rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
  1831.         } else if (LJ_52) {
  1832.           ta = IRT_NIL;  /* Force metamethod for different types. */
  1833.         } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
  1834.                      (tc == IRT_FALSE || tc == IRT_TRUE))) {
  1835.           break/* Interpreter will throw for two different types. */
  1836.         }
  1837.       }
  1838.       rec_comp_prep(J);
  1839.       irop = (int)op - (int)BC_ISLT + (int)IR_LT;
  1840.       if (ta == IRT_NUM) {
  1841.         if ((irop & 1)) irop ^= 4/* ISGE/ISGT are unordered. */
  1842.         if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
  1843.           irop ^= 5;
  1844.       } else if (ta == IRT_INT) {
  1845.         if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
  1846.           irop ^= 1;
  1847.       } else if (ta == IRT_STR) {
  1848.         if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
  1849.         ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc);
  1850.         rc = lj_ir_kint(J, 0);
  1851.         ta = IRT_INT;
  1852.       } else {
  1853.         rec_mm_comp(J, &ix, (int)op);
  1854.         break;
  1855.       }
  1856.       emitir(IRTG(irop, ta), ra, rc);
  1857.       rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1);
  1858.     }
  1859.     break;

  1860.   case BC_ISEQV: case BC_ISNEV:
  1861.   case BC_ISEQS: case BC_ISNES:
  1862.   case BC_ISEQN: case BC_ISNEN:
  1863.   case BC_ISEQP: case BC_ISNEP:
  1864. #if LJ_HASFFI
  1865.     if (tref_iscdata(ra) || tref_iscdata(rc)) {
  1866.       rec_mm_comp_cdata(J, &ix, op, MM_eq);
  1867.       break;
  1868.     }
  1869. #endif
  1870.     /* Emit nothing for two non-table, non-udata consts. */
  1871.     if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
  1872.       int diff;
  1873.       rec_comp_prep(J);
  1874.       diff = lj_record_objcmp(J, ra, rc, rav, rcv);
  1875.       if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra)))
  1876.         rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff);
  1877.       else if (diff == 1/* Only check __eq if different, but same type. */
  1878.         rec_mm_equal(J, &ix, (int)op);
  1879.     }
  1880.     break;

  1881.   /* -- Unary test and copy ops ------------------------------------------- */

  1882.   case BC_ISTC: case BC_ISFC:
  1883.     if ((op & 1) == tref_istruecond(rc))
  1884.       rc = 0/* Don't store if condition is not true. */
  1885.     /* fallthrough */
  1886.   case BC_IST: case BC_ISF:  /* Type specialization suffices. */
  1887.     if (bc_a(pc[1]) < J->maxslot)
  1888.       J->maxslot = bc_a(pc[1]);  /* Shrink used slots. */
  1889.     break;

  1890.   case BC_ISTYPE: case BC_ISNUM:
  1891.     /* These coercions need to correspond with lj_meta_istype(). */
  1892.     if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
  1893.       ra = lj_opt_narrow_toint(J, ra);
  1894.     else if (rc == ~LJ_TNUMX+2)
  1895.       ra = lj_ir_tonum(J, ra);
  1896.     else if (rc == ~LJ_TSTR+1)
  1897.       ra = lj_ir_tostr(J, ra);
  1898.     /* else: type specialization suffices. */
  1899.     J->base[bc_a(ins)] = ra;
  1900.     break;

  1901.   /* -- Unary ops --------------------------------------------------------- */

  1902.   case BC_NOT:
  1903.     /* Type specialization already forces const result. */
  1904.     rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
  1905.     break;

  1906.   case BC_LEN:
  1907.     if (tref_isstr(rc))
  1908.       rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
  1909.     else if (!LJ_52 && tref_istab(rc))
  1910.       rc = lj_ir_call(J, IRCALL_lj_tab_len, rc);
  1911.     else
  1912.       rc = rec_mm_len(J, rc, rcv);
  1913.     break;

  1914.   /* -- Arithmetic ops ---------------------------------------------------- */

  1915.   case BC_UNM:
  1916.     if (tref_isnumber_str(rc)) {
  1917.       rc = lj_opt_narrow_unm(J, rc, rcv);
  1918.     } else {
  1919.       ix.tab = rc;
  1920.       copyTV(J->L, &ix.tabv, rcv);
  1921.       rc = rec_mm_arith(J, &ix, MM_unm);
  1922.     }
  1923.     break;

  1924.   case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
  1925.     /* Swap rb/rc and rbv/rcv. rav is temp. */
  1926.     ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
  1927.     copyTV(J->L, rav, rbv);
  1928.     copyTV(J->L, rbv, rcv);
  1929.     copyTV(J->L, rcv, rav);
  1930.     if (op == BC_MODNV)
  1931.       goto recmod;
  1932.     /* fallthrough */
  1933.   case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
  1934.   case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
  1935.     MMS mm = bcmode_mm(op);
  1936.     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
  1937.       rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv,
  1938.                                (int)mm - (int)MM_add + (int)IR_ADD);
  1939.     else
  1940.       rc = rec_mm_arith(J, &ix, mm);
  1941.     break;
  1942.     }

  1943.   case BC_MODVN: case BC_MODVV:
  1944.   recmod:
  1945.     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
  1946.       rc = lj_opt_narrow_mod(J, rb, rc, rcv);
  1947.     else
  1948.       rc = rec_mm_arith(J, &ix, MM_mod);
  1949.     break;

  1950.   case BC_POW:
  1951.     if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
  1952.       rc = lj_opt_narrow_pow(J, lj_ir_tonum(J, rb), rc, rcv);
  1953.     else
  1954.       rc = rec_mm_arith(J, &ix, MM_pow);
  1955.     break;

  1956.   /* -- Miscellaneous ops ------------------------------------------------- */

  1957.   case BC_CAT:
  1958.     rc = rec_cat(J, rb, rc);
  1959.     break;

  1960.   /* -- Constant and move ops --------------------------------------------- */

  1961.   case BC_MOV:
  1962.     /* Clear gap of method call to avoid resurrecting previous refs. */
  1963.     if (ra > J->maxslot) J->base[ra-1] = 0;
  1964.     break;
  1965.   case BC_KSTR: case BC_KNUM: case BC_KPRI:
  1966.     break;
  1967.   case BC_KSHORT:
  1968.     rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
  1969.     break;
  1970.   case BC_KNIL:
  1971.     while (ra <= rc)
  1972.       J->base[ra++] = TREF_NIL;
  1973.     if (rc >= J->maxslot) J->maxslot = rc+1;
  1974.     break;
  1975. #if LJ_HASFFI
  1976.   case BC_KCDATA:
  1977.     rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA);
  1978.     break;
  1979. #endif

  1980.   /* -- Upvalue and function ops ------------------------------------------ */

  1981.   case BC_UGET:
  1982.     rc = rec_upvalue(J, rc, 0);
  1983.     break;
  1984.   case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
  1985.     rec_upvalue(J, ra, rc);
  1986.     break;

  1987.   /* -- Table ops --------------------------------------------------------- */

  1988.   case BC_GGET: case BC_GSET:
  1989.     settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
  1990.     ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
  1991.     ix.idxchain = LJ_MAX_IDXCHAIN;
  1992.     rc = lj_record_idx(J, &ix);
  1993.     break;

  1994.   case BC_TGETB: case BC_TSETB:
  1995.     setintV(&ix.keyv, (int32_t)rc);
  1996.     ix.key = lj_ir_kint(J, (int32_t)rc);
  1997.     /* fallthrough */
  1998.   case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
  1999.     ix.idxchain = LJ_MAX_IDXCHAIN;
  2000.     rc = lj_record_idx(J, &ix);
  2001.     break;
  2002.   case BC_TGETR: case BC_TSETR:
  2003.     ix.idxchain = 0;
  2004.     rc = lj_record_idx(J, &ix);
  2005.     break;

  2006.   case BC_TSETM:
  2007.     rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
  2008.     break;

  2009.   case BC_TNEW:
  2010.     rc = rec_tnew(J, rc);
  2011.     break;
  2012.   case BC_TDUP:
  2013.     rc = emitir(IRTG(IR_TDUP, IRT_TAB),
  2014.                 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
  2015.     break;

  2016.   /* -- Calls and vararg handling ----------------------------------------- */

  2017.   case BC_ITERC:
  2018.     J->base[ra] = getslot(J, ra-3-LJ_FR2);
  2019.     J->base[ra+1] = getslot(J, ra-2-LJ_FR2);
  2020.     J->base[ra+2] = getslot(J, ra-1-LJ_FR2);
  2021.     { /* Do the actual copy now because lj_record_call needs the values. */
  2022.       TValue *b = &J->L->base[ra];
  2023.       copyTV(J->L, b, b-3-LJ_FR2);
  2024.       copyTV(J->L, b+1, b-2-LJ_FR2);
  2025.       copyTV(J->L, b+2, b-1-LJ_FR2);
  2026.     }
  2027.     lj_record_call(J, ra, (ptrdiff_t)rc-1);
  2028.     break;

  2029.   /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
  2030.   case BC_CALLM:
  2031.     rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
  2032.     /* fallthrough */
  2033.   case BC_CALL:
  2034.     lj_record_call(J, ra, (ptrdiff_t)rc-1);
  2035.     break;

  2036.   case BC_CALLMT:
  2037.     rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
  2038.     /* fallthrough */
  2039.   case BC_CALLT:
  2040.     lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
  2041.     break;

  2042.   case BC_VARG:
  2043.     rec_varg(J, ra, (ptrdiff_t)rb-1);
  2044.     break;

  2045.   /* -- Returns ----------------------------------------------------------- */

  2046.   case BC_RETM:
  2047.     /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
  2048.     rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
  2049.     /* fallthrough */
  2050.   case BC_RET: case BC_RET0: case BC_RET1:
  2051. #if LJ_HASPROFILE
  2052.     rec_profile_ret(J);
  2053. #endif
  2054.     lj_record_ret(J, ra, (ptrdiff_t)rc-1);
  2055.     break;

  2056.   /* -- Loops and branches ------------------------------------------------ */

  2057.   case BC_FORI:
  2058.     if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
  2059.       J->loopref = J->cur.nins;
  2060.     break;
  2061.   case BC_JFORI:
  2062.     lua_assert(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL);
  2063.     if (rec_for(J, pc, 0) != LOOPEV_LEAVE)  /* Link to existing loop. */
  2064.       lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
  2065.     /* Continue tracing if the loop is not entered. */
  2066.     break;

  2067.   case BC_FORL:
  2068.     rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
  2069.     break;
  2070.   case BC_ITERL:
  2071.     rec_loop_interp(J, pc, rec_iterl(J, *pc));
  2072.     break;
  2073.   case BC_LOOP:
  2074.     rec_loop_interp(J, pc, rec_loop(J, ra));
  2075.     break;

  2076.   case BC_JFORL:
  2077.     rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
  2078.     break;
  2079.   case BC_JITERL:
  2080.     rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
  2081.     break;
  2082.   case BC_JLOOP:
  2083.     rec_loop_jit(J, rc, rec_loop(J, ra));
  2084.     break;

  2085.   case BC_IFORL:
  2086.   case BC_IITERL:
  2087.   case BC_ILOOP:
  2088.   case BC_IFUNCF:
  2089.   case BC_IFUNCV:
  2090.     lj_trace_err(J, LJ_TRERR_BLACKL);
  2091.     break;

  2092.   case BC_JMP:
  2093.     if (ra < J->maxslot)
  2094.       J->maxslot = ra;  /* Shrink used slots. */
  2095.     break;

  2096.   /* -- Function headers -------------------------------------------------- */

  2097.   case BC_FUNCF:
  2098.     rec_func_lua(J);
  2099.     break;
  2100.   case BC_JFUNCF:
  2101.     rec_func_jit(J, rc);
  2102.     break;

  2103.   case BC_FUNCV:
  2104.     rec_func_vararg(J);
  2105.     rec_func_lua(J);
  2106.     break;
  2107.   case BC_JFUNCV:
  2108.     lua_assert(0);  /* Cannot happen. No hotcall counting for varag funcs. */
  2109.     break;

  2110.   case BC_FUNCC:
  2111.   case BC_FUNCCW:
  2112.     lj_ffrecord_func(J);
  2113.     break;

  2114.   default:
  2115.     if (op >= BC__MAX) {
  2116.       lj_ffrecord_func(J);
  2117.       break;
  2118.     }
  2119.     /* fallthrough */
  2120.   case BC_ITERN:
  2121.   case BC_ISNEXT:
  2122.   case BC_UCLO:
  2123.   case BC_FNEW:
  2124.     setintV(&J->errinfo, (int32_t)op);
  2125.     lj_trace_err_info(J, LJ_TRERR_NYIBC);
  2126.     break;
  2127.   }

  2128.   /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
  2129.   if (bcmode_a(op) == BCMdst && rc) {
  2130.     J->base[ra] = rc;
  2131.     if (ra >= J->maxslot) J->maxslot = ra+1;
  2132.   }

  2133. #undef rav
  2134. #undef rbv
  2135. #undef rcv

  2136.   /* Limit the number of recorded IR instructions. */
  2137.   if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord])
  2138.     lj_trace_err(J, LJ_TRERR_TRACEOV);
  2139. }

  2140. /* -- Recording setup ----------------------------------------------------- */

  2141. /* Setup recording for a root trace started by a hot loop. */
  2142. static const BCIns *rec_setup_root(jit_State *J)
  2143. {
  2144.   /* Determine the next PC and the bytecode range for the loop. */
  2145.   const BCIns *pcj, *pc = J->pc;
  2146.   BCIns ins = *pc;
  2147.   BCReg ra = bc_a(ins);
  2148.   switch (bc_op(ins)) {
  2149.   case BC_FORL:
  2150.     J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
  2151.     pc += 1+bc_j(ins);
  2152.     J->bc_min = pc;
  2153.     break;
  2154.   case BC_ITERL:
  2155.     lua_assert(bc_op(pc[-1]) == BC_ITERC);
  2156.     J->maxslot = ra + bc_b(pc[-1]) - 1;
  2157.     J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
  2158.     pc += 1+bc_j(ins);
  2159.     lua_assert(bc_op(pc[-1]) == BC_JMP);
  2160.     J->bc_min = pc;
  2161.     break;
  2162.   case BC_LOOP:
  2163.     /* Only check BC range for real loops, but not for "repeat until true". */
  2164.     pcj = pc + bc_j(ins);
  2165.     ins = *pcj;
  2166.     if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
  2167.       J->bc_min = pcj+1 + bc_j(ins);
  2168.       J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
  2169.     }
  2170.     J->maxslot = ra;
  2171.     pc++;
  2172.     break;
  2173.   case BC_RET:
  2174.   case BC_RET0:
  2175.   case BC_RET1:
  2176.     /* No bytecode range check for down-recursive root traces. */
  2177.     J->maxslot = ra + bc_d(ins) - 1;
  2178.     break;
  2179.   case BC_FUNCF:
  2180.     /* No bytecode range check for root traces started by a hot call. */
  2181.     J->maxslot = J->pt->numparams;
  2182.     pc++;
  2183.     break;
  2184.   case BC_CALLM:
  2185.   case BC_CALL:
  2186.   case BC_ITERC:
  2187.     /* No bytecode range check for stitched traces. */
  2188.     pc++;
  2189.     break;
  2190.   default:
  2191.     lua_assert(0);
  2192.     break;
  2193.   }
  2194.   return pc;
  2195. }

  2196. /* Setup for recording a new trace. */
  2197. void lj_record_setup(jit_State *J)
  2198. {
  2199.   uint32_t i;

  2200.   /* Initialize state related to current trace. */
  2201.   memset(J->slot, 0, sizeof(J->slot));
  2202.   memset(J->chain, 0, sizeof(J->chain));
  2203.   memset(J->bpropcache, 0, sizeof(J->bpropcache));
  2204.   J->scev.idx = REF_NIL;
  2205.   setmref(J->scev.pc, NULL);

  2206.   J->baseslot = 1/* Invoking function is at base[-1]. */
  2207.   J->base = J->slot + J->baseslot;
  2208.   J->maxslot = 0;
  2209.   J->framedepth = 0;
  2210.   J->retdepth = 0;

  2211.   J->instunroll = J->param[JIT_P_instunroll];
  2212.   J->loopunroll = J->param[JIT_P_loopunroll];
  2213.   J->tailcalled = 0;
  2214.   J->loopref = 0;

  2215.   J->bc_min = NULL/* Means no limit. */
  2216.   J->bc_extent = ~(MSize)0;

  2217.   /* Emit instructions for fixed references. Also triggers initial IR alloc. */
  2218.   emitir_raw(IRT(IR_BASE, IRT_P32), J->parent, J->exitno);
  2219.   for (i = 0; i <= 2; i++) {
  2220.     IRIns *ir = IR(REF_NIL-i);
  2221.     ir->i = 0;
  2222.     ir->t.irt = (uint8_t)(IRT_NIL+i);
  2223.     ir->o = IR_KPRI;
  2224.     ir->prev = 0;
  2225.   }
  2226.   J->cur.nk = REF_TRUE;

  2227.   J->startpc = J->pc;
  2228.   setmref(J->cur.startpc, J->pc);
  2229.   if (J->parent) {  /* Side trace. */
  2230.     GCtrace *T = traceref(J, J->parent);
  2231.     TraceNo root = T->root ? T->root : J->parent;
  2232.     J->cur.root = (uint16_t)root;
  2233.     J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
  2234.     /* Check whether we could at least potentially form an extra loop. */
  2235.     if (J->exitno == 0 && T->snap[0].nent == 0) {
  2236.       /* We can narrow a FORL for some side traces, too. */
  2237.       if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI &&
  2238.           bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
  2239.         lj_snap_add(J);
  2240.         rec_for_loop(J, J->pc-1, &J->scev, 1);
  2241.         goto sidecheck;
  2242.       }
  2243.     } else {
  2244.       J->startpc = NULL/* Prevent forming an extra loop. */
  2245.     }
  2246.     lj_snap_replay(J, T);
  2247.   sidecheck:
  2248.     if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
  2249.         T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
  2250.                                     J->param[JIT_P_tryside]) {
  2251.       lj_record_stop(J, LJ_TRLINK_INTERP, 0);
  2252.     }
  2253.   } else/* Root trace. */
  2254.     J->cur.root = 0;
  2255.     J->cur.startins = *J->pc;
  2256.     J->pc = rec_setup_root(J);
  2257.     /* Note: the loop instruction itself is recorded at the end and not
  2258.     ** at the start! So snapshot #0 needs to point to the *next* instruction.
  2259.     */
  2260.     lj_snap_add(J);
  2261.     if (bc_op(J->cur.startins) == BC_FORL)
  2262.       rec_for_loop(J, J->pc-1, &J->scev, 1);
  2263.     else if (bc_op(J->cur.startins) == BC_ITERC)
  2264.       J->startpc = NULL;
  2265.     if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
  2266.       lj_trace_err(J, LJ_TRERR_STACKOV);
  2267.   }
  2268. #if LJ_HASPROFILE
  2269.   J->prev_pt = NULL;
  2270.   J->prev_line = -1;
  2271. #endif
  2272. #ifdef LUAJIT_ENABLE_CHECKHOOK
  2273.   /* Regularly check for instruction/line hooks from compiled code and
  2274.   ** exit to the interpreter if the hooks are set.
  2275.   **
  2276.   ** This is a compile-time option and disabled by default, since the
  2277.   ** hook checks may be quite expensive in tight loops.
  2278.   **
  2279.   ** Note this is only useful if hooks are *not* set most of the time.
  2280.   ** Use this only if you want to *asynchronously* interrupt the execution.
  2281.   **
  2282.   ** You can set the instruction hook via lua_sethook() with a count of 1
  2283.   ** from a signal handler or another native thread. Please have a look
  2284.   ** at the first few functions in luajit.c for an example (Ctrl-C handler).
  2285.   */
  2286.   {
  2287.     TRef tr = emitir(IRT(IR_XLOAD, IRT_U8),
  2288.                      lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE);
  2289.     tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT)));
  2290.     emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
  2291.   }
  2292. #endif
  2293. }

  2294. #undef IR
  2295. #undef emitir_raw
  2296. #undef emitir

  2297. #endif