One Level Up Top Level

src/lj_snap.c - luajit-2.0-src

Functions defined

Macros defined

Source code

  1. /*
  2. ** Snapshot handling.
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. */

  6. ‌#define lj_snap_c
  7. ‌#define LUA_CORE

  9. #include "lj_obj.h"

  11. #if LJ_HASJIT

  13. #include "lj_gc.h"
  14. #include "lj_tab.h"
  15. #include "lj_state.h"
  16. #include "lj_frame.h"
  17. #include "lj_bc.h"
  18. #include "lj_ir.h"
  19. #include "lj_jit.h"
  20. #include "lj_iropt.h"
  21. #include "lj_trace.h"
  22. #include "lj_snap.h"
  23. #include "lj_target.h"
  24. #if LJ_HASFFI
  25. #include "lj_ctype.h"
  26. #include "lj_cdata.h"
  27. #endif

  29. /* Some local macros to save typing. Undef'd at the end. */
  30. ‌#define IR(ref)                (&J->cur.ir[(ref)])

  32. /* Pass IR on to next optimization in chain (FOLD). */
  33. ‌#define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))

  35. /* Emit raw IR without passing through optimizations. */
  36. ‌#define emitir_raw(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))

  38. /* -- Snapshot buffer allocation ------------------------------------------ */

  40. /* Grow snapshot buffer. */
  41. ‌void lj_snap_grow_buf_(jit_State *J, MSize need)
  42. {
  43.   MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
  44.   if (need > maxsnap)
  45.     lj_trace_err(J, LJ_TRERR_SNAPOV);
  46.   lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
  47.   J->cur.snap = J->snapbuf;
  48. }

  50. /* Grow snapshot map buffer. */
  51. ‌void lj_snap_grow_map_(jit_State *J, MSize need)
  52. {
  53.   if (need < 2*J->sizesnapmap)
  54.     need = 2*J->sizesnapmap;
  55.   else if (need < 64)
  56.     need = 64;
  57.   J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
  58.                     J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry));
  59.   J->cur.snapmap = J->snapmapbuf;
  60.   J->sizesnapmap = need;
  61. }

  63. /* -- Snapshot generation ------------------------------------------------- */

  65. /* Add all modified slots to the snapshot. */
  66. ‌static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
  67. {
  68.   IRRef retf = J->chain[IR_RETF];  /* Limits SLOAD restore elimination. */
  69.   BCReg s;
  70.   MSize n = 0;
  71.   for (s = 0; s < nslots; s++) {
  72.     TRef tr = J->slot[s];
  73.     IRRef ref = tref_ref(tr);
  74.     if (ref) {
  75.       SnapEntry sn = SNAP_TR(s, tr);
  76.       IRIns *ir = IR(ref);
  77.       if (!(sn & (SNAP_CONT|SNAP_FRAME)) &&
  78.           ir->o == IR_SLOAD && ir->op1 == s && ref > retf) {
  79.         /* No need to snapshot unmodified non-inherited slots. */
  80.         if (!(ir->op2 & IRSLOAD_INHERIT))
  81.           continue;
  82.         /* No need to restore readonly slots and unmodified non-parent slots. */
  83.         if (!(LJ_DUALNUM && (ir->op2 & IRSLOAD_CONVERT)) &&
  84.             (ir->op2 & (IRSLOAD_READONLY|IRSLOAD_PARENT)) != IRSLOAD_PARENT)
  85.           sn |= SNAP_NORESTORE;
  86.       }
  87.       if (LJ_SOFTFP && irt_isnum(ir->t))
  88.         sn |= SNAP_SOFTFPNUM;
  89.       map[n++] = sn;
  90.     }
  91.   }
  92.   return n;
  93. }

  95. /* Add frame links at the end of the snapshot. */
  96. ‌static BCReg snapshot_framelinks(jit_State *J, SnapEntry *map)
  97. {
  98.   cTValue *frame = J->L->base - 1;
  99.   cTValue *lim = J->L->base - J->baseslot;
  100.   GCfunc *fn = frame_func(frame);
  101.   cTValue *ftop = isluafunc(fn) ? (frame+funcproto(fn)->framesize) : J->L->top;
  102.   MSize f = 0;
  103.   lua_assert(!LJ_FR2);  /* TODO_FR2: store 64 bit PCs. */
  104.   map[f++] = SNAP_MKPC(J->pc);  /* The current PC is always the first entry. */
  105.   while (frame > lim) {  /* Backwards traversal of all frames above base. */
  106.     if (frame_islua(frame)) {
  107.       map[f++] = SNAP_MKPC(frame_pc(frame));
  108.       frame = frame_prevl(frame);
  109.     } else if (frame_iscont(frame)) {
  110.       map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
  111.       map[f++] = SNAP_MKPC(frame_contpc(frame));
  112.       frame = frame_prevd(frame);
  113.     } else {
  114.       lua_assert(!frame_isc(frame));
  115.       map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
  116.       frame = frame_prevd(frame);
  117.       continue;
  118.     }
  119.     if (frame + funcproto(frame_func(frame))->framesize > ftop)
  120.       ftop = frame + funcproto(frame_func(frame))->framesize;
  121.   }
  122.   lua_assert(f == (MSize)(1 + J->framedepth));
  123.   return (BCReg)(ftop - lim);
  124. }

  126. /* Take a snapshot of the current stack. */
  127. ‌static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
  128. {
  129.   BCReg nslots = J->baseslot + J->maxslot;
  130.   MSize nent;
  131.   SnapEntry *p;
  132.   /* Conservative estimate. */
  133.   lj_snap_grow_map(J, nsnapmap + nslots + (MSize)J->framedepth+1);
  134.   p = &J->cur.snapmap[nsnapmap];
  135.   nent = snapshot_slots(J, p, nslots);
  136.   snap->topslot = (uint8_t)snapshot_framelinks(J, p + nent);
  137.   snap->mapofs = (uint16_t)nsnapmap;
  138.   snap->ref = (IRRef1)J->cur.nins;
  139.   snap->nent = (uint8_t)nent;
  140.   snap->nslots = (uint8_t)nslots;
  141.   snap->count = 0;
  142.   J->cur.nsnapmap = (uint16_t)(nsnapmap + nent + 1 + J->framedepth);
  143. }

  145. /* Add or merge a snapshot. */
  146. ‌void lj_snap_add(jit_State *J)
  147. {
  148.   MSize nsnap = J->cur.nsnap;
  149.   MSize nsnapmap = J->cur.nsnapmap;
  150.   /* Merge if no ins. inbetween or if requested and no guard inbetween. */
  151.   if (J->mergesnap ? !irt_isguard(J->guardemit) :
  152.       (nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) {
  153.     if (nsnap == 1) {  /* But preserve snap #0 PC. */
  154.       emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
  155.       goto nomerge;
  156.     }
  157.     nsnapmap = J->cur.snap[--nsnap].mapofs;
  158.   } else {
  159.   nomerge:
  160.     lj_snap_grow_buf(J, nsnap+1);
  161.     J->cur.nsnap = (uint16_t)(nsnap+1);
  162.   }
  163.   J->mergesnap = 0;
  164.   J->guardemit.irt = 0;
  165.   snapshot_stack(J, &J->cur.snap[nsnap], nsnapmap);
  166. }

  168. /* -- Snapshot modification ----------------------------------------------- */

  170. ‌#define SNAP_USEDEF_SLOTS        (LJ_MAX_JSLOTS+LJ_STACK_EXTRA)

  172. /* Find unused slots with reaching-definitions bytecode data-flow analysis. */
  173. ‌static BCReg snap_usedef(jit_State *J, uint8_t *udf,
  174.                          const BCIns *pc, BCReg maxslot)
  175. {
  176.   BCReg s;
  177.   GCobj *o;

  179.   if (maxslot == 0) return 0;
  180. #ifdef LUAJIT_USE_VALGRIND
  181.   /* Avoid errors for harmless reads beyond maxslot. */
  182.   memset(udf, 1, SNAP_USEDEF_SLOTS);
  183. #else
  184.   memset(udf, 1, maxslot);
  185. #endif

  187.   /* Treat open upvalues as used. */
  188.   o = gcref(J->L->openupval);
  189.   while (o) {
  190.     if (uvval(gco2uv(o)) < J->L->base) break;
  191.     udf[uvval(gco2uv(o)) - J->L->base] = 0;
  192.     o = gcref(o->gch.nextgc);
  193.   }

  195. ‌#define USE_SLOT(s)                udf[(s)] &= ~1
  196. ‌#define DEF_SLOT(s)                udf[(s)] *= 3

  198.   /* Scan through following bytecode and check for uses/defs. */
  199.   lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
  200.   for (;;) {
  201.     BCIns ins = *pc++;
  202.     BCOp op = bc_op(ins);
  203.     switch (bcmode_b(op)) {
  204.     case BCMvar: USE_SLOT(bc_b(ins)); break;
  205.     default: break;
  206.     }
  207.     switch (bcmode_c(op)) {
  208.     case BCMvar: USE_SLOT(bc_c(ins)); break;
  209.     case BCMrbase:
  210.       lua_assert(op == BC_CAT);
  211.       for (s = bc_b(ins); s <= bc_c(ins); s++) USE_SLOT(s);
  212.       for (; s < maxslot; s++) DEF_SLOT(s);
  213.       break;
  214.     case BCMjump:
  215.     handle_jump: {
  216.       BCReg minslot = bc_a(ins);
  217.       if (op >= BC_FORI && op <= BC_JFORL) minslot += FORL_EXT;
  218.       else if (op >= BC_ITERL && op <= BC_JITERL) minslot += bc_b(pc[-2])-1;
  219.       else if (op == BC_UCLO) { pc += bc_j(ins); break; }
  220.       for (s = minslot; s < maxslot; s++) DEF_SLOT(s);
  221.       return minslot < maxslot ? minslot : maxslot;
  222.       }
  223.     case BCMlit:
  224.       if (op == BC_JFORL || op == BC_JITERL || op == BC_JLOOP) {
  225.         goto handle_jump;
  226.       } else if (bc_isret(op)) {
  227.         BCReg top = op == BC_RETM ? maxslot : (bc_a(ins) + bc_d(ins)-1);
  228.         for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
  229.         for (; s < top; s++) USE_SLOT(s);
  230.         for (; s < maxslot; s++) DEF_SLOT(s);
  231.         return 0;
  232.       }
  233.       break;
  234.     case BCMfunc: return maxslot;  /* NYI: will abort, anyway. */
  235.     default: break;
  236.     }
  237.     switch (bcmode_a(op)) {
  238.     case BCMvar: USE_SLOT(bc_a(ins)); break;
  239.     case BCMdst:
  240.        if (!(op == BC_ISTC || op == BC_ISFC)) DEF_SLOT(bc_a(ins));
  241.        break;
  242.     case BCMbase:
  243.       if (op >= BC_CALLM && op <= BC_VARG) {
  244.         BCReg top = (op == BC_CALLM || op == BC_CALLMT || bc_c(ins) == 0) ?
  245.                     maxslot : (bc_a(ins) + bc_c(ins)+LJ_FR2);
  246.         if (LJ_FR2) DEF_SLOT(bc_a(ins)+1);
  247.         s = bc_a(ins) - ((op == BC_ITERC || op == BC_ITERN) ? 3 : 0);
  248.         for (; s < top; s++) USE_SLOT(s);
  249.         for (; s < maxslot; s++) DEF_SLOT(s);
  250.         if (op == BC_CALLT || op == BC_CALLMT) {
  251.           for (s = 0; s < bc_a(ins); s++) DEF_SLOT(s);
  252.           return 0;
  253.         }
  254.       } else if (op == BC_KNIL) {
  255.         for (s = bc_a(ins); s <= bc_d(ins); s++) DEF_SLOT(s);
  256.       } else if (op == BC_TSETM) {
  257.         for (s = bc_a(ins)-1; s < maxslot; s++) USE_SLOT(s);
  258.       }
  259.       break;
  260.     default: break;
  261.     }
  262.     lua_assert(pc >= proto_bc(J->pt) && pc < proto_bc(J->pt) + J->pt->sizebc);
  263.   }

  265. ‌#undef USE_SLOT
  266. ‌#undef DEF_SLOT

  268.   return 0;  /* unreachable */
  269. }

  271. /* Purge dead slots before the next snapshot. */
  272. ‌void lj_snap_purge(jit_State *J)
  273. {
  274.   uint8_t udf[SNAP_USEDEF_SLOTS];
  275.   BCReg maxslot = J->maxslot;
  276.   BCReg s = snap_usedef(J, udf, J->pc, maxslot);
  277.   for (; s < maxslot; s++)
  278.     if (udf[s] != 0)
  279.       J->base[s] = 0;  /* Purge dead slots. */
  280. }

  282. /* Shrink last snapshot. */
  283. ‌void lj_snap_shrink(jit_State *J)
  284. {
  285.   SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
  286.   SnapEntry *map = &J->cur.snapmap[snap->mapofs];
  287.   MSize n, m, nlim, nent = snap->nent;
  288.   uint8_t udf[SNAP_USEDEF_SLOTS];
  289.   BCReg maxslot = J->maxslot;
  290.   BCReg minslot = snap_usedef(J, udf, snap_pc(map[nent]), maxslot);
  291.   BCReg baseslot = J->baseslot;
  292.   maxslot += baseslot;
  293.   minslot += baseslot;
  294.   snap->nslots = (uint8_t)maxslot;
  295.   for (n = m = 0; n < nent; n++) {  /* Remove unused slots from snapshot. */
  296.     BCReg s = snap_slot(map[n]);
  297.     if (s < minslot || (s < maxslot && udf[s-baseslot] == 0))
  298.       map[m++] = map[n];  /* Only copy used slots. */
  299.   }
  300.   snap->nent = (uint8_t)m;
  301.   nlim = J->cur.nsnapmap - snap->mapofs - 1;
  302.   while (n <= nlim) map[m++] = map[n++];  /* Move PC + frame links down. */
  303.   J->cur.nsnapmap = (uint16_t)(snap->mapofs + m);  /* Free up space in map. */
  304. }

  306. /* -- Snapshot access ----------------------------------------------------- */

  308. /* Initialize a Bloom Filter with all renamed refs.
  309. ** There are very few renames (often none), so the filter has
  310. ** very few bits set. This makes it suitable for negative filtering.
  311. */
  312. ‌static BloomFilter snap_renamefilter(GCtrace *T, SnapNo lim)
  313. {
  314.   BloomFilter rfilt = 0;
  315.   IRIns *ir;
  316.   for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
  317.     if (ir->op2 <= lim)
  318.       bloomset(rfilt, ir->op1);
  319.   return rfilt;
  320. }

  322. /* Process matching renames to find the original RegSP. */
  323. ‌static RegSP snap_renameref(GCtrace *T, SnapNo lim, IRRef ref, RegSP rs)
  324. {
  325.   IRIns *ir;
  326.   for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
  327.     if (ir->op1 == ref && ir->op2 <= lim)
  328.       rs = ir->prev;
  329.   return rs;
  330. }

  332. /* Copy RegSP from parent snapshot to the parent links of the IR. */
  333. ‌IRIns *lj_snap_regspmap(GCtrace *T, SnapNo snapno, IRIns *ir)
  334. {
  335.   SnapShot *snap = &T->snap[snapno];
  336.   SnapEntry *map = &T->snapmap[snap->mapofs];
  337.   BloomFilter rfilt = snap_renamefilter(T, snapno);
  338.   MSize n = 0;
  339.   IRRef ref = 0;
  340.   for ( ; ; ir++) {
  341.     uint32_t rs;
  342.     if (ir->o == IR_SLOAD) {
  343.       if (!(ir->op2 & IRSLOAD_PARENT)) break;
  344.       for ( ; ; n++) {
  345.         lua_assert(n < snap->nent);
  346.         if (snap_slot(map[n]) == ir->op1) {
  347.           ref = snap_ref(map[n++]);
  348.           break;
  349.         }
  350.       }
  351.     } else if (LJ_SOFTFP && ir->o == IR_HIOP) {
  352.       ref++;
  353.     } else if (ir->o == IR_PVAL) {
  354.       ref = ir->op1 + REF_BIAS;
  355.     } else {
  356.       break;
  357.     }
  358.     rs = T->ir[ref].prev;
  359.     if (bloomtest(rfilt, ref))
  360.       rs = snap_renameref(T, snapno, ref, rs);
  361.     ir->prev = (uint16_t)rs;
  362.     lua_assert(regsp_used(rs));
  363.   }
  364.   return ir;
  365. }

  367. /* -- Snapshot replay ----------------------------------------------------- */

  369. /* Replay constant from parent trace. */
  370. ‌static TRef snap_replay_const(jit_State *J, IRIns *ir)
  371. {
  372.   /* Only have to deal with constants that can occur in stack slots. */
  373.   switch ((IROp)ir->o) {
  374.   case IR_KPRI: return TREF_PRI(irt_type(ir->t));
  375.   case IR_KINT: return lj_ir_kint(J, ir->i);
  376.   case IR_KGC: return lj_ir_kgc(J, ir_kgc(ir), irt_t(ir->t));
  377.   case IR_KNUM: return lj_ir_k64(J, IR_KNUM, ir_knum(ir));
  378.   case IR_KINT64: return lj_ir_k64(J, IR_KINT64, ir_kint64(ir));
  379.   case IR_KPTR: return lj_ir_kptr(J, ir_kptr(ir));  /* Continuation. */
  380.   default: lua_assert(0); return TREF_NIL; break;
  381.   }
  382. }

  384. /* De-duplicate parent reference. */
  385. ‌static TRef snap_dedup(jit_State *J, SnapEntry *map, MSize nmax, IRRef ref)
  386. {
  387.   MSize j;
  388.   for (j = 0; j < nmax; j++)
  389.     if (snap_ref(map[j]) == ref)
  390.       return J->slot[snap_slot(map[j])] & ~(SNAP_CONT|SNAP_FRAME);
  391.   return 0;
  392. }

  394. /* Emit parent reference with de-duplication. */
  395. ‌static TRef snap_pref(jit_State *J, GCtrace *T, SnapEntry *map, MSize nmax,
  396.                       BloomFilter seen, IRRef ref)
  397. {
  398.   IRIns *ir = &T->ir[ref];
  399.   TRef tr;
  400.   if (irref_isk(ref))
  401.     tr = snap_replay_const(J, ir);
  402.   else if (!regsp_used(ir->prev))
  403.     tr = 0;
  404.   else if (!bloomtest(seen, ref) || (tr = snap_dedup(J, map, nmax, ref)) == 0)
  405.     tr = emitir(IRT(IR_PVAL, irt_type(ir->t)), ref - REF_BIAS, 0);
  406.   return tr;
  407. }

  409. /* Check whether a sunk store corresponds to an allocation. Slow path. */
  410. ‌static int snap_sunk_store2(jit_State *J, IRIns *ira, IRIns *irs)
  411. {
  412.   if (irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
  413.       irs->o == IR_FSTORE || irs->o == IR_XSTORE) {
  414.     IRIns *irk = IR(irs->op1);
  415.     if (irk->o == IR_AREF || irk->o == IR_HREFK)
  416.       irk = IR(irk->op1);
  417.     return (IR(irk->op1) == ira);
  418.   }
  419.   return 0;
  420. }

  422. /* Check whether a sunk store corresponds to an allocation. Fast path. */
  423. ‌static LJ_AINLINE int snap_sunk_store(jit_State *J, IRIns *ira, IRIns *irs)
  424. {
  425.   if (irs->s != 255)
  426.     return (ira + irs->s == irs);  /* Fast check. */
  427.   return snap_sunk_store2(J, ira, irs);
  428. }

  430. /* Replay snapshot state to setup side trace. */
  431. ‌void lj_snap_replay(jit_State *J, GCtrace *T)
  432. {
  433.   SnapShot *snap = &T->snap[J->exitno];
  434.   SnapEntry *map = &T->snapmap[snap->mapofs];
  435.   MSize n, nent = snap->nent;
  436.   BloomFilter seen = 0;
  437.   int pass23 = 0;
  438.   J->framedepth = 0;
  439.   /* Emit IR for slots inherited from parent snapshot. */
  440.   for (n = 0; n < nent; n++) {
  441.     SnapEntry sn = map[n];
  442.     BCReg s = snap_slot(sn);
  443.     IRRef ref = snap_ref(sn);
  444.     IRIns *ir = &T->ir[ref];
  445.     TRef tr;
  446.     /* The bloom filter avoids O(nent^2) overhead for de-duping slots. */
  447.     if (bloomtest(seen, ref) && (tr = snap_dedup(J, map, n, ref)) != 0)
  448.       goto setslot;
  449.     bloomset(seen, ref);
  450.     if (irref_isk(ref)) {
  451.       tr = snap_replay_const(J, ir);
  452.     } else if (!regsp_used(ir->prev)) {
  453.       pass23 = 1;
  454.       lua_assert(s != 0);
  455.       tr = s;
  456.     } else {
  457.       IRType t = irt_type(ir->t);
  458.       uint32_t mode = IRSLOAD_INHERIT|IRSLOAD_PARENT;
  459.       if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM)) t = IRT_NUM;
  460.       if (ir->o == IR_SLOAD) mode |= (ir->op2 & IRSLOAD_READONLY);
  461.       tr = emitir_raw(IRT(IR_SLOAD, t), s, mode);
  462.     }
  463.   setslot:
  464.     J->slot[s] = tr | (sn&(SNAP_CONT|SNAP_FRAME));  /* Same as TREF_* flags. */
  465.     J->framedepth += ((sn & (SNAP_CONT|SNAP_FRAME)) && s);
  466.     if ((sn & SNAP_FRAME))
  467.       J->baseslot = s+1;
  468.   }
  469.   if (pass23) {
  470.     IRIns *irlast = &T->ir[snap->ref];
  471.     pass23 = 0;
  472.     /* Emit dependent PVALs. */
  473.     for (n = 0; n < nent; n++) {
  474.       SnapEntry sn = map[n];
  475.       IRRef refp = snap_ref(sn);
  476.       IRIns *ir = &T->ir[refp];
  477.       if (regsp_reg(ir->r) == RID_SUNK) {
  478.         if (J->slot[snap_slot(sn)] != snap_slot(sn)) continue;
  479.         pass23 = 1;
  480.         lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
  481.                    ir->o == IR_CNEW || ir->o == IR_CNEWI);
  482.         if (ir->op1 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op1);
  483.         if (ir->op2 >= T->nk) snap_pref(J, T, map, nent, seen, ir->op2);
  484.         if (LJ_HASFFI && ir->o == IR_CNEWI) {
  485.           if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP)
  486.             snap_pref(J, T, map, nent, seen, (ir+1)->op2);
  487.         } else {
  488.           IRIns *irs;
  489.           for (irs = ir+1; irs < irlast; irs++)
  490.             if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
  491.               if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
  492.                 snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
  493.               else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
  494.                        irs+1 < irlast && (irs+1)->o == IR_HIOP)
  495.                 snap_pref(J, T, map, nent, seen, (irs+1)->op2);
  496.             }
  497.         }
  498.       } else if (!irref_isk(refp) && !regsp_used(ir->prev)) {
  499.         lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
  500.         J->slot[snap_slot(sn)] = snap_pref(J, T, map, nent, seen, ir->op1);
  501.       }
  502.     }
  503.     /* Replay sunk instructions. */
  504.     for (n = 0; pass23 && n < nent; n++) {
  505.       SnapEntry sn = map[n];
  506.       IRRef refp = snap_ref(sn);
  507.       IRIns *ir = &T->ir[refp];
  508.       if (regsp_reg(ir->r) == RID_SUNK) {
  509.         TRef op1, op2;
  510.         if (J->slot[snap_slot(sn)] != snap_slot(sn)) {  /* De-dup allocs. */
  511.           J->slot[snap_slot(sn)] = J->slot[J->slot[snap_slot(sn)]];
  512.           continue;
  513.         }
  514.         op1 = ir->op1;
  515.         if (op1 >= T->nk) op1 = snap_pref(J, T, map, nent, seen, op1);
  516.         op2 = ir->op2;
  517.         if (op2 >= T->nk) op2 = snap_pref(J, T, map, nent, seen, op2);
  518.         if (LJ_HASFFI && ir->o == IR_CNEWI) {
  519.           if (LJ_32 && refp+1 < T->nins && (ir+1)->o == IR_HIOP) {
  520.             lj_needsplit(J);  /* Emit joining HIOP. */
  521.             op2 = emitir_raw(IRT(IR_HIOP, IRT_I64), op2,
  522.                              snap_pref(J, T, map, nent, seen, (ir+1)->op2));
  523.           }
  524.           J->slot[snap_slot(sn)] = emitir(ir->ot, op1, op2);
  525.         } else {
  526.           IRIns *irs;
  527.           TRef tr = emitir(ir->ot, op1, op2);
  528.           J->slot[snap_slot(sn)] = tr;
  529.           for (irs = ir+1; irs < irlast; irs++)
  530.             if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
  531.               IRIns *irr = &T->ir[irs->op1];
  532.               TRef val, key = irr->op2, tmp = tr;
  533.               if (irr->o != IR_FREF) {
  534.                 IRIns *irk = &T->ir[key];
  535.                 if (irr->o == IR_HREFK)
  536.                   key = lj_ir_kslot(J, snap_replay_const(J, &T->ir[irk->op1]),
  537.                                     irk->op2);
  538.                 else
  539.                   key = snap_replay_const(J, irk);
  540.                 if (irr->o == IR_HREFK || irr->o == IR_AREF) {
  541.                   IRIns *irf = &T->ir[irr->op1];
  542.                   tmp = emitir(irf->ot, tmp, irf->op2);
  543.                 }
  544.               }
  545.               tmp = emitir(irr->ot, tmp, key);
  546.               val = snap_pref(J, T, map, nent, seen, irs->op2);
  547.               if (val == 0) {
  548.                 IRIns *irc = &T->ir[irs->op2];
  549.                 lua_assert(irc->o == IR_CONV && irc->op2 == IRCONV_NUM_INT);
  550.                 val = snap_pref(J, T, map, nent, seen, irc->op1);
  551.                 val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
  552.               } else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
  553.                          irs+1 < irlast && (irs+1)->o == IR_HIOP) {
  554.                 IRType t = IRT_I64;
  555.                 if (LJ_SOFTFP && irt_type((irs+1)->t) == IRT_SOFTFP)
  556.                   t = IRT_NUM;
  557.                 lj_needsplit(J);
  558.                 if (irref_isk(irs->op2) && irref_isk((irs+1)->op2)) {
  559.                   uint64_t k = (uint32_t)T->ir[irs->op2].i +
  560.                                ((uint64_t)T->ir[(irs+1)->op2].i << 32);
  561.                   val = lj_ir_k64(J, t == IRT_I64 ? IR_KINT64 : IR_KNUM,
  562.                                   lj_ir_k64_find(J, k));
  563.                 } else {
  564.                   val = emitir_raw(IRT(IR_HIOP, t), val,
  565.                           snap_pref(J, T, map, nent, seen, (irs+1)->op2));
  566.                 }
  567.                 tmp = emitir(IRT(irs->o, t), tmp, val);
  568.                 continue;
  569.               }
  570.               tmp = emitir(irs->ot, tmp, val);
  571.             } else if (LJ_HASFFI && irs->o == IR_XBAR && ir->o == IR_CNEW) {
  572.               emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
  573.             }
  574.         }
  575.       }
  576.     }
  577.   }
  578.   J->base = J->slot + J->baseslot;
  579.   J->maxslot = snap->nslots - J->baseslot;
  580.   lj_snap_add(J);
  581.   if (pass23)  /* Need explicit GC step _after_ initial snapshot. */
  582.     emitir_raw(IRTG(IR_GCSTEP, IRT_NIL), 0, 0);
  583. }

  585. /* -- Snapshot restore ---------------------------------------------------- */

  587. static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
  588.                         SnapNo snapno, BloomFilter rfilt,
  589.                         IRIns *ir, TValue *o);

  591. /* Restore a value from the trace exit state. */
  592. ‌static void snap_restoreval(jit_State *J, GCtrace *T, ExitState *ex,
  593.                             SnapNo snapno, BloomFilter rfilt,
  594.                             IRRef ref, TValue *o)
  595. {
  596.   IRIns *ir = &T->ir[ref];
  597.   IRType1 t = ir->t;
  598.   RegSP rs = ir->prev;
  599.   if (irref_isk(ref)) {  /* Restore constant slot. */
  600.     lj_ir_kvalue(J->L, o, ir);
  601.     return;
  602.   }
  603.   if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
  604.     rs = snap_renameref(T, snapno, ref, rs);
  605.   lua_assert(!LJ_GC64);  /* TODO_GC64: handle 64 bit references. */
  606.   if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
  607.     int32_t *sps = &ex->spill[regsp_spill(rs)];
  608.     if (irt_isinteger(t)) {
  609.       setintV(o, *sps);
  610. #if !LJ_SOFTFP
  611.     } else if (irt_isnum(t)) {
  612.       o->u64 = *(uint64_t *)sps;
  613. #endif
  614.     } else if (LJ_64 && irt_islightud(t)) {
  615.       /* 64 bit lightuserdata which may escape already has the tag bits. */
  616.       o->u64 = *(uint64_t *)sps;
  617.     } else {
  618.       lua_assert(!irt_ispri(t));  /* PRI refs never have a spill slot. */
  619.       setgcV(J->L, o, (GCobj *)(uintptr_t)*(GCSize *)sps, irt_toitype(t));
  620.     }
  621.   } else {  /* Restore from register. */
  622.     Reg r = regsp_reg(rs);
  623.     if (ra_noreg(r)) {
  624.       lua_assert(ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
  625.       snap_restoreval(J, T, ex, snapno, rfilt, ir->op1, o);
  626.       if (LJ_DUALNUM) setnumV(o, (lua_Number)intV(o));
  627.       return;
  628.     } else if (irt_isinteger(t)) {
  629.       setintV(o, (int32_t)ex->gpr[r-RID_MIN_GPR]);
  630. #if !LJ_SOFTFP
  631.     } else if (irt_isnum(t)) {
  632.       setnumV(o, ex->fpr[r-RID_MIN_FPR]);
  633. #endif
  634.     } else if (LJ_64 && irt_islightud(t)) {
  635.       /* 64 bit lightuserdata which may escape already has the tag bits. */
  636.       o->u64 = ex->gpr[r-RID_MIN_GPR];
  637.     } else if (irt_ispri(t)) {
  638.       setpriV(o, irt_toitype(t));
  639.     } else {
  640.       setgcV(J->L, o, (GCobj *)ex->gpr[r-RID_MIN_GPR], irt_toitype(t));
  641.     }
  642.   }
  643. }

  645. #if LJ_HASFFI
  646. /* Restore raw data from the trace exit state. */
  647. ‌static void snap_restoredata(GCtrace *T, ExitState *ex,
  648.                              SnapNo snapno, BloomFilter rfilt,
  649.                              IRRef ref, void *dst, CTSize sz)
  650. {
  651.   IRIns *ir = &T->ir[ref];
  652.   RegSP rs = ir->prev;
  653.   int32_t *src;
  654.   uint64_t tmp;
  655.   if (irref_isk(ref)) {
  656.     if (ir->o == IR_KNUM || ir->o == IR_KINT64) {
  657.       src = mref(ir->ptr, int32_t);
  658.     } else if (sz == 8) {
  659.       tmp = (uint64_t)(uint32_t)ir->i;
  660.       src = (int32_t *)&tmp;
  661.     } else {
  662.       src = &ir->i;
  663.     }
  664.   } else {
  665.     if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
  666.       rs = snap_renameref(T, snapno, ref, rs);
  667.     if (ra_hasspill(regsp_spill(rs))) {
  668.       src = &ex->spill[regsp_spill(rs)];
  669.       if (sz == 8 && !irt_is64(ir->t)) {
  670.         tmp = (uint64_t)(uint32_t)*src;
  671.         src = (int32_t *)&tmp;
  672.       }
  673.     } else {
  674.       Reg r = regsp_reg(rs);
  675.       if (ra_noreg(r)) {
  676.         /* Note: this assumes CNEWI is never used for SOFTFP split numbers. */
  677.         lua_assert(sz == 8 && ir->o == IR_CONV && ir->op2 == IRCONV_NUM_INT);
  678.         snap_restoredata(T, ex, snapno, rfilt, ir->op1, dst, 4);
  679.         *(lua_Number *)dst = (lua_Number)*(int32_t *)dst;
  680.         return;
  681.       }
  682.       src = (int32_t *)&ex->gpr[r-RID_MIN_GPR];
  683. #if !LJ_SOFTFP
  684.       if (r >= RID_MAX_GPR) {
  685.         src = (int32_t *)&ex->fpr[r-RID_MIN_FPR];
  686. #if LJ_TARGET_PPC
  687.         if (sz == 4) {  /* PPC FPRs are always doubles. */
  688.           *(float *)dst = (float)*(double *)src;
  689.           return;
  690.         }
  691. #else
  692.         if (LJ_BE && sz == 4) src++;
  693. #endif
  694.       }
  695. #endif
  696.     }
  697.   }
  698.   lua_assert(sz == 1 || sz == 2 || sz == 4 || sz == 8);
  699.   if (sz == 4) *(int32_t *)dst = *src;
  700.   else if (sz == 8) *(int64_t *)dst = *(int64_t *)src;
  701.   else if (sz == 1) *(int8_t *)dst = (int8_t)*src;
  702.   else *(int16_t *)dst = (int16_t)*src;
  703. }
  704. #endif

  706. /* Unsink allocation from the trace exit state. Unsink sunk stores. */
  707. ‌static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
  708.                         SnapNo snapno, BloomFilter rfilt,
  709.                         IRIns *ir, TValue *o)
  710. {
  711.   lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP ||
  712.              ir->o == IR_CNEW || ir->o == IR_CNEWI);
  713. #if LJ_HASFFI
  714.   if (ir->o == IR_CNEW || ir->o == IR_CNEWI) {
  715.     CTState *cts = ctype_cts(J->L);
  716.     CTypeID id = (CTypeID)T->ir[ir->op1].i;
  717.     CTSize sz = lj_ctype_size(cts, id);
  718.     GCcdata *cd = lj_cdata_new(cts, id, sz);
  719.     setcdataV(J->L, o, cd);
  720.     if (ir->o == IR_CNEWI) {
  721.       uint8_t *p = (uint8_t *)cdataptr(cd);
  722.       lua_assert(sz == 4 || sz == 8);
  723.       if (LJ_32 && sz == 8 && ir+1 < T->ir + T->nins && (ir+1)->o == IR_HIOP) {
  724.         snap_restoredata(T, ex, snapno, rfilt, (ir+1)->op2, LJ_LE?p+4:p, 4);
  725.         if (LJ_BE) p += 4;
  726.         sz = 4;
  727.       }
  728.       snap_restoredata(T, ex, snapno, rfilt, ir->op2, p, sz);
  729.     } else {
  730.       IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
  731.       for (irs = ir+1; irs < irlast; irs++)
  732.         if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
  733.           IRIns *iro = &T->ir[T->ir[irs->op1].op2];
  734.           uint8_t *p = (uint8_t *)cd;
  735.           CTSize szs;
  736.           lua_assert(irs->o == IR_XSTORE && T->ir[irs->op1].o == IR_ADD);
  737.           lua_assert(iro->o == IR_KINT || iro->o == IR_KINT64);
  738.           if (irt_is64(irs->t)) szs = 8;
  739.           else if (irt_isi8(irs->t) || irt_isu8(irs->t)) szs = 1;
  740.           else if (irt_isi16(irs->t) || irt_isu16(irs->t)) szs = 2;
  741.           else szs = 4;
  742.           if (LJ_64 && iro->o == IR_KINT64)
  743.             p += (int64_t)ir_k64(iro)->u64;
  744.           else
  745.             p += iro->i;
  746.           lua_assert(p >= (uint8_t *)cdataptr(cd) &&
  747.                      p + szs <= (uint8_t *)cdataptr(cd) + sz);
  748.           if (LJ_32 && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
  749.             lua_assert(szs == 4);
  750.             snap_restoredata(T, ex, snapno, rfilt, (irs+1)->op2, LJ_LE?p+4:p,4);
  751.             if (LJ_BE) p += 4;
  752.           }
  753.           snap_restoredata(T, ex, snapno, rfilt, irs->op2, p, szs);
  754.         }
  755.     }
  756.   } else
  757. #endif
  758.   {
  759.     IRIns *irs, *irlast;
  760.     GCtab *t = ir->o == IR_TNEW ? lj_tab_new(J->L, ir->op1, ir->op2) :
  761.                                   lj_tab_dup(J->L, ir_ktab(&T->ir[ir->op1]));
  762.     settabV(J->L, o, t);
  763.     irlast = &T->ir[T->snap[snapno].ref];
  764.     for (irs = ir+1; irs < irlast; irs++)
  765.       if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
  766.         IRIns *irk = &T->ir[irs->op1];
  767.         TValue tmp, *val;
  768.         lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
  769.                    irs->o == IR_FSTORE);
  770.         if (irk->o == IR_FREF) {
  771.           lua_assert(irk->op2 == IRFL_TAB_META);
  772.           snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, &tmp);
  773.           /* NOBARRIER: The table is new (marked white). */
  774.           setgcref(t->metatable, obj2gco(tabV(&tmp)));
  775.         } else {
  776.           irk = &T->ir[irk->op2];
  777.           if (irk->o == IR_KSLOT) irk = &T->ir[irk->op1];
  778.           lj_ir_kvalue(J->L, &tmp, irk);
  779.           val = lj_tab_set(J->L, t, &tmp);
  780.           /* NOBARRIER: The table is new (marked white). */
  781.           snap_restoreval(J, T, ex, snapno, rfilt, irs->op2, val);
  782.           if (LJ_SOFTFP && irs+1 < T->ir + T->nins && (irs+1)->o == IR_HIOP) {
  783.             snap_restoreval(J, T, ex, snapno, rfilt, (irs+1)->op2, &tmp);
  784.             val->u32.hi = tmp.u32.lo;
  785.           }
  786.         }
  787.       }
  788.   }
  789. }

  791. /* Restore interpreter state from exit state with the help of a snapshot. */
  792. ‌const BCIns *lj_snap_restore(jit_State *J, void *exptr)
  793. {
  794.   ExitState *ex = (ExitState *)exptr;
  795.   SnapNo snapno = J->exitno;  /* For now, snapno == exitno. */
  796.   GCtrace *T = traceref(J, J->parent);
  797.   SnapShot *snap = &T->snap[snapno];
  798.   MSize n, nent = snap->nent;
  799.   SnapEntry *map = &T->snapmap[snap->mapofs];
  800.   SnapEntry *flinks = &T->snapmap[snap_nextofs(T, snap)-1];
  801.   ptrdiff_t ftsz0;
  802.   TValue *frame;
  803.   BloomFilter rfilt = snap_renamefilter(T, snapno);
  804.   const BCIns *pc = snap_pc(map[nent]);
  805.   lua_State *L = J->L;

  807.   /* Set interpreter PC to the next PC to get correct error messages. */
  808.   setcframe_pc(cframe_raw(L->cframe), pc+1);

  810.   /* Make sure the stack is big enough for the slots from the snapshot. */
  811.   if (LJ_UNLIKELY(L->base + snap->topslot >= tvref(L->maxstack))) {
  812.     L->top = curr_topL(L);
  813.     lj_state_growstack(L, snap->topslot - curr_proto(L)->framesize);
  814.   }

  816.   /* Fill stack slots with data from the registers and spill slots. */
  817.   frame = L->base-1;
  818.   ftsz0 = frame_ftsz(frame);  /* Preserve link to previous frame in slot #0. */
  819.   for (n = 0; n < nent; n++) {
  820.     SnapEntry sn = map[n];
  821.     if (!(sn & SNAP_NORESTORE)) {
  822.       TValue *o = &frame[snap_slot(sn)];
  823.       IRRef ref = snap_ref(sn);
  824.       IRIns *ir = &T->ir[ref];
  825.       if (ir->r == RID_SUNK) {
  826.         MSize j;
  827.         for (j = 0; j < n; j++)
  828.           if (snap_ref(map[j]) == ref) {  /* De-duplicate sunk allocations. */
  829.             copyTV(L, o, &frame[snap_slot(map[j])]);
  830.             goto dupslot;
  831.           }
  832.         snap_unsink(J, T, ex, snapno, rfilt, ir, o);
  833.       dupslot:
  834.         continue;
  835.       }
  836.       snap_restoreval(J, T, ex, snapno, rfilt, ref, o);
  837.       if (LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && tvisint(o)) {
  838.         TValue tmp;
  839.         snap_restoreval(J, T, ex, snapno, rfilt, ref+1, &tmp);
  840.         o->u32.hi = tmp.u32.lo;
  841.       } else if ((sn & (SNAP_CONT|SNAP_FRAME))) {
  842.         lua_assert(!LJ_FR2);  /* TODO_FR2: store 64 bit PCs. */
  843.         /* Overwrite tag with frame link. */
  844.         setframe_ftsz(o, snap_slot(sn) != 0 ? (int32_t)*flinks-- : ftsz0);
  845.         L->base = o+1;
  846.       }
  847.     }
  848.   }
  849.   lua_assert(map + nent == flinks);

  851.   /* Compute current stack top. */
  852.   switch (bc_op(*pc)) {
  853.   default:
  854.     if (bc_op(*pc) < BC_FUNCF) {
  855.       L->top = curr_topL(L);
  856.       break;
  857.     }
  858.     /* fallthrough */
  859.   case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
  860.     L->top = frame + snap->nslots;
  861.     break;
  862.   }
  863.   return pc;
  864. }

  866. ‌#undef IR
  867. ‌#undef emitir_raw
  868. ‌#undef emitir

  870. #endif

One Level Up Top Level