One Level Up
Top Level
src/lj_dispatch.c - luajit-2.0-src
Global variables defined
Functions defined
Macros defined
Source code
- #define lj_dispatch_c
- #define LUA_CORE
- #include "lj_obj.h"
- #include "lj_err.h"
- #include "lj_buf.h"
- #include "lj_func.h"
- #include "lj_str.h"
- #include "lj_tab.h"
- #include "lj_meta.h"
- #include "lj_debug.h"
- #include "lj_state.h"
- #include "lj_frame.h"
- #include "lj_bc.h"
- #include "lj_ff.h"
- #include "lj_strfmt.h"
- #if LJ_HASJIT
- #include "lj_jit.h"
- #endif
- #if LJ_HASFFI
- #include "lj_ccallback.h"
- #endif
- #include "lj_trace.h"
- #include "lj_dispatch.h"
- #if LJ_HASPROFILE
- #include "lj_profile.h"
- #endif
- #include "lj_vm.h"
- #include "luajit.h"
- LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
- #if LJ_TARGET_MIPS
- #include <math.h>
- LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
- lua_State *co);
- #if !LJ_HASJIT
- #define lj_dispatch_stitch lj_dispatch_ins
- #endif
- #if !LJ_HASPROFILE
- #define lj_dispatch_profile lj_dispatch_ins
- #endif
- #define GOTFUNC(name) (ASMFunction)name,
- static const ASMFunction dispatch_got[] = {
- GOTDEF(GOTFUNC)
- };
- #undef GOTFUNC
- #endif
- void lj_dispatch_init(GG_State *GG)
- {
- uint32_t i;
- ASMFunction *disp = GG->dispatch;
- for (i = 0; i < GG_LEN_SDISP; i++)
- disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
- for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
- disp[i] = makeasmfunc(lj_bc_ofs[i]);
-
- disp[BC_FORL] = disp[BC_IFORL];
- disp[BC_ITERL] = disp[BC_IITERL];
- disp[BC_LOOP] = disp[BC_ILOOP];
- disp[BC_FUNCF] = disp[BC_IFUNCF];
- disp[BC_FUNCV] = disp[BC_IFUNCV];
- GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
- for (i = 0; i < GG_NUM_ASMFF; i++)
- GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
- #if LJ_TARGET_MIPS
- memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
- #endif
- }
- #if LJ_HASJIT
- void lj_dispatch_init_hotcount(global_State *g)
- {
- int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
- HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
- HotCount *hotcount = G2GG(g)->hotcount;
- uint32_t i;
- for (i = 0; i < HOTCOUNT_SIZE; i++)
- hotcount[i] = start;
- }
- #endif
- #define DISPMODE_CALL 0x01
- #define DISPMODE_RET 0x02
- #define DISPMODE_INS 0x04
- #define DISPMODE_JIT 0x10
- #define DISPMODE_REC 0x20
- #define DISPMODE_PROF 0x40
- void lj_dispatch_update(global_State *g)
- {
- uint8_t oldmode = g->dispatchmode;
- uint8_t mode = 0;
- #if LJ_HASJIT
- mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
- mode |= G2J(g)->state != LJ_TRACE_IDLE ?
- (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
- #endif
- #if LJ_HASPROFILE
- mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
- #endif
- mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
- mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
- mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
- if (oldmode != mode) {
- ASMFunction *disp = G2GG(g)->dispatch;
- ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
- g->dispatchmode = mode;
-
- if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
- f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
- f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
- f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
- f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
- f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
- } else {
- f_forl = disp[GG_LEN_DDISP+BC_IFORL];
- f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
- f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
- f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
- f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
- }
-
- disp[GG_LEN_DDISP+BC_FORL] = f_forl;
- disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
- disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
-
- if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
-
- if (!(mode & DISPMODE_INS)) {
-
- memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
-
- if ((mode & DISPMODE_RET)) {
- disp[BC_RETM] = lj_vm_rethook;
- disp[BC_RET] = lj_vm_rethook;
- disp[BC_RET0] = lj_vm_rethook;
- disp[BC_RET1] = lj_vm_rethook;
- }
- } else {
-
- ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
- (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
- uint32_t i;
- for (i = 0; i < GG_LEN_SDISP; i++)
- disp[i] = f;
- }
- } else if (!(mode & DISPMODE_INS)) {
-
- disp[BC_FORL] = f_forl;
- disp[BC_ITERL] = f_iterl;
- disp[BC_LOOP] = f_loop;
-
- if ((mode & DISPMODE_RET)) {
- disp[BC_RETM] = lj_vm_rethook;
- disp[BC_RET] = lj_vm_rethook;
- disp[BC_RET0] = lj_vm_rethook;
- disp[BC_RET1] = lj_vm_rethook;
- } else {
- disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
- disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
- disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
- disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
- }
- }
-
- if ((oldmode ^ mode) & DISPMODE_CALL) {
- uint32_t i;
- if ((mode & DISPMODE_CALL) == 0) {
- for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
- disp[i] = makeasmfunc(lj_bc_ofs[i]);
- } else {
- for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
- disp[i] = lj_vm_callhook;
- }
- }
- if (!(mode & DISPMODE_CALL)) {
- disp[BC_FUNCF] = f_funcf;
- disp[BC_FUNCV] = f_funcv;
- }
- #if LJ_HASJIT
-
- if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
- lj_dispatch_init_hotcount(g);
- #endif
- }
- }
- #if LJ_HASJIT
- static void setptmode(global_State *g, GCproto *pt, int mode)
- {
- if ((mode & LUAJIT_MODE_ON)) {
- pt->flags &= ~PROTO_NOJIT;
- lj_trace_reenableproto(pt);
- } else {
- if (!(mode & LUAJIT_MODE_FLUSH))
- pt->flags |= PROTO_NOJIT;
- lj_trace_flushproto(g, pt);
- }
- }
- static void setptmode_all(global_State *g, GCproto *pt, int mode)
- {
- ptrdiff_t i;
- if (!(pt->flags & PROTO_CHILD)) return;
- for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
- GCobj *o = proto_kgc(pt, i);
- if (o->gch.gct == ~LJ_TPROTO) {
- setptmode(g, gco2pt(o), mode);
- setptmode_all(g, gco2pt(o), mode);
- }
- }
- }
- #endif
- int luaJIT_setmode(lua_State *L, int idx, int mode)
- {
- global_State *g = G(L);
- int mm = mode & LUAJIT_MODE_MASK;
- lj_trace_abort(g);
-
- if ((g->hookmask & HOOK_GC))
- lj_err_caller(L, LJ_ERR_NOGCMM);
- switch (mm) {
- #if LJ_HASJIT
- case LUAJIT_MODE_ENGINE:
- if ((mode & LUAJIT_MODE_FLUSH)) {
- lj_trace_flushall(L);
- } else {
- if (!(mode & LUAJIT_MODE_ON))
- G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
- #if LJ_TARGET_X86ORX64
- else if ((G2J(g)->flags & JIT_F_SSE2))
- G2J(g)->flags |= (uint32_t)JIT_F_ON;
- else
- return 0;
- #else
- else
- G2J(g)->flags |= (uint32_t)JIT_F_ON;
- #endif
- lj_dispatch_update(g);
- }
- break;
- case LUAJIT_MODE_FUNC:
- case LUAJIT_MODE_ALLFUNC:
- case LUAJIT_MODE_ALLSUBFUNC: {
- cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
- idx > 0 ? L->base + (idx-1) : L->top + idx;
- GCproto *pt;
- if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
- pt = funcproto(&gcval(tv)->fn);
- else if (tvisproto(tv))
- pt = protoV(tv);
- else
- return 0;
- if (mm != LUAJIT_MODE_ALLSUBFUNC)
- setptmode(g, pt, mode);
- if (mm != LUAJIT_MODE_FUNC)
- setptmode_all(g, pt, mode);
- break;
- }
- case LUAJIT_MODE_TRACE:
- if (!(mode & LUAJIT_MODE_FLUSH))
- return 0;
- lj_trace_flush(G2J(g), idx);
- break;
- #else
- case LUAJIT_MODE_ENGINE:
- case LUAJIT_MODE_FUNC:
- case LUAJIT_MODE_ALLFUNC:
- case LUAJIT_MODE_ALLSUBFUNC:
- UNUSED(idx);
- if ((mode & LUAJIT_MODE_ON))
- return 0;
- break;
- #endif
- case LUAJIT_MODE_WRAPCFUNC:
- if ((mode & LUAJIT_MODE_ON)) {
- if (idx != 0) {
- cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
- if (tvislightud(tv))
- g->wrapf = (lua_CFunction)lightudV(tv);
- else
- return 0;
- } else {
- return 0;
- }
- g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
- } else {
- g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
- }
- break;
- default:
- return 0;
- }
- return 1;
- }
- LUA_API void LUAJIT_VERSION_SYM(void)
- {
- }
- LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
- {
- global_State *g = G(L);
- mask &= HOOK_EVENTMASK;
- if (func == NULL || mask == 0) { mask = 0; func = NULL; }
- g->hookf = func;
- g->hookcount = g->hookcstart = (int32_t)count;
- g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
- lj_trace_abort(g);
- lj_dispatch_update(g);
- return 1;
- }
- LUA_API lua_Hook lua_gethook(lua_State *L)
- {
- return G(L)->hookf;
- }
- LUA_API int lua_gethookmask(lua_State *L)
- {
- return G(L)->hookmask & HOOK_EVENTMASK;
- }
- LUA_API int lua_gethookcount(lua_State *L)
- {
- return (int)G(L)->hookcstart;
- }
- static void callhook(lua_State *L, int event, BCLine line)
- {
- global_State *g = G(L);
- lua_Hook hookf = g->hookf;
- if (hookf && !hook_active(g)) {
- lua_Debug ar;
- lj_trace_abort(g);
- ar.event = event;
- ar.currentline = line;
-
- ar.i_ci = (int)((L->base-1) - tvref(L->stack));
- lj_state_checkstack(L, 1+LUA_MINSTACK);
- #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
- lj_profile_hook_enter(g);
- #else
- hook_enter(g);
- #endif
- hookf(L, &ar);
- lua_assert(hook_active(g));
- setgcref(g->cur_L, obj2gco(L));
- #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
- lj_profile_hook_leave(g);
- #else
- hook_leave(g);
- #endif
- }
- }
- static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
- {
- BCIns ins = pc[-1];
- if (bc_op(ins) == BC_UCLO)
- ins = pc[bc_j(ins)];
- switch (bc_op(ins)) {
- case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
- case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
- case BC_TSETM: return bc_a(ins) + nres-1;
- default: return pt->framesize;
- }
- }
- void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
- {
- ERRNO_SAVE
- GCfunc *fn = curr_func(L);
- GCproto *pt = funcproto(fn);
- void *cf = cframe_raw(L->cframe);
- const BCIns *oldpc = cframe_pc(cf);
- global_State *g = G(L);
- BCReg slots;
- setcframe_pc(cf, pc);
- slots = cur_topslot(pt, pc, cframe_multres_n(cf));
- L->top = L->base + slots;
- #if LJ_HASJIT
- {
- jit_State *J = G2J(g);
- if (J->state != LJ_TRACE_IDLE) {
- #ifdef LUA_USE_ASSERT
- ptrdiff_t delta = L->top - L->base;
- #endif
- J->L = L;
- lj_trace_ins(J, pc-1);
- lua_assert(L->top - L->base == delta);
- }
- }
- #endif
- if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
- g->hookcount = g->hookcstart;
- callhook(L, LUA_HOOKCOUNT, -1);
- L->top = L->base + slots;
- }
- if ((g->hookmask & LUA_MASKLINE)) {
- BCPos npc = proto_bcpos(pt, pc) - 1;
- BCPos opc = proto_bcpos(pt, oldpc) - 1;
- BCLine line = lj_debug_line(pt, npc);
- if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
- callhook(L, LUA_HOOKLINE, line);
- L->top = L->base + slots;
- }
- }
- if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
- callhook(L, LUA_HOOKRET, -1);
- ERRNO_RESTORE
- }
- static int call_init(lua_State *L, GCfunc *fn)
- {
- if (isluafunc(fn)) {
- GCproto *pt = funcproto(fn);
- int numparams = pt->numparams;
- int gotparams = (int)(L->top - L->base);
- int need = pt->framesize;
- if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
- lj_state_checkstack(L, (MSize)need);
- numparams -= gotparams;
- return numparams >= 0 ? numparams : 0;
- } else {
- lj_state_checkstack(L, LUA_MINSTACK);
- return 0;
- }
- }
- ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
- {
- ERRNO_SAVE
- GCfunc *fn = curr_func(L);
- BCOp op;
- global_State *g = G(L);
- #if LJ_HASJIT
- jit_State *J = G2J(g);
- #endif
- int missing = call_init(L, fn);
- #if LJ_HASJIT
- J->L = L;
- if ((uintptr_t)pc & 1) {
- #ifdef LUA_USE_ASSERT
- ptrdiff_t delta = L->top - L->base;
- #endif
- pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
- lj_trace_hot(J, pc);
- lua_assert(L->top - L->base == delta);
- goto out;
- } else if (J->state != LJ_TRACE_IDLE &&
- !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
- #ifdef LUA_USE_ASSERT
- ptrdiff_t delta = L->top - L->base;
- #endif
-
- lj_trace_ins(J, pc-1);
- lua_assert(L->top - L->base == delta);
- }
- #endif
- if ((g->hookmask & LUA_MASKCALL)) {
- int i;
- for (i = 0; i < missing; i++)
- setnilV(L->top++);
- callhook(L, LUA_HOOKCALL, -1);
-
- while (missing-- > 0 && tvisnil(L->top - 1))
- L->top--;
- }
- #if LJ_HASJIT
- out:
- #endif
- op = bc_op(pc[-1]);
- #if LJ_HASJIT
-
- if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
- (op == BC_FUNCF || op == BC_FUNCV))
- op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
- #endif
- ERRNO_RESTORE
- return makeasmfunc(lj_bc_ofs[op]);
- }
- #if LJ_HASJIT
- void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
- {
- ERRNO_SAVE
- lua_State *L = J->L;
- void *cf = cframe_raw(L->cframe);
- const BCIns *oldpc = cframe_pc(cf);
- setcframe_pc(cf, pc);
-
- L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
- lj_trace_stitch(J, pc-1);
- setcframe_pc(cf, oldpc);
- ERRNO_RESTORE
- }
- #endif
- #if LJ_HASPROFILE
- void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
- {
- ERRNO_SAVE
- GCfunc *fn = curr_func(L);
- GCproto *pt = funcproto(fn);
- void *cf = cframe_raw(L->cframe);
- const BCIns *oldpc = cframe_pc(cf);
- global_State *g;
- setcframe_pc(cf, pc);
- L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
- lj_profile_interpreter(L);
- setcframe_pc(cf, oldpc);
- g = G(L);
- setgcref(g->cur_L, obj2gco(L));
- setvmstate(g, INTERP);
- ERRNO_RESTORE
- }
- #endif
One Level Up
Top Level