One Level Up
Top Level
src/lj_gc.c - luajit-2.0-src
Global variables defined
Data types defined
Functions defined
Macros defined
Source code
- #define lj_gc_c
- #define LUA_CORE
- #include "lj_obj.h"
- #include "lj_gc.h"
- #include "lj_err.h"
- #include "lj_buf.h"
- #include "lj_str.h"
- #include "lj_tab.h"
- #include "lj_func.h"
- #include "lj_udata.h"
- #include "lj_meta.h"
- #include "lj_state.h"
- #include "lj_frame.h"
- #if LJ_HASFFI
- #include "lj_ctype.h"
- #include "lj_cdata.h"
- #endif
- #include "lj_trace.h"
- #include "lj_vm.h"
- #define GCSTEPSIZE 1024u
- #define GCSWEEPMAX 40
- #define GCSWEEPCOST 10
- #define GCFINALIZECOST 100
- #define white2gray(x) ((x)->gch.marked &= (uint8_t)~LJ_GC_WHITES)
- #define gray2black(x) ((x)->gch.marked |= LJ_GC_BLACK)
- #define isfinalized(u) ((u)->marked & LJ_GC_FINALIZED)
- #define gc_marktv(g, tv) \
- { lua_assert(!tvisgcv(tv) || (~itype(tv) == gcval(tv)->gch.gct)); \
- if (tviswhite(tv)) gc_mark(g, gcV(tv)); }
- #define gc_markobj(g, o) \
- { if (iswhite(obj2gco(o))) gc_mark(g, obj2gco(o)); }
- #define gc_mark_str(s) ((s)->marked &= (uint8_t)~LJ_GC_WHITES)
- static void gc_mark(global_State *g, GCobj *o)
- {
- int gct = o->gch.gct;
- lua_assert(iswhite(o) && !isdead(g, o));
- white2gray(o);
- if (LJ_UNLIKELY(gct == ~LJ_TUDATA)) {
- GCtab *mt = tabref(gco2ud(o)->metatable);
- gray2black(o);
- if (mt) gc_markobj(g, mt);
- gc_markobj(g, tabref(gco2ud(o)->env));
- } else if (LJ_UNLIKELY(gct == ~LJ_TUPVAL)) {
- GCupval *uv = gco2uv(o);
- gc_marktv(g, uvval(uv));
- if (uv->closed)
- gray2black(o);
- } else if (gct != ~LJ_TSTR && gct != ~LJ_TCDATA) {
- lua_assert(gct == ~LJ_TFUNC || gct == ~LJ_TTAB ||
- gct == ~LJ_TTHREAD || gct == ~LJ_TPROTO);
- setgcrefr(o->gch.gclist, g->gc.gray);
- setgcref(g->gc.gray, o);
- }
- }
- static void gc_mark_gcroot(global_State *g)
- {
- ptrdiff_t i;
- for (i = 0; i < GCROOT_MAX; i++)
- if (gcref(g->gcroot[i]) != NULL)
- gc_markobj(g, gcref(g->gcroot[i]));
- }
- static void gc_mark_start(global_State *g)
- {
- setgcrefnull(g->gc.gray);
- setgcrefnull(g->gc.grayagain);
- setgcrefnull(g->gc.weak);
- gc_markobj(g, mainthread(g));
- gc_markobj(g, tabref(mainthread(g)->env));
- gc_marktv(g, &g->registrytv);
- gc_mark_gcroot(g);
- g->gc.state = GCSpropagate;
- }
- static void gc_mark_uv(global_State *g)
- {
- GCupval *uv;
- for (uv = uvnext(&g->uvhead); uv != &g->uvhead; uv = uvnext(uv)) {
- lua_assert(uvprev(uvnext(uv)) == uv && uvnext(uvprev(uv)) == uv);
- if (isgray(obj2gco(uv)))
- gc_marktv(g, uvval(uv));
- }
- }
- static void gc_mark_mmudata(global_State *g)
- {
- GCobj *root = gcref(g->gc.mmudata);
- GCobj *u = root;
- if (u) {
- do {
- u = gcnext(u);
- makewhite(g, u);
- gc_mark(g, u);
- } while (u != root);
- }
- }
- size_t lj_gc_separateudata(global_State *g, int all)
- {
- size_t m = 0;
- GCRef *p = &mainthread(g)->nextgc;
- GCobj *o;
- while ((o = gcref(*p)) != NULL) {
- if (!(iswhite(o) || all) || isfinalized(gco2ud(o))) {
- p = &o->gch.nextgc;
- } else if (!lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc)) {
- markfinalized(o);
- p = &o->gch.nextgc;
- } else {
- m += sizeudata(gco2ud(o));
- markfinalized(o);
- *p = o->gch.nextgc;
- if (gcref(g->gc.mmudata)) {
- GCobj *root = gcref(g->gc.mmudata);
- setgcrefr(o->gch.nextgc, root->gch.nextgc);
- setgcref(root->gch.nextgc, o);
- setgcref(g->gc.mmudata, o);
- } else {
- setgcref(o->gch.nextgc, o);
- setgcref(g->gc.mmudata, o);
- }
- }
- }
- return m;
- }
- static int gc_traverse_tab(global_State *g, GCtab *t)
- {
- int weak = 0;
- cTValue *mode;
- GCtab *mt = tabref(t->metatable);
- if (mt)
- gc_markobj(g, mt);
- mode = lj_meta_fastg(g, mt, MM_mode);
- if (mode && tvisstr(mode)) {
- const char *modestr = strVdata(mode);
- int c;
- while ((c = *modestr++)) {
- if (c == 'k') weak |= LJ_GC_WEAKKEY;
- else if (c == 'v') weak |= LJ_GC_WEAKVAL;
- else if (c == 'K') weak = (int)(~0u & ~LJ_GC_WEAKVAL);
- }
- if (weak > 0) {
- t->marked = (uint8_t)((t->marked & ~LJ_GC_WEAK) | weak);
- setgcrefr(t->gclist, g->gc.weak);
- setgcref(g->gc.weak, obj2gco(t));
- }
- }
- if (weak == LJ_GC_WEAK)
- return 1;
- if (!(weak & LJ_GC_WEAKVAL)) {
- MSize i, asize = t->asize;
- for (i = 0; i < asize; i++)
- gc_marktv(g, arrayslot(t, i));
- }
- if (t->hmask > 0) {
- Node *node = noderef(t->node);
- MSize i, hmask = t->hmask;
- for (i = 0; i <= hmask; i++) {
- Node *n = &node[i];
- if (!tvisnil(&n->val)) {
- lua_assert(!tvisnil(&n->key));
- if (!(weak & LJ_GC_WEAKKEY)) gc_marktv(g, &n->key);
- if (!(weak & LJ_GC_WEAKVAL)) gc_marktv(g, &n->val);
- }
- }
- }
- return weak;
- }
- static void gc_traverse_func(global_State *g, GCfunc *fn)
- {
- gc_markobj(g, tabref(fn->c.env));
- if (isluafunc(fn)) {
- uint32_t i;
- lua_assert(fn->l.nupvalues <= funcproto(fn)->sizeuv);
- gc_markobj(g, funcproto(fn));
- for (i = 0; i < fn->l.nupvalues; i++)
- gc_markobj(g, &gcref(fn->l.uvptr[i])->uv);
- } else {
- uint32_t i;
- for (i = 0; i < fn->c.nupvalues; i++)
- gc_marktv(g, &fn->c.upvalue[i]);
- }
- }
- #if LJ_HASJIT
- static void gc_marktrace(global_State *g, TraceNo traceno)
- {
- GCobj *o = obj2gco(traceref(G2J(g), traceno));
- lua_assert(traceno != G2J(g)->cur.traceno);
- if (iswhite(o)) {
- white2gray(o);
- setgcrefr(o->gch.gclist, g->gc.gray);
- setgcref(g->gc.gray, o);
- }
- }
- static void gc_traverse_trace(global_State *g, GCtrace *T)
- {
- IRRef ref;
- if (T->traceno == 0) return;
- for (ref = T->nk; ref < REF_TRUE; ref++) {
- IRIns *ir = &T->ir[ref];
- if (ir->o == IR_KGC)
- gc_markobj(g, ir_kgc(ir));
- }
- if (T->link) gc_marktrace(g, T->link);
- if (T->nextroot) gc_marktrace(g, T->nextroot);
- if (T->nextside) gc_marktrace(g, T->nextside);
- gc_markobj(g, gcref(T->startpt));
- }
- #define gc_traverse_curtrace(g) gc_traverse_trace(g, &G2J(g)->cur)
- #else
- #define gc_traverse_curtrace(g) UNUSED(g)
- #endif
- static void gc_traverse_proto(global_State *g, GCproto *pt)
- {
- ptrdiff_t i;
- gc_mark_str(proto_chunkname(pt));
- for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++)
- gc_markobj(g, proto_kgc(pt, i));
- #if LJ_HASJIT
- if (pt->trace) gc_marktrace(g, pt->trace);
- #endif
- }
- static MSize gc_traverse_frames(global_State *g, lua_State *th)
- {
- TValue *frame, *top = th->top-1, *bot = tvref(th->stack);
-
- for (frame = th->base-1; frame > bot+LJ_FR2; frame = frame_prev(frame)) {
- GCfunc *fn = frame_func(frame);
- TValue *ftop = frame;
- if (isluafunc(fn)) ftop += funcproto(fn)->framesize;
- if (ftop > top) top = ftop;
- if (!LJ_FR2) gc_markobj(g, fn);
- }
- top++;
- if (top > tvref(th->maxstack)) top = tvref(th->maxstack);
- return (MSize)(top - bot);
- }
- static void gc_traverse_thread(global_State *g, lua_State *th)
- {
- TValue *o, *top = th->top;
- for (o = tvref(th->stack)+1+LJ_FR2; o < top; o++)
- gc_marktv(g, o);
- if (g->gc.state == GCSatomic) {
- top = tvref(th->stack) + th->stacksize;
- for (; o < top; o++)
- setnilV(o);
- }
- gc_markobj(g, tabref(th->env));
- lj_state_shrinkstack(th, gc_traverse_frames(g, th));
- }
- static size_t propagatemark(global_State *g)
- {
- GCobj *o = gcref(g->gc.gray);
- int gct = o->gch.gct;
- lua_assert(isgray(o));
- gray2black(o);
- setgcrefr(g->gc.gray, o->gch.gclist);
- if (LJ_LIKELY(gct == ~LJ_TTAB)) {
- GCtab *t = gco2tab(o);
- if (gc_traverse_tab(g, t) > 0)
- black2gray(o);
- return sizeof(GCtab) + sizeof(TValue) * t->asize +
- sizeof(Node) * (t->hmask + 1);
- } else if (LJ_LIKELY(gct == ~LJ_TFUNC)) {
- GCfunc *fn = gco2func(o);
- gc_traverse_func(g, fn);
- return isluafunc(fn) ? sizeLfunc((MSize)fn->l.nupvalues) :
- sizeCfunc((MSize)fn->c.nupvalues);
- } else if (LJ_LIKELY(gct == ~LJ_TPROTO)) {
- GCproto *pt = gco2pt(o);
- gc_traverse_proto(g, pt);
- return pt->sizept;
- } else if (LJ_LIKELY(gct == ~LJ_TTHREAD)) {
- lua_State *th = gco2th(o);
- setgcrefr(th->gclist, g->gc.grayagain);
- setgcref(g->gc.grayagain, o);
- black2gray(o);
- gc_traverse_thread(g, th);
- return sizeof(lua_State) + sizeof(TValue) * th->stacksize;
- } else {
- #if LJ_HASJIT
- GCtrace *T = gco2trace(o);
- gc_traverse_trace(g, T);
- return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
- T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry);
- #else
- lua_assert(0);
- return 0;
- #endif
- }
- }
- static size_t gc_propagate_gray(global_State *g)
- {
- size_t m = 0;
- while (gcref(g->gc.gray) != NULL)
- m += propagatemark(g);
- return m;
- }
- typedef void (LJ_FASTCALL *GCFreeFunc)(global_State *g, GCobj *o);
- static const GCFreeFunc gc_freefunc[] = {
- (GCFreeFunc)lj_str_free,
- (GCFreeFunc)lj_func_freeuv,
- (GCFreeFunc)lj_state_free,
- (GCFreeFunc)lj_func_freeproto,
- (GCFreeFunc)lj_func_free,
- #if LJ_HASJIT
- (GCFreeFunc)lj_trace_free,
- #else
- (GCFreeFunc)0,
- #endif
- #if LJ_HASFFI
- (GCFreeFunc)lj_cdata_free,
- #else
- (GCFreeFunc)0,
- #endif
- (GCFreeFunc)lj_tab_free,
- (GCFreeFunc)lj_udata_free
- };
- #define gc_fullsweep(g, p) gc_sweep(g, (p), ~(uint32_t)0)
- static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
- {
-
- int ow = otherwhite(g);
- GCobj *o;
- while ((o = gcref(*p)) != NULL && lim-- > 0) {
- if (o->gch.gct == ~LJ_TTHREAD)
- gc_fullsweep(g, &gco2th(o)->openupval);
- if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) {
- lua_assert(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED));
- makewhite(g, o);
- p = &o->gch.nextgc;
- } else {
- lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED);
- setgcrefr(*p, o->gch.nextgc);
- if (o == gcref(g->gc.root))
- setgcrefr(g->gc.root, o->gch.nextgc);
- gc_freefunc[o->gch.gct - ~LJ_TSTR](g, o);
- }
- }
- return p;
- }
- static int gc_mayclear(cTValue *o, int val)
- {
- if (tvisgcv(o)) {
- if (tvisstr(o)) {
- gc_mark_str(strV(o));
- return 0;
- }
- if (iswhite(gcV(o)))
- return 1;
- if (tvisudata(o) && val && isfinalized(udataV(o)))
- return 1;
- }
- return 0;
- }
- static void gc_clearweak(GCobj *o)
- {
- while (o) {
- GCtab *t = gco2tab(o);
- lua_assert((t->marked & LJ_GC_WEAK));
- if ((t->marked & LJ_GC_WEAKVAL)) {
- MSize i, asize = t->asize;
- for (i = 0; i < asize; i++) {
-
- TValue *tv = arrayslot(t, i);
- if (gc_mayclear(tv, 1))
- setnilV(tv);
- }
- }
- if (t->hmask > 0) {
- Node *node = noderef(t->node);
- MSize i, hmask = t->hmask;
- for (i = 0; i <= hmask; i++) {
- Node *n = &node[i];
-
- if (!tvisnil(&n->val) && (gc_mayclear(&n->key, 0) ||
- gc_mayclear(&n->val, 1)))
- setnilV(&n->val);
- }
- }
- o = gcref(t->gclist);
- }
- }
- static void gc_call_finalizer(global_State *g, lua_State *L,
- cTValue *mo, GCobj *o)
- {
-
- uint8_t oldh = hook_save(g);
- GCSize oldt = g->gc.threshold;
- int errcode;
- TValue *top;
- lj_trace_abort(g);
- hook_entergc(g);
- g->gc.threshold = LJ_MAX_MEM;
- top = L->top;
- copyTV(L, top++, mo);
- if (LJ_FR2) setnilV(top++);
- setgcV(L, top, o, ~o->gch.gct);
- L->top = top+1;
- errcode = lj_vm_pcall(L, top, 1+0, -1);
- hook_restore(g, oldh);
- g->gc.threshold = oldt;
- if (errcode)
- lj_err_throw(L, errcode);
- }
- static void gc_finalize(lua_State *L)
- {
- global_State *g = G(L);
- GCobj *o = gcnext(gcref(g->gc.mmudata));
- cTValue *mo;
- lua_assert(tvref(g->jit_base) == NULL);
-
- if (o == gcref(g->gc.mmudata))
- setgcrefnull(g->gc.mmudata);
- else
- setgcrefr(gcref(g->gc.mmudata)->gch.nextgc, o->gch.nextgc);
- #if LJ_HASFFI
- if (o->gch.gct == ~LJ_TCDATA) {
- TValue tmp, *tv;
-
- setgcrefr(o->gch.nextgc, g->gc.root);
- setgcref(g->gc.root, o);
- makewhite(g, o);
- o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
-
- setcdataV(L, &tmp, gco2cd(o));
- tv = lj_tab_set(L, ctype_ctsG(g)->finalizer, &tmp);
- if (!tvisnil(tv)) {
- g->gc.nocdatafin = 0;
- copyTV(L, &tmp, tv);
- setnilV(tv);
- gc_call_finalizer(g, L, &tmp, o);
- }
- return;
- }
- #endif
-
- setgcrefr(o->gch.nextgc, mainthread(g)->nextgc);
- setgcref(mainthread(g)->nextgc, o);
- makewhite(g, o);
-
- mo = lj_meta_fastg(g, tabref(gco2ud(o)->metatable), MM_gc);
- if (mo)
- gc_call_finalizer(g, L, mo, o);
- }
- void lj_gc_finalize_udata(lua_State *L)
- {
- while (gcref(G(L)->gc.mmudata) != NULL)
- gc_finalize(L);
- }
- #if LJ_HASFFI
- void lj_gc_finalize_cdata(lua_State *L)
- {
- global_State *g = G(L);
- CTState *cts = ctype_ctsG(g);
- if (cts) {
- GCtab *t = cts->finalizer;
- Node *node = noderef(t->node);
- ptrdiff_t i;
- setgcrefnull(t->metatable);
- for (i = (ptrdiff_t)t->hmask; i >= 0; i--)
- if (!tvisnil(&node[i].val) && tviscdata(&node[i].key)) {
- GCobj *o = gcV(&node[i].key);
- TValue tmp;
- makewhite(g, o);
- o->gch.marked &= (uint8_t)~LJ_GC_CDATA_FIN;
- copyTV(L, &tmp, &node[i].val);
- setnilV(&node[i].val);
- gc_call_finalizer(g, L, &tmp, o);
- }
- }
- }
- #endif
- void lj_gc_freeall(global_State *g)
- {
- MSize i, strmask;
-
- g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED;
- gc_fullsweep(g, &g->gc.root);
- strmask = g->strmask;
- for (i = 0; i <= strmask; i++)
- gc_fullsweep(g, &g->strhash[i]);
- }
- static void atomic(global_State *g, lua_State *L)
- {
- size_t udsize;
- gc_mark_uv(g);
- gc_propagate_gray(g);
- setgcrefr(g->gc.gray, g->gc.weak);
- setgcrefnull(g->gc.weak);
- lua_assert(!iswhite(obj2gco(mainthread(g))));
- gc_markobj(g, L);
- gc_traverse_curtrace(g);
- gc_mark_gcroot(g);
- gc_propagate_gray(g);
- setgcrefr(g->gc.gray, g->gc.grayagain);
- setgcrefnull(g->gc.grayagain);
- gc_propagate_gray(g);
- udsize = lj_gc_separateudata(g, 0);
- gc_mark_mmudata(g);
- udsize += gc_propagate_gray(g);
-
- gc_clearweak(gcref(g->gc.weak));
- lj_buf_shrink(L, &g->tmpbuf);
-
- g->gc.currentwhite = (uint8_t)otherwhite(g);
- g->strempty.marked = g->gc.currentwhite;
- setmref(g->gc.sweep, &g->gc.root);
- g->gc.estimate = g->gc.total - (GCSize)udsize;
- }
- static size_t gc_onestep(lua_State *L)
- {
- global_State *g = G(L);
- switch (g->gc.state) {
- case GCSpause:
- gc_mark_start(g);
- return 0;
- case GCSpropagate:
- if (gcref(g->gc.gray) != NULL)
- return propagatemark(g);
- g->gc.state = GCSatomic;
- return 0;
- case GCSatomic:
- if (tvref(g->jit_base))
- return LJ_MAX_MEM;
- atomic(g, L);
- g->gc.state = GCSsweepstring;
- g->gc.sweepstr = 0;
- return 0;
- case GCSsweepstring: {
- GCSize old = g->gc.total;
- gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]);
- if (g->gc.sweepstr > g->strmask)
- g->gc.state = GCSsweep;
- lua_assert(old >= g->gc.total);
- g->gc.estimate -= old - g->gc.total;
- return GCSWEEPCOST;
- }
- case GCSsweep: {
- GCSize old = g->gc.total;
- setmref(g->gc.sweep, gc_sweep(g, mref(g->gc.sweep, GCRef), GCSWEEPMAX));
- lua_assert(old >= g->gc.total);
- g->gc.estimate -= old - g->gc.total;
- if (gcref(*mref(g->gc.sweep, GCRef)) == NULL) {
- if (g->strnum <= (g->strmask >> 2) && g->strmask > LJ_MIN_STRTAB*2-1)
- lj_str_resize(L, g->strmask >> 1);
- if (gcref(g->gc.mmudata)) {
- g->gc.state = GCSfinalize;
- #if LJ_HASFFI
- g->gc.nocdatafin = 1;
- #endif
- } else {
- g->gc.state = GCSpause;
- g->gc.debt = 0;
- }
- }
- return GCSWEEPMAX*GCSWEEPCOST;
- }
- case GCSfinalize:
- if (gcref(g->gc.mmudata) != NULL) {
- if (tvref(g->jit_base))
- return LJ_MAX_MEM;
- gc_finalize(L);
- if (g->gc.estimate > GCFINALIZECOST)
- g->gc.estimate -= GCFINALIZECOST;
- return GCFINALIZECOST;
- }
- #if LJ_HASFFI
- if (!g->gc.nocdatafin) lj_tab_rehash(L, ctype_ctsG(g)->finalizer);
- #endif
- g->gc.state = GCSpause;
- g->gc.debt = 0;
- return 0;
- default:
- lua_assert(0);
- return 0;
- }
- }
- int LJ_FASTCALL lj_gc_step(lua_State *L)
- {
- global_State *g = G(L);
- GCSize lim;
- int32_t ostate = g->vmstate;
- setvmstate(g, GC);
- lim = (GCSTEPSIZE/100) * g->gc.stepmul;
- if (lim == 0)
- lim = LJ_MAX_MEM;
- if (g->gc.total > g->gc.threshold)
- g->gc.debt += g->gc.total - g->gc.threshold;
- do {
- lim -= (GCSize)gc_onestep(L);
- if (g->gc.state == GCSpause) {
- g->gc.threshold = (g->gc.estimate/100) * g->gc.pause;
- g->vmstate = ostate;
- return 1;
- }
- } while (sizeof(lim) == 8 ? ((int64_t)lim > 0) : ((int32_t)lim > 0));
- if (g->gc.debt < GCSTEPSIZE) {
- g->gc.threshold = g->gc.total + GCSTEPSIZE;
- g->vmstate = ostate;
- return -1;
- } else {
- g->gc.debt -= GCSTEPSIZE;
- g->gc.threshold = g->gc.total;
- g->vmstate = ostate;
- return 0;
- }
- }
- void LJ_FASTCALL lj_gc_step_fixtop(lua_State *L)
- {
- if (curr_funcisL(L)) L->top = curr_topL(L);
- lj_gc_step(L);
- }
- #if LJ_HASJIT
- int LJ_FASTCALL lj_gc_step_jit(global_State *g, MSize steps)
- {
- lua_State *L = gco2th(gcref(g->cur_L));
- L->base = tvref(G(L)->jit_base);
- L->top = curr_topL(L);
- while (steps-- > 0 && lj_gc_step(L) == 0)
- ;
-
- return (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize);
- }
- #endif
- void lj_gc_fullgc(lua_State *L)
- {
- global_State *g = G(L);
- int32_t ostate = g->vmstate;
- setvmstate(g, GC);
- if (g->gc.state <= GCSatomic) {
- setmref(g->gc.sweep, &g->gc.root);
- setgcrefnull(g->gc.gray);
- setgcrefnull(g->gc.grayagain);
- setgcrefnull(g->gc.weak);
- g->gc.state = GCSsweepstring;
- g->gc.sweepstr = 0;
- }
- while (g->gc.state == GCSsweepstring || g->gc.state == GCSsweep)
- gc_onestep(L);
- lua_assert(g->gc.state == GCSfinalize || g->gc.state == GCSpause);
-
- g->gc.state = GCSpause;
- do { gc_onestep(L); } while (g->gc.state != GCSpause);
- g->gc.threshold = (g->gc.estimate/100) * g->gc.pause;
- g->vmstate = ostate;
- }
- void lj_gc_barrierf(global_State *g, GCobj *o, GCobj *v)
- {
- lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o));
- lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
- lua_assert(o->gch.gct != ~LJ_TTAB);
-
- if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
- gc_mark(g, v);
- else
- makewhite(g, o);
- }
- void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv)
- {
- #define TV2MARKED(x) \
- (*((uint8_t *)(x) - offsetof(GCupval, tv) + offsetof(GCupval, marked)))
- if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
- gc_mark(g, gcV(tv));
- else
- TV2MARKED(tv) = (TV2MARKED(tv) & (uint8_t)~LJ_GC_COLORS) | curwhite(g);
- #undef TV2MARKED
- }
- void lj_gc_closeuv(global_State *g, GCupval *uv)
- {
- GCobj *o = obj2gco(uv);
-
- copyTV(mainthread(g), &uv->tv, uvval(uv));
- setmref(uv->v, &uv->tv);
- uv->closed = 1;
- setgcrefr(o->gch.nextgc, g->gc.root);
- setgcref(g->gc.root, o);
- if (isgray(o)) {
- if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) {
- gray2black(o);
- if (tviswhite(&uv->tv))
- lj_gc_barrierf(g, o, gcV(&uv->tv));
- } else {
- makewhite(g, o);
- lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
- }
- }
- }
- #if LJ_HASJIT
- void lj_gc_barriertrace(global_State *g, uint32_t traceno)
- {
- if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
- gc_marktrace(g, traceno);
- }
- #endif
- void *lj_mem_realloc(lua_State *L, void *p, GCSize osz, GCSize nsz)
- {
- global_State *g = G(L);
- lua_assert((osz == 0) == (p == NULL));
- p = g->allocf(g->allocd, p, osz, nsz);
- if (p == NULL && nsz > 0)
- lj_err_mem(L);
- lua_assert((nsz == 0) == (p == NULL));
- lua_assert(checkptrGC(p));
- g->gc.total = (g->gc.total - osz) + nsz;
- return p;
- }
- void * LJ_FASTCALL lj_mem_newgco(lua_State *L, GCSize size)
- {
- global_State *g = G(L);
- GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size);
- if (o == NULL)
- lj_err_mem(L);
- lua_assert(checkptrGC(o));
- g->gc.total += size;
- setgcrefr(o->gch.nextgc, g->gc.root);
- setgcref(g->gc.root, o);
- newwhite(g, o);
- return o;
- }
- void *lj_mem_grow(lua_State *L, void *p, MSize *szp, MSize lim, MSize esz)
- {
- MSize sz = (*szp) << 1;
- if (sz < LJ_MIN_VECSZ)
- sz = LJ_MIN_VECSZ;
- if (sz > lim)
- sz = lim;
- p = lj_mem_realloc(L, p, (*szp)*esz, sz*esz);
- *szp = sz;
- return p;
- }
One Level Up
Top Level