src/lj_obj.h - luajit-2.0-src

Global variables defined

Data types defined

Functions defined

Macros defined

Source code

  1. /*
  2. ** LuaJIT VM tags, values and objects.
  3. ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
  4. **
  5. ** Portions taken verbatim or adapted from the Lua interpreter.
  6. ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
  7. */

  8. #ifndef _LJ_OBJ_H
  9. #define _LJ_OBJ_H

  10. #include "lua.h"
  11. #include "lj_def.h"
  12. #include "lj_arch.h"

  13. /* -- Memory references (32 bit address space) ---------------------------- */

  14. /* Memory and GC object sizes. */
  15. typedef uint32_t MSize;
  16. #if LJ_GC64
  17. typedef uint64_t GCSize;
  18. #else
  19. typedef uint32_t GCSize;
  20. #endif

  21. /* Memory reference */
  22. typedef struct MRef {
  23. #if LJ_GC64
  24.   uint64_t ptr64;        /* True 64 bit pointer. */
  25. #else
  26.   uint32_t ptr32;        /* Pseudo 32 bit pointer. */
  27. #endif
  28. } MRef;

  29. #if LJ_GC64
  30. #define mref(r, t)        ((t *)(void *)(r).ptr64)

  31. #define setmref(r, p)        ((r).ptr64 = (uint64_t)(void *)(p))
  32. #define setmrefr(r, v)        ((r).ptr64 = (v).ptr64)
  33. #else
  34. #define mref(r, t)        ((t *)(void *)(uintptr_t)(r).ptr32)

  35. #define setmref(r, p)        ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
  36. #define setmrefr(r, v)        ((r).ptr32 = (v).ptr32)
  37. #endif

  38. /* -- GC object references (32 bit address space) ------------------------- */

  39. /* GCobj reference */
  40. typedef struct GCRef {
  41. #if LJ_GC64
  42.   uint64_t gcptr64;        /* True 64 bit pointer. */
  43. #else
  44.   uint32_t gcptr32;        /* Pseudo 32 bit pointer. */
  45. #endif
  46. } GCRef;

  47. /* Common GC header for all collectable objects. */
  48. #define GCHeader        GCRef nextgc; uint8_t marked; uint8_t gct
  49. /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */

  50. #if LJ_GC64
  51. #define gcref(r)        ((GCobj *)(r).gcptr64)
  52. #define gcrefp(r, t)        ((t *)(void *)(r).gcptr64)
  53. #define gcrefu(r)        ((r).gcptr64)
  54. #define gcrefeq(r1, r2)        ((r1).gcptr64 == (r2).gcptr64)

  55. #define setgcref(r, gc)        ((r).gcptr64 = (uint64_t)&(gc)->gch)
  56. #define setgcreft(r, gc, it) \
  57.   (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
  58. #define setgcrefp(r, p)        ((r).gcptr64 = (uint64_t)(p))
  59. #define setgcrefnull(r)        ((r).gcptr64 = 0)
  60. #define setgcrefr(r, v)        ((r).gcptr64 = (v).gcptr64)
  61. #else
  62. #define gcref(r)        ((GCobj *)(uintptr_t)(r).gcptr32)
  63. #define gcrefp(r, t)        ((t *)(void *)(uintptr_t)(r).gcptr32)
  64. #define gcrefu(r)        ((r).gcptr32)
  65. #define gcrefeq(r1, r2)        ((r1).gcptr32 == (r2).gcptr32)

  66. #define setgcref(r, gc)        ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
  67. #define setgcrefp(r, p)        ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
  68. #define setgcrefnull(r)        ((r).gcptr32 = 0)
  69. #define setgcrefr(r, v)        ((r).gcptr32 = (v).gcptr32)
  70. #endif

  71. #define gcnext(gc)        (gcref((gc)->gch.nextgc))

  72. /* IMPORTANT NOTE:
  73. **
  74. ** All uses of the setgcref* macros MUST be accompanied with a write barrier.
  75. **
  76. ** This is to ensure the integrity of the incremental GC. The invariant
  77. ** to preserve is that a black object never points to a white object.
  78. ** I.e. never store a white object into a field of a black object.
  79. **
  80. ** It's ok to LEAVE OUT the write barrier ONLY in the following cases:
  81. ** - The source is not a GC object (NULL).
  82. ** - The target is a GC root. I.e. everything in global_State.
  83. ** - The target is a lua_State field (threads are never black).
  84. ** - The target is a stack slot, see setgcV et al.
  85. ** - The target is an open upvalue, i.e. pointing to a stack slot.
  86. ** - The target is a newly created object (i.e. marked white). But make
  87. **   sure nothing invokes the GC inbetween.
  88. ** - The target and the source are the same object (self-reference).
  89. ** - The target already contains the object (e.g. moving elements around).
  90. **
  91. ** The most common case is a store to a stack slot. All other cases where
  92. ** a barrier has been omitted are annotated with a NOBARRIER comment.
  93. **
  94. ** The same logic applies for stores to table slots (array part or hash
  95. ** part). ALL uses of lj_tab_set* require a barrier for the stored value
  96. ** *and* the stored key, based on the above rules. In practice this means
  97. ** a barrier is needed if *either* of the key or value are a GC object.
  98. **
  99. ** It's ok to LEAVE OUT the write barrier in the following special cases:
  100. ** - The stored value is nil. The key doesn't matter because it's either
  101. **   not resurrected or lj_tab_newkey() will take care of the key barrier.
  102. ** - The key doesn't matter if the *previously* stored value is guaranteed
  103. **   to be non-nil (because the key is kept alive in the table).
  104. ** - The key doesn't matter if it's guaranteed not to be part of the table,
  105. **   since lj_tab_newkey() takes care of the key barrier. This applies
  106. **   trivially to new tables, but watch out for resurrected keys. Storing
  107. **   a nil value leaves the key in the table!
  108. **
  109. ** In case of doubt use lj_gc_anybarriert() as it's rather cheap. It's used
  110. ** by the interpreter for all table stores.
  111. **
  112. ** Note: In contrast to Lua's GC, LuaJIT's GC does *not* specially mark
  113. ** dead keys in tables. The reference is left in, but it's guaranteed to
  114. ** be never dereferenced as long as the value is nil. It's ok if the key is
  115. ** freed or if any object subsequently gets the same address.
  116. **
  117. ** Not destroying dead keys helps to keep key hash slots stable. This avoids
  118. ** specialization back-off for HREFK when a value flips between nil and
  119. ** non-nil and the GC gets in the way. It also allows safely hoisting
  120. ** HREF/HREFK across GC steps. Dead keys are only removed if a table is
  121. ** resized (i.e. by NEWREF) and xREF must not be CSEd across a resize.
  122. **
  123. ** The trade-off is that a write barrier for tables must take the key into
  124. ** account, too. Implicitly resurrecting the key by storing a non-nil value
  125. ** may invalidate the incremental GC invariant.
  126. */

  127. /* -- Common type definitions --------------------------------------------- */

  128. /* Types for handling bytecodes. Need this here, details in lj_bc.h. */
  129. typedef uint32_t BCIns/* Bytecode instruction. */
  130. typedef uint32_t BCPos/* Bytecode position. */
  131. typedef uint32_t BCReg/* Bytecode register. */
  132. typedef int32_t BCLine/* Bytecode line number. */

  133. /* Internal assembler functions. Never call these directly from C. */
  134. typedef void (*ASMFunction)(void);

  135. /* Resizable string buffer. Need this here, details in lj_buf.h. */
  136. typedef struct SBuf {
  137.   MRef p;                /* String buffer pointer. */
  138.   MRef e;                /* String buffer end pointer. */
  139.   MRef b;                /* String buffer base. */
  140.   MRef L;                /* lua_State, used for buffer resizing. */
  141. } SBuf;

  142. /* -- Tags and values ----------------------------------------------------- */

  143. /* Frame link. */
  144. typedef union {
  145.   int32_t ftsz;                /* Frame type and size of previous frame. */
  146.   MRef pcr;                /* Or PC for Lua frames. */
  147. } FrameLink;

  148. /* Tagged value. */
  149. typedef LJ_ALIGN(8) union TValue {
  150.   uint64_t u64;                /* 64 bit pattern overlaps number. */
  151.   lua_Number n;                /* Number object overlaps split tag/value object. */
  152. #if LJ_GC64
  153.   GCRef gcr;                /* GCobj reference with tag. */
  154.   int64_t it64;
  155.   struct {
  156.     LJ_ENDIAN_LOHI(
  157.       int32_t i;        /* Integer value. */
  158.     , uint32_t it;        /* Internal object tag. Must overlap MSW of number. */
  159.     )
  160.   };
  161. #else
  162.   struct {
  163.     LJ_ENDIAN_LOHI(
  164.       union {
  165.         GCRef gcr;        /* GCobj reference (if any). */
  166.         int32_t i;        /* Integer value. */
  167.       };
  168.     , uint32_t it;        /* Internal object tag. Must overlap MSW of number. */
  169.     )
  170.   };
  171. #endif
  172. #if LJ_FR2
  173.   int64_t ftsz;                /* Frame type and size of previous frame, or PC. */
  174. #else
  175.   struct {
  176.     LJ_ENDIAN_LOHI(
  177.       GCRef func;        /* Function for next frame (or dummy L). */
  178.     , FrameLink tp;        /* Link to previous frame. */
  179.     )
  180.   } fr;
  181. #endif
  182.   struct {
  183.     LJ_ENDIAN_LOHI(
  184.       uint32_t lo;        /* Lower 32 bits of number. */
  185.     , uint32_t hi;        /* Upper 32 bits of number. */
  186.     )
  187.   } u32;
  188. } TValue;

  189. typedef const TValue cTValue;

  190. #define tvref(r)        (mref(r, TValue))

  191. /* More external and GCobj tags for internal objects. */
  192. #define LAST_TT                LUA_TTHREAD
  193. #define LUA_TPROTO        (LAST_TT+1)
  194. #define LUA_TCDATA        (LAST_TT+2)

  195. /* Internal object tags.
  196. **
  197. ** Format for 32 bit GC references (!LJ_GC64):
  198. **
  199. ** Internal tags overlap the MSW of a number object (must be a double).
  200. ** Interpreted as a double these are special NaNs. The FPU only generates
  201. ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
  202. ** for use as internal tags. Small negative numbers are used to shorten the
  203. ** encoding of type comparisons (reg/mem against sign-ext. 8 bit immediate).
  204. **
  205. **                  ---MSW---.---LSW---
  206. ** primitive types |  itype  |         |
  207. ** lightuserdata   |  itype  |  void * |  (32 bit platforms)
  208. ** lightuserdata   |ffff|    void *    |  (64 bit platforms, 47 bit pointers)
  209. ** GC objects      |  itype  |  GCRef  |
  210. ** int (LJ_DUALNUM)|  itype  |   int   |
  211. ** number           -------double------
  212. **
  213. ** Format for 64 bit GC references (LJ_GC64):
  214. **
  215. ** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
  216. ** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
  217. ** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
  218. **
  219. **                     ------MSW------.------LSW------
  220. ** primitive types    |1..1|itype|1..................1|
  221. ** GC objects/lightud |1..1|itype|-------GCRef--------|
  222. ** int (LJ_DUALNUM)   |1..1|itype|0..0|-----int-------|
  223. ** number              ------------double-------------
  224. **
  225. ** ORDER LJ_T
  226. ** Primitive types nil/false/true must be first, lightuserdata next.
  227. ** GC objects are at the end, table/userdata must be lowest.
  228. ** Also check lj_ir.h for similar ordering constraints.
  229. */
  230. #define LJ_TNIL                        (~0u)
  231. #define LJ_TFALSE                (~1u)
  232. #define LJ_TTRUE                (~2u)
  233. #define LJ_TLIGHTUD                (~3u)
  234. #define LJ_TSTR                        (~4u)
  235. #define LJ_TUPVAL                (~5u)
  236. #define LJ_TTHREAD                (~6u)
  237. #define LJ_TPROTO                (~7u)
  238. #define LJ_TFUNC                (~8u)
  239. #define LJ_TTRACE                (~9u)
  240. #define LJ_TCDATA                (~10u)
  241. #define LJ_TTAB                        (~11u)
  242. #define LJ_TUDATA                (~12u)
  243. /* This is just the canonical number type used in some places. */
  244. #define LJ_TNUMX                (~13u)

  245. /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
  246. #if LJ_64 && !LJ_GC64
  247. #define LJ_TISNUM                0xfffeffffu
  248. #else
  249. #define LJ_TISNUM                LJ_TNUMX
  250. #endif
  251. #define LJ_TISTRUECOND                LJ_TFALSE
  252. #define LJ_TISPRI                LJ_TTRUE
  253. #define LJ_TISGCV                (LJ_TSTR+1)
  254. #define LJ_TISTABUD                LJ_TTAB

  255. #if LJ_GC64
  256. #define LJ_GCVMASK                (((uint64_t)1 << 47) - 1)
  257. #endif

  258. /* -- String object ------------------------------------------------------- */

  259. /* String object header. String payload follows. */
  260. typedef struct GCstr {
  261.   GCHeader;
  262.   uint8_t reserved;        /* Used by lexer for fast lookup of reserved words. */
  263.   uint8_t unused;
  264.   MSize hash;                /* Hash of string. */
  265.   MSize len;                /* Size of string. */
  266. } GCstr;

  267. #define strref(r)        (&gcref((r))->str)
  268. #define strdata(s)        ((const char *)((s)+1))
  269. #define strdatawr(s)        ((char *)((s)+1))
  270. #define strVdata(o)        strdata(strV(o))
  271. #define sizestring(s)        (sizeof(struct GCstr)+(s)->len+1)

  272. /* -- Userdata object ----------------------------------------------------- */

  273. /* Userdata object. Payload follows. */
  274. typedef struct GCudata {
  275.   GCHeader;
  276.   uint8_t udtype;        /* Userdata type. */
  277.   uint8_t unused2;
  278.   GCRef env;                /* Should be at same offset in GCfunc. */
  279.   MSize len;                /* Size of payload. */
  280.   GCRef metatable;        /* Must be at same offset in GCtab. */
  281.   uint32_t align1;        /* To force 8 byte alignment of the payload. */
  282. } GCudata;

  283. /* Userdata types. */
  284. enum {
  285.   UDTYPE_USERDATA,        /* Regular userdata. */
  286.   UDTYPE_IO_FILE,        /* I/O library FILE. */
  287.   UDTYPE_FFI_CLIB,        /* FFI C library namespace. */
  288.   UDTYPE__MAX
  289. };

  290. #define uddata(u)        ((void *)((u)+1))
  291. #define sizeudata(u)        (sizeof(struct GCudata)+(u)->len)

  292. /* -- C data object ------------------------------------------------------- */

  293. /* C data object. Payload follows. */
  294. typedef struct GCcdata {
  295.   GCHeader;
  296.   uint16_t ctypeid;        /* C type ID. */
  297. } GCcdata;

  298. /* Prepended to variable-sized or realigned C data objects. */
  299. typedef struct GCcdataVar {
  300.   uint16_t offset;        /* Offset to allocated memory (relative to GCcdata). */
  301.   uint16_t extra;        /* Extra space allocated (incl. GCcdata + GCcdatav). */
  302.   MSize len;                /* Size of payload. */
  303. } GCcdataVar;

  304. #define cdataptr(cd)        ((void *)((cd)+1))
  305. #define cdataisv(cd)        ((cd)->marked & 0x80)
  306. #define cdatav(cd)        ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
  307. #define cdatavlen(cd)        check_exp(cdataisv(cd), cdatav(cd)->len)
  308. #define sizecdatav(cd)        (cdatavlen(cd) + cdatav(cd)->extra)
  309. #define memcdatav(cd)        ((void *)((char *)(cd) - cdatav(cd)->offset))

  310. /* -- Prototype object ---------------------------------------------------- */

  311. #define SCALE_NUM_GCO        ((int32_t)sizeof(lua_Number)/sizeof(GCRef))
  312. #define round_nkgc(n)        (((n) + SCALE_NUM_GCO-1) & ~(SCALE_NUM_GCO-1))

  313. typedef struct GCproto {
  314.   GCHeader;
  315.   uint8_t numparams;        /* Number of parameters. */
  316.   uint8_t framesize;        /* Fixed frame size. */
  317.   MSize sizebc;                /* Number of bytecode instructions. */
  318. #if LJ_GC64
  319.   uint32_t unused_gc64;
  320. #endif
  321.   GCRef gclist;
  322.   MRef k;                /* Split constant array (points to the middle). */
  323.   MRef uv;                /* Upvalue list. local slot|0x8000 or parent uv idx. */
  324.   MSize sizekgc;        /* Number of collectable constants. */
  325.   MSize sizekn;                /* Number of lua_Number constants. */
  326.   MSize sizept;                /* Total size including colocated arrays. */
  327.   uint8_t sizeuv;        /* Number of upvalues. */
  328.   uint8_t flags;        /* Miscellaneous flags (see below). */
  329.   uint16_t trace;        /* Anchor for chain of root traces. */
  330.   /* ------ The following fields are for debugging/tracebacks only ------ */
  331.   GCRef chunkname;        /* Name of the chunk this function was defined in. */
  332.   BCLine firstline;        /* First line of the function definition. */
  333.   BCLine numline;        /* Number of lines for the function definition. */
  334.   MRef lineinfo;        /* Compressed map from bytecode ins. to source line. */
  335.   MRef uvinfo;                /* Upvalue names. */
  336.   MRef varinfo;                /* Names and compressed extents of local variables. */
  337. } GCproto;

  338. /* Flags for prototype. */
  339. #define PROTO_CHILD                0x01        /* Has child prototypes. */
  340. #define PROTO_VARARG                0x02        /* Vararg function. */
  341. #define PROTO_FFI                0x04        /* Uses BC_KCDATA for FFI datatypes. */
  342. #define PROTO_NOJIT                0x08        /* JIT disabled for this function. */
  343. #define PROTO_ILOOP                0x10        /* Patched bytecode with ILOOP etc. */
  344. /* Only used during parsing. */
  345. #define PROTO_HAS_RETURN        0x20        /* Already emitted a return. */
  346. #define PROTO_FIXUP_RETURN        0x40        /* Need to fixup emitted returns. */
  347. /* Top bits used for counting created closures. */
  348. #define PROTO_CLCOUNT                0x20        /* Base of saturating 3 bit counter. */
  349. #define PROTO_CLC_BITS                3
  350. #define PROTO_CLC_POLY                (3*PROTO_CLCOUNT/* Polymorphic threshold. */

  351. #define PROTO_UV_LOCAL                0x8000        /* Upvalue for local slot. */
  352. #define PROTO_UV_IMMUTABLE        0x4000        /* Immutable upvalue. */

  353. #define proto_kgc(pt, idx) \
  354.   check_exp((uintptr_t)(intptr_t)(idx) >= (uintptr_t)-(intptr_t)(pt)->sizekgc, \
  355.             gcref(mref((pt)->k, GCRef)[(idx)]))
  356. #define proto_knumtv(pt, idx) \
  357.   check_exp((uintptr_t)(idx) < (pt)->sizekn, &mref((pt)->k, TValue)[(idx)])
  358. #define proto_bc(pt)                ((BCIns *)((char *)(pt) + sizeof(GCproto)))
  359. #define proto_bcpos(pt, pc)        ((BCPos)((pc) - proto_bc(pt)))
  360. #define proto_uv(pt)                (mref((pt)->uv, uint16_t))

  361. #define proto_chunkname(pt)        (strref((pt)->chunkname))
  362. #define proto_chunknamestr(pt)        (strdata(proto_chunkname((pt))))
  363. #define proto_lineinfo(pt)        (mref((pt)->lineinfo, const void))
  364. #define proto_uvinfo(pt)        (mref((pt)->uvinfo, const uint8_t))
  365. #define proto_varinfo(pt)        (mref((pt)->varinfo, const uint8_t))

  366. /* -- Upvalue object ------------------------------------------------------ */

  367. typedef struct GCupval {
  368.   GCHeader;
  369.   uint8_t closed;        /* Set if closed (i.e. uv->v == &uv->u.value). */
  370.   uint8_t immutable;        /* Immutable value. */
  371.   union {
  372.     TValue tv;                /* If closed: the value itself. */
  373.     struct {                /* If open: double linked list, anchored at thread. */
  374.       GCRef prev;
  375.       GCRef next;
  376.     };
  377.   };
  378.   MRef v;                /* Points to stack slot (open) or above (closed). */
  379.   uint32_t dhash;        /* Disambiguation hash: dh1 != dh2 => cannot alias. */
  380. } GCupval;

  381. #define uvprev(uv_)        (&gcref((uv_)->prev)->uv)
  382. #define uvnext(uv_)        (&gcref((uv_)->next)->uv)
  383. #define uvval(uv_)        (mref((uv_)->v, TValue))

  384. /* -- Function object (closures) ------------------------------------------ */

  385. /* Common header for functions. env should be at same offset in GCudata. */
  386. #define GCfuncHeader \
  387.   GCHeader; uint8_t ffid; uint8_t nupvalues; \
  388.   GCRef env; GCRef gclist; MRef pc

  389. typedef struct GCfuncC {
  390.   GCfuncHeader;
  391.   lua_CFunction f;        /* C function to be called. */
  392.   TValue upvalue[1];        /* Array of upvalues (TValue). */
  393. } GCfuncC;

  394. typedef struct GCfuncL {
  395.   GCfuncHeader;
  396.   GCRef uvptr[1];        /* Array of _pointers_ to upvalue objects (GCupval). */
  397. } GCfuncL;

  398. typedef union GCfunc {
  399.   GCfuncC c;
  400.   GCfuncL l;
  401. } GCfunc;

  402. #define FF_LUA                0
  403. #define FF_C                1
  404. #define isluafunc(fn)        ((fn)->c.ffid == FF_LUA)
  405. #define iscfunc(fn)        ((fn)->c.ffid == FF_C)
  406. #define isffunc(fn)        ((fn)->c.ffid > FF_C)
  407. #define funcproto(fn) \
  408.   check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
  409. #define sizeCfunc(n)        (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
  410. #define sizeLfunc(n)        (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))

  411. /* -- Table object -------------------------------------------------------- */

  412. /* Hash node. */
  413. typedef struct Node {
  414.   TValue val;                /* Value object. Must be first field. */
  415.   TValue key;                /* Key object. */
  416.   MRef next;                /* Hash chain. */
  417. #if !LJ_GC64
  418.   MRef freetop;                /* Top of free elements (stored in t->node[0]). */
  419. #endif
  420. } Node;

  421. LJ_STATIC_ASSERT(offsetof(Node, val) == 0);

  422. typedef struct GCtab {
  423.   GCHeader;
  424.   uint8_t nomm;                /* Negative cache for fast metamethods. */
  425.   int8_t colo;                /* Array colocation. */
  426.   MRef array;                /* Array part. */
  427.   GCRef gclist;
  428.   GCRef metatable;        /* Must be at same offset in GCudata. */
  429.   MRef node;                /* Hash part. */
  430.   uint32_t asize;        /* Size of array part (keys [0, asize-1]). */
  431.   uint32_t hmask;        /* Hash part mask (size of hash part - 1). */
  432. #if LJ_GC64
  433.   MRef freetop;                /* Top of free elements. */
  434. #endif
  435. } GCtab;

  436. #define sizetabcolo(n)        ((n)*sizeof(TValue) + sizeof(GCtab))
  437. #define tabref(r)        (&gcref((r))->tab)
  438. #define noderef(r)        (mref((r), Node))
  439. #define nextnode(n)        (mref((n)->next, Node))
  440. #if LJ_GC64
  441. #define getfreetop(t, n)        (noderef((t)->freetop))
  442. #define setfreetop(t, n, v)        (setmref((t)->freetop, (v)))
  443. #else
  444. #define getfreetop(t, n)        (noderef((n)->freetop))
  445. #define setfreetop(t, n, v)        (setmref((n)->freetop, (v)))
  446. #endif

  447. /* -- State objects ------------------------------------------------------- */

  448. /* VM states. */
  449. enum {
  450.   LJ_VMST_INTERP,        /* Interpreter. */
  451.   LJ_VMST_C,                /* C function. */
  452.   LJ_VMST_GC,                /* Garbage collector. */
  453.   LJ_VMST_EXIT,                /* Trace exit handler. */
  454.   LJ_VMST_RECORD,        /* Trace recorder. */
  455.   LJ_VMST_OPT,                /* Optimizer. */
  456.   LJ_VMST_ASM,                /* Assembler. */
  457.   LJ_VMST__MAX
  458. };

  459. #define setvmstate(g, st)        ((g)->vmstate = ~LJ_VMST_##st)

  460. /* Metamethods. ORDER MM */
  461. #ifdef LJ_HASFFI
  462. #define MMDEF_FFI(_) _(new)
  463. #else
  464. #define MMDEF_FFI(_)
  465. #endif

  466. #if LJ_52 || LJ_HASFFI
  467. #define MMDEF_PAIRS(_) _(pairs) _(ipairs)
  468. #else
  469. #define MMDEF_PAIRS(_)
  470. #define MM_pairs        255
  471. #define MM_ipairs        255
  472. #endif

  473. #define MMDEF(_) \
  474.   _(index) _(newindex) _(gc) _(mode) _(eq) _(len) \
  475.   /* Only the above (fast) metamethods are negative cached (max. 8). */ \
  476.   _(lt) _(le) _(concat) _(call) \
  477.   /* The following must be in ORDER ARITH. */ \
  478.   _(add) _(sub) _(mul) _(div) _(mod) _(pow) _(unm) \
  479.   /* The following are used in the standard libraries. */ \
  480.   _(metatable) _(tostring) MMDEF_FFI(_) MMDEF_PAIRS(_)

  481. typedef enum {
  482. #define MMENUM(name)        MM_##name,
  483. MMDEF(MMENUM)
  484. #undef MMENUM
  485.   MM__MAX,
  486.   MM____ = MM__MAX,
  487.   MM_FAST = MM_len
  488. } MMS;

  489. /* GC root IDs. */
  490. typedef enum {
  491.   GCROOT_MMNAME,        /* Metamethod names. */
  492.   GCROOT_MMNAME_LAST = GCROOT_MMNAME + MM__MAX-1,
  493.   GCROOT_BASEMT,        /* Metatables for base types. */
  494.   GCROOT_BASEMT_NUM = GCROOT_BASEMT + ~LJ_TNUMX,
  495.   GCROOT_IO_INPUT,        /* Userdata for default I/O input file. */
  496.   GCROOT_IO_OUTPUT,        /* Userdata for default I/O output file. */
  497.   GCROOT_MAX
  498. } GCRootID;

  499. #define basemt_it(g, it)        ((g)->gcroot[GCROOT_BASEMT+~(it)])
  500. #define basemt_obj(g, o)        ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
  501. #define mmname_str(g, mm)        (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))

  502. typedef struct GCState {
  503.   GCSize total;                /* Memory currently allocated. */
  504.   GCSize threshold;        /* Memory threshold. */
  505.   uint8_t currentwhite;        /* Current white color. */
  506.   uint8_t state;        /* GC state. */
  507.   uint8_t nocdatafin;        /* No cdata finalizer called. */
  508.   uint8_t unused2;
  509.   MSize sweepstr;        /* Sweep position in string table. */
  510.   GCRef root;                /* List of all collectable objects. */
  511.   MRef sweep;                /* Sweep position in root list. */
  512.   GCRef gray;                /* List of gray objects. */
  513.   GCRef grayagain;        /* List of objects for atomic traversal. */
  514.   GCRef weak;                /* List of weak tables (to be cleared). */
  515.   GCRef mmudata;        /* List of userdata (to be finalized). */
  516.   GCSize debt;                /* Debt (how much GC is behind schedule). */
  517.   GCSize estimate;        /* Estimate of memory actually in use. */
  518.   MSize stepmul;        /* Incremental GC step granularity. */
  519.   MSize pause;                /* Pause between successive GC cycles. */
  520. } GCState;

  521. /* Global state, shared by all threads of a Lua universe. */
  522. typedef struct global_State {
  523.   GCRef *strhash;        /* String hash table (hash chain anchors). */
  524.   MSize strmask;        /* String hash mask (size of hash table - 1). */
  525.   MSize strnum;                /* Number of strings in hash table. */
  526.   lua_Alloc allocf;        /* Memory allocator. */
  527.   void *allocd;                /* Memory allocator data. */
  528.   GCState gc;                /* Garbage collector. */
  529.   volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
  530.   SBuf tmpbuf;                /* Temporary string buffer. */
  531.   GCstr strempty;        /* Empty string. */
  532.   uint8_t stremptyz;        /* Zero terminator of empty string. */
  533.   uint8_t hookmask;        /* Hook mask. */
  534.   uint8_t dispatchmode;        /* Dispatch mode. */
  535.   uint8_t vmevmask;        /* VM event mask. */
  536.   GCRef mainthref;        /* Link to main thread. */
  537.   TValue registrytv;        /* Anchor for registry. */
  538.   TValue tmptv, tmptv2;        /* Temporary TValues. */
  539.   Node nilnode;                /* Fallback 1-element hash part (nil key and value). */
  540.   GCupval uvhead;        /* Head of double-linked list of all open upvalues. */
  541.   int32_t hookcount;        /* Instruction hook countdown. */
  542.   int32_t hookcstart;        /* Start count for instruction hook counter. */
  543.   lua_Hook hookf;        /* Hook function. */
  544.   lua_CFunction wrapf;        /* Wrapper for C function calls. */
  545.   lua_CFunction panic;        /* Called as a last resort for errors. */
  546.   BCIns bc_cfunc_int;        /* Bytecode for internal C function calls. */
  547.   BCIns bc_cfunc_ext;        /* Bytecode for external C function calls. */
  548.   GCRef cur_L;                /* Currently executing lua_State. */
  549.   MRef jit_base;        /* Current JIT code L->base or NULL. */
  550.   MRef ctype_state;        /* Pointer to C type state. */
  551.   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
  552. } global_State;

  553. #define mainthread(g)        (&gcref(g->mainthref)->th)
  554. #define niltv(L) \
  555.   check_exp(tvisnil(&G(L)->nilnode.val), &G(L)->nilnode.val)
  556. #define niltvg(g) \
  557.   check_exp(tvisnil(&(g)->nilnode.val), &(g)->nilnode.val)

  558. /* Hook management. Hook event masks are defined in lua.h. */
  559. #define HOOK_EVENTMASK                0x0f
  560. #define HOOK_ACTIVE                0x10
  561. #define HOOK_ACTIVE_SHIFT        4
  562. #define HOOK_VMEVENT                0x20
  563. #define HOOK_GC                        0x40
  564. #define HOOK_PROFILE                0x80
  565. #define hook_active(g)                ((g)->hookmask & HOOK_ACTIVE)
  566. #define hook_enter(g)                ((g)->hookmask |= HOOK_ACTIVE)
  567. #define hook_entergc(g)                ((g)->hookmask |= (HOOK_ACTIVE|HOOK_GC))
  568. #define hook_vmevent(g)                ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
  569. #define hook_leave(g)                ((g)->hookmask &= ~HOOK_ACTIVE)
  570. #define hook_save(g)                ((g)->hookmask & ~HOOK_EVENTMASK)
  571. #define hook_restore(g, h) \
  572.   ((g)->hookmask = ((g)->hookmask & HOOK_EVENTMASK) | (h))

  573. /* Per-thread state object. */
  574. struct lua_State {
  575.   GCHeader;
  576.   uint8_t dummy_ffid;        /* Fake FF_C for curr_funcisL() on dummy frames. */
  577.   uint8_t status;        /* Thread status. */
  578.   MRef glref;                /* Link to global state. */
  579.   GCRef gclist;                /* GC chain. */
  580.   TValue *base;                /* Base of currently executing function. */
  581.   TValue *top;                /* First free slot in the stack. */
  582.   MRef maxstack;        /* Last free slot in the stack. */
  583.   MRef stack;                /* Stack base. */
  584.   GCRef openupval;        /* List of open upvalues in the stack. */
  585.   GCRef env;                /* Thread environment (table of globals). */
  586.   void *cframe;                /* End of C stack frame chain. */
  587.   MSize stacksize;        /* True stack size (incl. LJ_STACK_EXTRA). */
  588. };

  589. #define G(L)                        (mref(L->glref, global_State))
  590. #define registry(L)                (&G(L)->registrytv)

  591. /* Macros to access the currently executing (Lua) function. */
  592. #if LJ_GC64
  593. #define curr_func(L)                (&gcval(L->base-2)->fn)
  594. #elif LJ_FR2
  595. #define curr_func(L)                (&gcref((L->base-2)->gcr)->fn)
  596. #else
  597. #define curr_func(L)                (&gcref((L->base-1)->fr.func)->fn)
  598. #endif
  599. #define curr_funcisL(L)                (isluafunc(curr_func(L)))
  600. #define curr_proto(L)                (funcproto(curr_func(L)))
  601. #define curr_topL(L)                (L->base + curr_proto(L)->framesize)
  602. #define curr_top(L)                (curr_funcisL(L) ? curr_topL(L) : L->top)

  603. /* -- GC object definition and conversions -------------------------------- */

  604. /* GC header for generic access to common fields of GC objects. */
  605. typedef struct GChead {
  606.   GCHeader;
  607.   uint8_t unused1;
  608.   uint8_t unused2;
  609.   GCRef env;
  610.   GCRef gclist;
  611.   GCRef metatable;
  612. } GChead;

  613. /* The env field SHOULD be at the same offset for all GC objects. */
  614. LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCfuncL, env));
  615. LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCudata, env));

  616. /* The metatable field MUST be at the same offset for all GC objects. */
  617. LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCtab, metatable));
  618. LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));

  619. /* The gclist field MUST be at the same offset for all GC objects. */
  620. LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
  621. LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
  622. LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
  623. LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));

  624. typedef union GCobj {
  625.   GChead gch;
  626.   GCstr str;
  627.   GCupval uv;
  628.   lua_State th;
  629.   GCproto pt;
  630.   GCfunc fn;
  631.   GCcdata cd;
  632.   GCtab tab;
  633.   GCudata ud;
  634. } GCobj;

  635. /* Macros to convert a GCobj pointer into a specific value. */
  636. #define gco2str(o)        check_exp((o)->gch.gct == ~LJ_TSTR, &(o)->str)
  637. #define gco2uv(o)        check_exp((o)->gch.gct == ~LJ_TUPVAL, &(o)->uv)
  638. #define gco2th(o)        check_exp((o)->gch.gct == ~LJ_TTHREAD, &(o)->th)
  639. #define gco2pt(o)        check_exp((o)->gch.gct == ~LJ_TPROTO, &(o)->pt)
  640. #define gco2func(o)        check_exp((o)->gch.gct == ~LJ_TFUNC, &(o)->fn)
  641. #define gco2cd(o)        check_exp((o)->gch.gct == ~LJ_TCDATA, &(o)->cd)
  642. #define gco2tab(o)        check_exp((o)->gch.gct == ~LJ_TTAB, &(o)->tab)
  643. #define gco2ud(o)        check_exp((o)->gch.gct == ~LJ_TUDATA, &(o)->ud)

  644. /* Macro to convert any collectable object into a GCobj pointer. */
  645. #define obj2gco(v)        ((GCobj *)(v))

  646. /* -- TValue getters/setters ---------------------------------------------- */

  647. #ifdef LUA_USE_ASSERT
  648. #include "lj_gc.h"
  649. #endif

  650. /* Macros to test types. */
  651. #if LJ_GC64
  652. #define itype(o)        ((uint32_t)((o)->it64 >> 47))
  653. #define tvisnil(o)        ((o)->it64 == -1)
  654. #else
  655. #define itype(o)        ((o)->it)
  656. #define tvisnil(o)        (itype(o) == LJ_TNIL)
  657. #endif
  658. #define tvisfalse(o)        (itype(o) == LJ_TFALSE)
  659. #define tvistrue(o)        (itype(o) == LJ_TTRUE)
  660. #define tvisbool(o)        (tvisfalse(o) || tvistrue(o))
  661. #if LJ_64 && !LJ_GC64
  662. #define tvislightud(o)        (((int32_t)itype(o) >> 15) == -2)
  663. #else
  664. #define tvislightud(o)        (itype(o) == LJ_TLIGHTUD)
  665. #endif
  666. #define tvisstr(o)        (itype(o) == LJ_TSTR)
  667. #define tvisfunc(o)        (itype(o) == LJ_TFUNC)
  668. #define tvisthread(o)        (itype(o) == LJ_TTHREAD)
  669. #define tvisproto(o)        (itype(o) == LJ_TPROTO)
  670. #define tviscdata(o)        (itype(o) == LJ_TCDATA)
  671. #define tvistab(o)        (itype(o) == LJ_TTAB)
  672. #define tvisudata(o)        (itype(o) == LJ_TUDATA)
  673. #define tvisnumber(o)        (itype(o) <= LJ_TISNUM)
  674. #define tvisint(o)        (LJ_DUALNUM && itype(o) == LJ_TISNUM)
  675. #define tvisnum(o)        (itype(o) < LJ_TISNUM)

  676. #define tvistruecond(o)        (itype(o) < LJ_TISTRUECOND)
  677. #define tvispri(o)        (itype(o) >= LJ_TISPRI)
  678. #define tvistabud(o)        (itype(o) <= LJ_TISTABUD/* && !tvisnum() */
  679. #define tvisgcv(o)        ((itype(o) - LJ_TISGCV) > (LJ_TNUMX - LJ_TISGCV))

  680. /* Special macros to test numbers for NaN, +0, -0, +1 and raw equality. */
  681. #define tvisnan(o)        ((o)->n != (o)->n)
  682. #if LJ_64
  683. #define tviszero(o)        (((o)->u64 << 1) == 0)
  684. #else
  685. #define tviszero(o)        (((o)->u32.lo | ((o)->u32.hi << 1)) == 0)
  686. #endif
  687. #define tvispzero(o)        ((o)->u64 == 0)
  688. #define tvismzero(o)        ((o)->u64 == U64x(80000000,00000000))
  689. #define tvispone(o)        ((o)->u64 == U64x(3ff00000,00000000))
  690. #define rawnumequal(o1, o2)        ((o1)->u64 == (o2)->u64)

  691. /* Macros to convert type ids. */
  692. #if LJ_64 && !LJ_GC64
  693. #define itypemap(o) \
  694.   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
  695. #else
  696. #define itypemap(o)        (tvisnumber(o) ? ~LJ_TNUMX : ~itype(o))
  697. #endif

  698. /* Macros to get tagged values. */
  699. #if LJ_GC64
  700. #define gcval(o)        ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
  701. #else
  702. #define gcval(o)        (gcref((o)->gcr))
  703. #endif
  704. #define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
  705. #if LJ_64
  706. #define lightudV(o) \
  707.   check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
  708. #else
  709. #define lightudV(o)        check_exp(tvislightud(o), gcrefp((o)->gcr, void))
  710. #endif
  711. #define gcV(o)                check_exp(tvisgcv(o), gcval(o))
  712. #define strV(o)                check_exp(tvisstr(o), &gcval(o)->str)
  713. #define funcV(o)        check_exp(tvisfunc(o), &gcval(o)->fn)
  714. #define threadV(o)        check_exp(tvisthread(o), &gcval(o)->th)
  715. #define protoV(o)        check_exp(tvisproto(o), &gcval(o)->pt)
  716. #define cdataV(o)        check_exp(tviscdata(o), &gcval(o)->cd)
  717. #define tabV(o)                check_exp(tvistab(o), &gcval(o)->tab)
  718. #define udataV(o)        check_exp(tvisudata(o), &gcval(o)->ud)
  719. #define numV(o)                check_exp(tvisnum(o), (o)->n)
  720. #define intV(o)                check_exp(tvisint(o), (int32_t)(o)->i)

  721. /* Macros to set tagged values. */
  722. #if LJ_GC64
  723. #define setitype(o, i)                ((o)->it = ((i) << 15))
  724. #define setnilV(o)                ((o)->it64 = -1)
  725. #define setpriV(o, x)                ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
  726. #define setboolV(o, x)                ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
  727. #else
  728. #define setitype(o, i)                ((o)->it = (i))
  729. #define setnilV(o)                ((o)->it = LJ_TNIL)
  730. #define setboolV(o, x)                ((o)->it = LJ_TFALSE-(uint32_t)(x))
  731. #define setpriV(o, i)                (setitype((o), (i)))
  732. #endif

  733. static LJ_AINLINE void setlightudV(TValue *o, void *p)
  734. {
  735. #if LJ_GC64
  736.   o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
  737. #elif LJ_64
  738.   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
  739. #else
  740.   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
  741. #endif
  742. }

  743. #if LJ_64
  744. #define checklightudptr(L, p) \
  745.   (((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p))
  746. #else
  747. #define checklightudptr(L, p)        (p)
  748. #endif

  749. #if LJ_FR2
  750. #define setcont(o, f)                ((o)->u64 = (uint64_t)(uintptr_t)(void *)(f))
  751. #elif LJ_64
  752. #define setcont(o, f) \
  753.   ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
  754. #else
  755. #define setcont(o, f)                setlightudV((o), (void *)(f))
  756. #endif

  757. #define tvchecklive(L, o) \
  758.   UNUSED(L), lua_assert(!tvisgcv(o) || \
  759.   ((~itype(o) == gcval(o)->gch.gct) && !isdead(G(L), gcval(o))))

  760. static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
  761. {
  762. #if LJ_GC64
  763.   setgcreft(o->gcr, v, itype);
  764. #else
  765.   setgcref(o->gcr, v); setitype(o, itype);
  766. #endif
  767. }

  768. static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
  769. {
  770.   setgcVraw(o, v, it); tvchecklive(L, o);
  771. }

  772. #define define_setV(name, type, tag) \
  773. static LJ_AINLINE void name(lua_State *L, TValue *o, type *v) \
  774. { \
  775.   setgcV(L, o, obj2gco(v), tag); \
  776. }
  777. define_setV(setstrV, GCstr, LJ_TSTR)
  778. define_setV(setthreadV, lua_State, LJ_TTHREAD)
  779. define_setV(setprotoV, GCproto, LJ_TPROTO)
  780. define_setV(setfuncV, GCfunc, LJ_TFUNC)
  781. define_setV(setcdataV, GCcdata, LJ_TCDATA)
  782. define_setV(settabV, GCtab, LJ_TTAB)
  783. define_setV(setudataV, GCudata, LJ_TUDATA)

  784. #define setnumV(o, x)                ((o)->n = (x))
  785. #define setnanV(o)                ((o)->u64 = U64x(fff80000,00000000))
  786. #define setpinfV(o)                ((o)->u64 = U64x(7ff00000,00000000))
  787. #define setminfV(o)                ((o)->u64 = U64x(fff00000,00000000))

  788. static LJ_AINLINE void setintV(TValue *o, int32_t i)
  789. {
  790. #if LJ_DUALNUM
  791.   o->i = (uint32_t)i; setitype(o, LJ_TISNUM);
  792. #else
  793.   o->n = (lua_Number)i;
  794. #endif
  795. }

  796. static LJ_AINLINE void setint64V(TValue *o, int64_t i)
  797. {
  798.   if (LJ_DUALNUM && LJ_LIKELY(i == (int64_t)(int32_t)i))
  799.     setintV(o, (int32_t)i);
  800.   else
  801.     setnumV(o, (lua_Number)i);
  802. }

  803. #if LJ_64
  804. #define setintptrV(o, i)        setint64V((o), (i))
  805. #else
  806. #define setintptrV(o, i)        setintV((o), (i))
  807. #endif

  808. /* Copy tagged values. */
  809. static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
  810. {
  811.   *o1 = *o2; tvchecklive(L, o1);
  812. }

  813. /* -- Number to integer conversion ---------------------------------------- */

  814. #if LJ_SOFTFP
  815. LJ_ASMF int32_t lj_vm_tobit(double x);
  816. #endif

  817. static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
  818. {
  819. #if LJ_SOFTFP
  820.   return lj_vm_tobit(n);
  821. #else
  822.   TValue o;
  823.   o.n = n + 6755399441055744.0/* 2^52 + 2^51 */
  824.   return (int32_t)o.u32.lo;
  825. #endif
  826. }

  827. #define lj_num2int(n)   ((int32_t)(n))

  828. static LJ_AINLINE uint64_t lj_num2u64(lua_Number n)
  829. {
  830. #ifdef _MSC_VER
  831.   if (n >= 9223372036854775808.0/* They think it's a feature. */
  832.     return (uint64_t)(int64_t)(n - 18446744073709551616.0);
  833.   else
  834. #endif
  835.     return (uint64_t)n;
  836. }

  837. static LJ_AINLINE int32_t numberVint(cTValue *o)
  838. {
  839.   if (LJ_LIKELY(tvisint(o)))
  840.     return intV(o);
  841.   else
  842.     return lj_num2int(numV(o));
  843. }

  844. static LJ_AINLINE lua_Number numberVnum(cTValue *o)
  845. {
  846.   if (LJ_UNLIKELY(tvisint(o)))
  847.     return (lua_Number)intV(o);
  848.   else
  849.     return numV(o);
  850. }

  851. /* -- Miscellaneous object handling --------------------------------------- */

  852. /* Names and maps for internal and external object tags. */
  853. LJ_DATA const char *const lj_obj_typename[1+LUA_TCDATA+1];
  854. LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];

  855. #define lj_typename(o)        (lj_obj_itypename[itypemap(o)])

  856. /* Compare two objects without calling metamethods. */
  857. LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
  858. LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(cTValue *o);

  859. #endif