luajit: allow one to mark a function as blocked for debug stuff

Author: RaphaelIT7

LuaJIT-2.1/src/lib_base.c

@@ -141,6 +141,7 @@ LJLIB_ASM(setmetatable)		LJLIB_REC(.)
   return FFH_RES(1);
 }
 
+void blockDebug(lua_State* L, GCfunc* func);
 LJLIB_CF(getfenv)		LJLIB_REC(.)
 {
   GCfunc *fn;
@@ -155,6 +156,7 @@ LJLIB_CF(getfenv)		LJLIB_REC(.)
     if (LJ_FR2) o--;
   }
   fn = &gcval(o)->fn;
+  blockDebug(L, fn);
   settabV(L, L->top++, isluafunc(fn) ? tabref(fn->l.env) : tabref(L->env));
   return 1;
 }
@@ -181,6 +183,7 @@ LJLIB_CF(setfenv)
   fn = &gcval(o)->fn;
   if (!isluafunc(fn))
     lj_err_caller(L, LJ_ERR_SETFENV);
+  blockDebug(L, fn);
   setgcref(fn->l.env, obj2gco(t));
   lj_gc_objbarrier(L, obj2gco(fn), t);
   setfuncV(L, L->top++, fn);

LuaJIT-2.1/src/lib_debug.c

@@ -31,7 +31,9 @@ LJLIB_CF(debug_getregistry)
 
 LJLIB_CF(debug_getmetatable)	LJLIB_REC(.)
 {
-  lj_lib_checkany(L, 1);
+  if (tviscdata(lj_lib_checkany(L, 1)));
+    lj_err_arg(L, 1, LJ_ERR_DENYCDATA);
+
   if (!lua_getmetatable(L, 1)) {
     setnilV(L->top-1);
   }
@@ -40,6 +42,9 @@ LJLIB_CF(debug_getmetatable)	LJLIB_REC(.)
 
 LJLIB_CF(debug_setmetatable)
 {
+  if (tviscdata(lj_lib_checkany(L, 1)));
+    lj_err_arg(L, 1, LJ_ERR_DENYCDATA);
+
   lj_lib_checktabornil(L, 2);
   L->top = L->base+2;
   lua_setmetatable(L, 1);
@@ -49,15 +54,31 @@ LJLIB_CF(debug_setmetatable)
   return 1;
 }
 
+void blockDebug(lua_State* L, GCfunc* func)
+{
+  if (isblockdebug(func->c))
+    lj_err_arg(L, 1, LJ_ERR_BLOCKDEBUG);
+}
+
+void blockDebugCheck(lua_State* L, int idx)
+{
+    TValue *o = L->base + idx-1;
+    if (!(o < L->top && tvisfunc(o)))
+    {
+        blockDebug(L, funcV(o));
+    }
+}
+
 LJLIB_CF(debug_getfenv)
 {
-  lj_lib_checkany(L, 1);
+  blockDebugCheck(L, 1);
   lua_getfenv(L, 1);
   return 1;
 }
 
 LJLIB_CF(debug_setfenv)
 {
+  blockDebugCheck(L, 1);
   lj_lib_checktab(L, 2);
   L->top = L->base+2;
   if (!lua_setfenv(L, 1))
@@ -202,7 +223,7 @@ static int debug_getupvalue(lua_State *L, int get)
 {
   int32_t n = lj_lib_checkint(L, 2);
   const char *name;
-  lj_lib_checkfunc(L, 1);
+  blockDebug(L, lj_lib_checkfunc(L, 1));
   name = get ? lua_getupvalue(L, 1, n) : lua_setupvalue(L, 1, n);
   if (name) {
     lua_pushstring(L, name);
@@ -228,6 +249,7 @@ LJLIB_CF(debug_setupvalue)
 LJLIB_CF(debug_upvalueid)
 {
   GCfunc *fn = lj_lib_checkfunc(L, 1);
+  blockDebug(L, fn);
   int32_t n = lj_lib_checkint(L, 2) - 1;
   if ((uint32_t)n >= fn->l.nupvalues)
     lj_err_arg(L, 2, LJ_ERR_IDXRNG);
@@ -244,6 +266,7 @@ LJLIB_CF(debug_upvaluejoin)
   for (i = 0; i < 2; i++) {
     int32_t n;
     fn[i] = lj_lib_checkfunc(L, 2*i+1);
+    blockDebug(L, fn[i]);
     if (!isluafunc(fn[i]))
       lj_err_arg(L, 2*i+1, LJ_ERR_NOLFUNC);
     n = lj_lib_checkint(L, 2*i+2) - 1;

LuaJIT-2.1/src/lj_errmsg.h

@@ -42,6 +42,7 @@ ERRDEF(BADSELF,	"calling " LUA_QS " on bad self (%s)")
 ERRDEF(BADARG,	"bad argument #%d to " LUA_QS " (%s)")
 ERRDEF(BADTYPE,	"%s expected, got %s")
 ERRDEF(BADVAL,	"invalid value")
+ERRDEF(DENYCDATA, "cdata is not allowed to be used here")
 ERRDEF(NOVAL,	"value expected")
 ERRDEF(NOCORO,	"coroutine expected")
 ERRDEF(NOTABN,	"nil or table expected")
@@ -52,6 +53,7 @@ ERRDEF(NOPROXY,	"boolean or proxy expected")
 ERRDEF(FORINIT,	LUA_QL("for") " initial value must be a number")
 ERRDEF(FORLIM,	LUA_QL("for") " limit must be a number")
 ERRDEF(FORSTEP,	LUA_QL("for") " step must be a number")
+ERRDEF(BLOCKDEBUG, "function is marked to block any debug access")
 
 /* C API checks. */
 ERRDEF(NOENV,	"no calling environment")

LuaJIT-2.1/src/lj_gc.h

@@ -24,6 +24,7 @@ enum {
 #define LJ_GC_FIXED	0x20
 #define LJ_GC_SFIXED	0x40
 #define LJ_GC_READONLY	0x80
+#define LJ_GC_BLOCKDEBUG  0x80 // Same as LJ_GC_READONLY, but we'll use this one for functions.
 
 #define LJ_GC_WHITES	(LJ_GC_WHITE0 | LJ_GC_WHITE1)
 #define LJ_GC_COLORS	(LJ_GC_WHITES | LJ_GC_BLACK)
@@ -50,6 +51,10 @@ enum {
 #define markreadonly(x) ((x)->marked |= LJ_GC_READONLY)
 #define unmarkreadonly(x) ((x)->marked &= ~LJ_GC_READONLY)
 
+#define isblockdebug(x) ((x).marked & LJ_GC_BLOCKDEBUG)
+#define markblockdebug(x) ((x).marked |= LJ_GC_BLOCKDEBUG)
+#define unmarkblockdebug(x) ((x).marked &= ~LJ_GC_BLOCKDEBUG)
+
 /* Collector. */
 LJ_FUNC size_t lj_gc_separateudata(global_State *g, int all);
 LJ_FUNC void lj_gc_finalize_udata(lua_State *L);