Async environment for mods to do concurrent tasks (#11131)

src/script/common/CMakeLists.txt

@@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp
 	PARENT_SCOPE)
 

src/script/common/c_internal.cpp

@@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth)
 		infostream << script_get_backtrace(L) << std::endl;
 }
 
+void call_string_dump(lua_State *L, int idx)
+{
+	// Retrieve string.dump from insecure env to avoid it being tampered with
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP);
+	if (!lua_isnil(L, -1))
+		lua_getfield(L, -1, "string");
+	else
+		lua_getglobal(L, "string");
+	lua_getfield(L, -1, "dump");
+	lua_remove(L, -2); // remove _G
+	lua_remove(L, -2); // remove 'string' table
+	lua_pushvalue(L, idx);
+	lua_call(L, 1, 1);
+}

src/script/common/c_internal.h

@@ -56,6 +56,7 @@ extern "C" {
 #define CUSTOM_RIDX_BACKTRACE           (CUSTOM_RIDX_BASE + 3)
 #define CUSTOM_RIDX_HTTP_API_LUA        (CUSTOM_RIDX_BASE + 4)
 #define CUSTOM_RIDX_VECTOR_METATABLE    (CUSTOM_RIDX_BASE + 5)
+#define CUSTOM_RIDX_METATABLE_MAP       (CUSTOM_RIDX_BASE + 6)
 
 
 // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata
@@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode();
  * @param stack_depth How far on the stack to the first user function (ie: not builtin or core)
  */
 void log_deprecated(lua_State *L, std::string message, int stack_depth = 1);
+
+// Safely call string.dump on a function value
+// (does not pop, leaves one value on stack)
+void call_string_dump(lua_State *L, int idx);

src/script/common/c_packer.cpp

@@ -0,0 +1,583 @@
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include <cstdio>
+#include <cstring>
+#include <cmath>
+#include <cassert>
+#include <unordered_set>
+#include <unordered_map>
+#include "c_packer.h"
+#include "c_internal.h"
+#include "log.h"
+#include "debug.h"
+#include "threading/mutex_auto_lock.h"
+
+extern "C" {
+#include <lauxlib.h>
+}
+
+//
+// Helpers
+//
+
+// convert negative index to absolute position on Lua stack
+static inline int absidx(lua_State *L, int idx)
+{
+	assert(idx < 0);
+	return lua_gettop(L) + idx + 1;
+}
+
+// does the type put anything into PackedInstr::sdata?
+static inline bool uses_sdata(int type)
+{
+	switch (type) {
+		case LUA_TSTRING:
+		case LUA_TFUNCTION:
+		case LUA_TUSERDATA:
+			return true;
+		default:
+			return false;
+	}
+}
+
+// does the type put anything into PackedInstr::<union>?
+static inline bool uses_union(int type)
+{
+	switch (type) {
+		case LUA_TNIL:
+		case LUA_TSTRING:
+		case LUA_TFUNCTION:
+			return false;
+		default:
+			return true;
+	}
+}
+
+static inline bool can_set_into(int ktype, int vtype)
+{
+	switch (ktype) {
+		case LUA_TNUMBER:
+			return !uses_union(vtype);
+		case LUA_TSTRING:
+			return !uses_sdata(vtype);
+		default:
+			return false;
+	}
+}
+
+// is the key suitable for use with set_into?
+static inline bool suitable_key(lua_State *L, int idx)
+{
+	if (lua_type(L, idx) == LUA_TSTRING) {
+		// strings may not have a NULL byte (-> lua_setfield)
+		size_t len;
+		const char *str = lua_tolstring(L, idx, &len);
+		return strlen(str) == len;
+	} else {
+		assert(lua_type(L, idx) == LUA_TNUMBER);
+		// numbers must fit into an s32 and be integers (-> lua_rawseti)
+		lua_Number n = lua_tonumber(L, idx);
+		return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX;
+	}
+}
+
+namespace {
+	// checks if you left any values on the stack, for debugging
+	class StackChecker {
+		lua_State *L;
+		int top;
+	public:
+		StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {}
+		~StackChecker() {
+			assert(lua_gettop(L) >= top);
+			if (lua_gettop(L) > top) {
+				rawstream << "Lua stack not cleaned up: "
+					<< lua_gettop(L) << " != " << top
+					<< " (false-positive if exception thrown)" << std::endl;
+			}
+		}
+	};
+
+	// Since an std::vector may reallocate, this is the only safe way to keep
+	// a reference to a particular element.
+	template <typename T>
+	class VectorRef {
+		std::vector<T> *vec;
+		size_t idx;
+		VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {}
+	public:
+		static VectorRef<T> front(std::vector<T> &vec) {
+			return VectorRef(&vec, 0);
+		}
+		static VectorRef<T> back(std::vector<T> &vec) {
+			return VectorRef(&vec, vec.size() - 1);
+		}
+		T &operator*() { return (*vec)[idx]; }
+		T *operator->() { return &(*vec)[idx]; }
+	};
+
+	struct Packer {
+		PackInFunc fin;
+		PackOutFunc fout;
+	};
+
+	typedef std::pair<std::string, Packer> PackerTuple;
+}
+
+static inline auto emplace(PackedValue &pv, s16 type)
+{
+	pv.i.emplace_back();
+	auto ref = VectorRef<PackedInstr>::back(pv.i);
+	ref->type = type;
+	// Initialize fields that may be left untouched
+	if (type == LUA_TTABLE) {
+		ref->uidata1 = 0;
+		ref->uidata2 = 0;
+	} else if (type == LUA_TUSERDATA) {
+		ref->ptrdata = nullptr;
+	} else if (type == INSTR_POP) {
+		ref->sidata2 = 0;
+	}
+	return ref;
+}
+
+//
+// Management of registered packers
+//
+
+static std::unordered_map<std::string, Packer> g_packers;
+static std::mutex g_packers_lock;
+
+void script_register_packer(lua_State *L, const char *regname,
+	PackInFunc fin, PackOutFunc fout)
+{
+	// Store away callbacks
+	{
+		MutexAutoLock autolock(g_packers_lock);
+		auto it = g_packers.find(regname);
+		if (it == g_packers.end()) {
+			auto &ref = g_packers[regname];
+			ref.fin = fin;
+			ref.fout = fout;
+		} else {
+			FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout,
+				"Packer registered twice with mismatching callbacks");
+		}
+	}
+
+	// Save metatable so we can identify instances later
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	if (lua_isnil(L, -1)) {
+		lua_newtable(L);
+		lua_pushvalue(L, -1);
+		lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	}
+
+	luaL_getmetatable(L, regname);
+	FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name");
+
+	// CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... }
+	// check first
+	lua_pushstring(L, regname);
+	lua_rawget(L, -3);
+	if (!lua_isnil(L, -1)) {
+		FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2),
+				"Packer registered twice with inconsistent metatable");
+	}
+	lua_pop(L, 1);
+	// then set
+	lua_pushstring(L, regname);
+	lua_rawset(L, -3);
+
+	lua_pop(L, 1);
+}
+
+static bool find_packer(const char *regname, PackerTuple &out)
+{
+	MutexAutoLock autolock(g_packers_lock);
+	auto it = g_packers.find(regname);
+	if (it == g_packers.end())
+		return false;
+	// copy data for thread safety
+	out.first = it->first;
+	out.second = it->second;
+	return true;
+}
+
+static bool find_packer(lua_State *L, int idx, PackerTuple &out)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+#endif
+
+	// retrieve metatable of the object
+	if (lua_getmetatable(L, idx) != 1)
+		return false;
+
+	// use our global table to map it to the registry name
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	assert(lua_istable(L, -1));
+	lua_pushvalue(L, -2);
+	lua_rawget(L, -2);
+	if (lua_isnil(L, -1)) {
+		lua_pop(L, 3);
+		return false;
+	}
+
+	// load the associated data
+	bool found = find_packer(lua_tostring(L, -1), out);
+	FATAL_ERROR_IF(!found, "Inconsistent internal state");
+	lua_pop(L, 3);
+	return true;
+}
+
+//
+// Packing implementation
+//
+
+// recursively goes through the structure and ensures there are no circular references
+static void pack_validate(lua_State *L, int idx, std::unordered_set<const void*> &seen)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+	assert(idx > 0);
+#endif
+
+	if (lua_type(L, idx) != LUA_TTABLE)
+		return;
+
+	const void *ptr = lua_topointer(L, idx);
+	assert(ptr);
+
+	if (seen.find(ptr) != seen.end())
+		throw LuaError("Circular references cannot be packed (yet)");
+	seen.insert(ptr);
+
+	lua_checkstack(L, 5);
+	lua_pushnil(L);
+	while (lua_next(L, idx) != 0) {
+		// key at -2, value at -1
+		pack_validate(L, absidx(L, -2), seen);
+		pack_validate(L, absidx(L, -1), seen);
+
+		lua_pop(L, 1);
+	}
+
+	seen.erase(ptr);
+}
+
+static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+	assert(idx > 0);
+	assert(vidx > 0);
+#endif
+
+	switch (lua_type(L, idx)) {
+		case LUA_TNONE:
+		case LUA_TNIL:
+			return emplace(pv, LUA_TNIL);
+		case LUA_TBOOLEAN: {
+			auto r = emplace(pv, LUA_TBOOLEAN);
+			r->bdata = lua_toboolean(L, idx);
+			return r;
+		}
+		case LUA_TNUMBER: {
+			auto r = emplace(pv, LUA_TNUMBER);
+			r->ndata = lua_tonumber(L, idx);
+			return r;
+		}
+		case LUA_TSTRING: {
+			auto r = emplace(pv, LUA_TSTRING);
+			size_t len;
+			const char *str = lua_tolstring(L, idx, &len);
+			assert(str);
+			r->sdata.assign(str, len);
+			return r;
+		}
+		case LUA_TTABLE:
+			break; // execution continues
+		case LUA_TFUNCTION: {
+			auto r = emplace(pv, LUA_TFUNCTION);
+			call_string_dump(L, idx);
+			size_t len;
+			const char *str = lua_tolstring(L, -1, &len);
+			assert(str);
+			r->sdata.assign(str, len);
+			lua_pop(L, 1);
+			return r;
+		}
+		case LUA_TUSERDATA: {
+			PackerTuple ser;
+			if (!find_packer(L, idx, ser))
+				throw LuaError("Cannot serialize unsupported userdata");
+			pv.contains_userdata = true;
+			auto r = emplace(pv, LUA_TUSERDATA);
+			r->sdata = ser.first;
+			r->ptrdata = ser.second.fin(L, idx);
+			return r;
+		}
+		default: {
+			std::string err = "Cannot serialize type ";
+			err += lua_typename(L, lua_type(L, idx));
+			throw LuaError(err);
+		}
+	}
+
+	// LUA_TTABLE
+	lua_checkstack(L, 5);
+
+	auto rtable = emplace(pv, LUA_TTABLE);
+	const int vi_table = vidx++;
+
+	lua_pushnil(L);
+	while (lua_next(L, idx) != 0) {
+		// key at -2, value at -1
+		const int ktype = lua_type(L, -2), vtype = lua_type(L, -1);
+		if (ktype == LUA_TNUMBER)
+			rtable->uidata1++; // narr
+		else
+			rtable->uidata2++; // nrec
+
+		// check if we can use a shortcut
+		if (can_set_into(ktype, vtype) && suitable_key(L, -2)) {
+			// push only the value
+			auto rval = pack_inner(L, absidx(L, -1), vidx, pv);
+			rval->pop = vtype != LUA_TTABLE;
+			// and where to put it:
+			rval->set_into = vi_table;
+			if (ktype == LUA_TSTRING)
+				rval->sdata = lua_tostring(L, -2);
+			else
+				rval->sidata1 = lua_tointeger(L, -2);
+			// pop tables after the fact
+			if (!rval->pop) {
+				auto ri1 = emplace(pv, INSTR_POP);
+				ri1->sidata1 = vidx;
+			}
+		} else {
+			// push the key and value
+			pack_inner(L, absidx(L, -2), vidx, pv);
+			vidx++;
+			pack_inner(L, absidx(L, -1), vidx, pv);
+			vidx++;
+			// push an instruction to set them
+			auto ri1 = emplace(pv, INSTR_SETTABLE);
+			ri1->set_into = vi_table;
+			ri1->sidata1 = vidx - 2;
+			ri1->sidata2 = vidx - 1;
+			ri1->pop = true;
+			vidx -= 2;
+		}
+
+		lua_pop(L, 1);
+	}
+
+	assert(vidx == vi_table + 1);
+	return rtable;
+}
+
+PackedValue *script_pack(lua_State *L, int idx)
+{
+	if (idx < 0)
+		idx = absidx(L, idx);
+
+	std::unordered_set<const void*> seen;
+	pack_validate(L, idx, seen);
+	assert(seen.size() == 0);
+
+	// Actual serialization
+	PackedValue pv;
+	pack_inner(L, idx, 1, pv);
+
+	return new PackedValue(std::move(pv));
+}
+
+//
+// Unpacking implementation
+//
+
+void script_unpack(lua_State *L, PackedValue *pv)
+{
+	const int top = lua_gettop(L);
+	int ctr = 0;
+
+	for (auto &i : pv->i) {
+		// If leaving values on stack make sure there's space (every 5th iteration)
+		if (!i.pop && (ctr++) >= 5) {
+			lua_checkstack(L, 5);
+			ctr = 0;
+		}
+
+		/* Instructions */
+		switch (i.type) {
+			case INSTR_SETTABLE:
+				lua_pushvalue(L, top + i.sidata1); // key
+				lua_pushvalue(L, top + i.sidata2); // value
+				lua_rawset(L, top + i.set_into);
+				if (i.pop) {
+					if (i.sidata1 != i.sidata2) {
+						// removing moves indices so pop higher index first
+						lua_remove(L, top + std::max(i.sidata1, i.sidata2));
+						lua_remove(L, top + std::min(i.sidata1, i.sidata2));
+					} else {
+						lua_remove(L, top + i.sidata1);
+					}
+				}
+				continue;
+			case INSTR_POP:
+				lua_remove(L, top + i.sidata1);
+				if (i.sidata2 > 0)
+					lua_remove(L, top + i.sidata2);
+				continue;
+			default:
+				break;
+		}
+
+		/* Lua types */
+		switch (i.type) {
+			case LUA_TNIL:
+				lua_pushnil(L);
+				break;
+			case LUA_TBOOLEAN:
+				lua_pushboolean(L, i.bdata);
+				break;
+			case LUA_TNUMBER:
+				lua_pushnumber(L, i.ndata);
+				break;
+			case LUA_TSTRING:
+				lua_pushlstring(L, i.sdata.data(), i.sdata.size());
+				break;
+			case LUA_TTABLE:
+				lua_createtable(L, i.uidata1, i.uidata2);
+				break;
+			case LUA_TFUNCTION:
+				luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr);
+				break;
+			case LUA_TUSERDATA: {
+				PackerTuple ser;
+				sanity_check(find_packer(i.sdata.c_str(), ser));
+				ser.second.fout(L, i.ptrdata);
+				i.ptrdata = nullptr; // ownership taken by callback
+				break;
+			}
+			default:
+				assert(0);
+				break;
+		}
+
+		if (i.set_into) {
+			if (!i.pop)
+				lua_pushvalue(L, -1);
+			if (uses_sdata(i.type))
+				lua_rawseti(L, top + i.set_into, i.sidata1);
+			else
+				lua_setfield(L, top + i.set_into, i.sdata.c_str());
+		} else {
+			if (i.pop)
+				lua_pop(L, 1);
+		}
+	}
+
+	// as part of the unpacking process we take ownership of all userdata
+	pv->contains_userdata = false;
+	// leave exactly one value on the stack
+	lua_settop(L, top+1);
+}
+
+//
+// PackedValue
+//
+
+PackedValue::~PackedValue()
+{
+	if (!contains_userdata)
+		return;
+	for (auto &i : this->i) {
+		if (i.type == LUA_TUSERDATA && i.ptrdata) {
+			PackerTuple ser;
+			if (find_packer(i.sdata.c_str(), ser)) {
+				// tell it to deallocate object
+				ser.second.fout(nullptr, i.ptrdata);
+			} else {
+				assert(false);
+			}
+		}
+	}
+}
+
+//
+// script_dump_packed
+//
+
+#ifndef NDEBUG
+void script_dump_packed(const PackedValue *val)
+{
+	printf("instruction stream: [\n");
+	for (const auto &i : val->i) {
+		printf("\t(");
+		switch (i.type) {
+			case INSTR_SETTABLE:
+				printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2);
+				break;
+			case INSTR_POP:
+				printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2);
+				break;
+			case LUA_TNIL:
+				printf("nil");
+				break;
+			case LUA_TBOOLEAN:
+				printf(i.bdata ? "true" : "false");
+				break;
+			case LUA_TNUMBER:
+				printf("%f", i.ndata);
+				break;
+			case LUA_TSTRING:
+				printf("\"%s\"", i.sdata.c_str());
+				break;
+			case LUA_TTABLE:
+				printf("table(%d, %d)", i.uidata1, i.uidata2);
+				break;
+			case LUA_TFUNCTION:
+				printf("function(%d byte)", i.sdata.size());
+				break;
+			case LUA_TUSERDATA:
+				printf("userdata %s %p", i.sdata.c_str(), i.ptrdata);
+				break;
+			default:
+				printf("!!UNKNOWN!!");
+				break;
+		}
+		if (i.set_into) {
+			if (i.type >= 0 && uses_sdata(i.type))
+				printf(", k=%d, into=%d", i.sidata1, i.set_into);
+			else if (i.type >= 0)
+				printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into);
+			else
+				printf(", into=%d", i.set_into);
+		}
+		if (i.pop)
+			printf(", pop");
+		printf(")\n");
+	}
+	printf("]\n");
+}
+#endif

src/script/common/c_packer.h

@@ -0,0 +1,123 @@
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include "irrlichttypes.h"
+#include "util/basic_macros.h"
+
+extern "C" {
+#include <lua.h>
+}
+
+/*
+	This file defines an in-memory representation of Lua objects including
+	support for functions and userdata. It it used to move data between Lua
+	states and cannot be used for persistence or network transfer.
+*/
+
+#define INSTR_SETTABLE (-10)
+#define INSTR_POP      (-11)
+
+/**
+ * Represents a single instruction that pushes a new value or works with existing ones.
+ */
+struct PackedInstr
+{
+	s16 type; // LUA_T* or INSTR_*
+	u16 set_into; // set into table on stack
+	bool pop; // remove from stack?
+	union {
+		bool bdata; // boolean: value
+		lua_Number ndata; // number: value
+		struct {
+			u16 uidata1, uidata2; // table: narr, nrec
+		};
+		struct {
+			/*
+				SETTABLE: key index, value index
+				POP: indices to remove
+				otherwise w/ set_into: numeric key, -
+			*/
+			s32 sidata1, sidata2;
+		};
+		void *ptrdata; // userdata: implementation defined
+	};
+	/*
+		- string: value
+		- function: buffer
+		- w/ set_into: string key (no null bytes!)
+		- userdata: name in registry
+	*/
+	std::string sdata;
+
+	PackedInstr() : type(0), set_into(0), pop(false) {}
+};
+
+/**
+ * A packed value can be a primitive like a string or number but also a table
+ * including all of its contents. It is made up of a linear stream of
+ * 'instructions' that build the final value when executed.
+ */
+struct PackedValue
+{
+	std::vector<PackedInstr> i;
+	// Indicates whether there are any userdata pointers that need to be deallocated
+	bool contains_userdata = false;
+
+	PackedValue() = default;
+	~PackedValue();
+
+	DISABLE_CLASS_COPY(PackedValue)
+
+	ALLOW_CLASS_MOVE(PackedValue)
+};
+
+/*
+ * Packing callback: Turns a Lua value at given index into a void*
+ */
+typedef void *(*PackInFunc)(lua_State *L, int idx);
+/*
+ * Unpacking callback: Turns a void* back into the Lua value (left on top of stack)
+ *
+ * Note that this function must take ownership of the pointer, so make sure
+ * to free or keep the memory.
+ * `L` can be nullptr to indicate that data should just be discarded.
+ */
+typedef void (*PackOutFunc)(lua_State *L, void *ptr);
+/*
+ * Register a packable type with the name of its metatable.
+ *
+ * Even though the callbacks are global this must be called for every Lua state
+ * that supports objects of this type.
+ * This function is thread-safe.
+ */
+void script_register_packer(lua_State *L, const char *regname,
+		PackInFunc fin, PackOutFunc fout);
+
+// Pack a Lua value
+PackedValue *script_pack(lua_State *L, int idx);
+// Unpack a Lua value (left on top of stack)
+// Note that this may modify the PackedValue, you can't reuse it!
+void script_unpack(lua_State *L, PackedValue *val);
+
+// Dump contents of PackedValue to stdout for debugging
+void script_dump_packed(const PackedValue *val);