Async environment for mods to do concurrent tasks (#11131)

src/script/lua_api/l_craft.cpp

@@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top)
 	API_FCT(register_craft);
 	API_FCT(clear_craft);
 }
+
+void ModApiCraft::InitializeAsync(lua_State *L, int top)
+{
+	// all read-only functions
+	API_FCT(get_all_craft_recipes);
+	API_FCT(get_craft_recipe);
+	API_FCT(get_craft_result);
+}

src/script/lua_api/l_craft.h

@@ -45,4 +45,5 @@ private:
 
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_internal.h

@@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 // Retrieve Environment pointer as `env` (no map lock)
 #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK            \
-	Environment *env = (Environment *)getEnv(L); \
+	Environment *env = getEnv(L);                \
 	if (env == NULL)                             \
 		return 0
 

src/script/lua_api/l_item.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "itemdef.h"
 #include "nodedef.h"
 #include "server.h"
@@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L)
 	lua_setmetatable(L, -2);
 	return 1;
 }
+
 // Not callable from Lua
 int LuaItemStack::create(lua_State *L, const ItemStack &item)
 {
@@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg)
 	return *(LuaItemStack **)luaL_checkudata(L, narg, className);
 }
 
+void *LuaItemStack::packIn(lua_State *L, int idx)
+{
+	LuaItemStack *o = checkobject(L, idx);
+	return new ItemStack(o->getItem());
+}
+
+void LuaItemStack::packOut(lua_State *L, void *ptr)
+{
+	ItemStack *stack = reinterpret_cast<ItemStack*>(ptr);
+	if (L)
+		create(L, *stack);
+	delete stack;
+}
+
 void LuaItemStack::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L)
 
 	// Can be created from Lua (ItemStack(itemstack or itemstring or table or nil))
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 const char LuaItemStack::className[] = "ItemStack";
@@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top)
 	API_FCT(get_content_id);
 	API_FCT(get_name_from_content_id);
 }
+
+void ModApiItemMod::InitializeAsync(lua_State *L, int top)
+{
+	// all read-only functions
+	API_FCT(get_content_id);
+	API_FCT(get_name_from_content_id);
+}

src/script/lua_api/l_item.h

@@ -141,8 +141,11 @@ public:
 	// Not callable from Lua
 	static int create(lua_State *L, const ItemStack &item);
 	static LuaItemStack* checkobject(lua_State *L, int narg);
-	static void Register(lua_State *L);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
+	static void Register(lua_State *L);
 };
 
 class ModApiItemMod : public ModApiBase {
@@ -152,6 +155,8 @@ private:
 	static int l_register_alias_raw(lua_State *L);
 	static int l_get_content_id(lua_State *L);
 	static int l_get_name_from_content_id(lua_State *L);
+
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_noise.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "log.h"
 #include "porting.h"
 #include "util/numeric.h"
@@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg)
 }
 
 
+void *LuaPerlinNoise::packIn(lua_State *L, int idx)
+{
+	LuaPerlinNoise *o = checkobject(L, idx);
+	return new NoiseParams(o->np);
+}
+
+void LuaPerlinNoise::packOut(lua_State *L, void *ptr)
+{
+	NoiseParams *np = reinterpret_cast<NoiseParams*>(ptr);
+	if (L) {
+		LuaPerlinNoise *o = new LuaPerlinNoise(np);
+		*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+		luaL_getmetatable(L, className);
+		lua_setmetatable(L, -2);
+	}
+	delete np;
+}
+
+
 void LuaPerlinNoise::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L)
 	lua_pop(L, 1);
 
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 
@@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg)
 }
 
 
+struct NoiseMapParams {
+	NoiseParams np;
+	s32 seed;
+	v3s16 size;
+};
+
+void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx)
+{
+	LuaPerlinNoiseMap *o = checkobject(L, idx);
+	NoiseMapParams *ret = new NoiseMapParams();
+	ret->np = o->noise->np;
+	ret->seed = o->noise->seed;
+	ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz);
+	return ret;
+}
+
+void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr)
+{
+	NoiseMapParams *p = reinterpret_cast<NoiseMapParams*>(ptr);
+	if (L) {
+		LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size);
+		*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+		luaL_getmetatable(L, className);
+		lua_setmetatable(L, -2);
+	}
+	delete p;
+}
+
+
 void LuaPerlinNoiseMap::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L)
 	lua_pop(L, 1);
 
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 

src/script/lua_api/l_noise.h

@@ -52,6 +52,9 @@ public:
 
 	static LuaPerlinNoise *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };
 
@@ -91,6 +94,9 @@ public:
 
 	static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };
 

src/script/lua_api/l_server.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "cpp_api/s_base.h"
 #include "cpp_api/s_security.h"
 #include "scripting_server.h"
@@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L)
 	return 0;
 }
 
+// do_async_callback(func, params, mod_origin)
+int ModApiServer::l_do_async_callback(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+	ServerScripting *script = getScriptApi<ServerScripting>(L);
+
+	luaL_checktype(L, 1, LUA_TFUNCTION);
+	luaL_checktype(L, 2, LUA_TTABLE);
+	luaL_checktype(L, 3, LUA_TSTRING);
+
+	call_string_dump(L, 1);
+	size_t func_length;
+	const char *serialized_func_raw = lua_tolstring(L, -1, &func_length);
+
+	PackedValue *param = script_pack(L, 2);
+
+	std::string mod_origin = readParam<std::string>(L, 3);
+
+	u32 jobId = script->queueAsync(
+		std::string(serialized_func_raw, func_length),
+		param, mod_origin);
+
+	lua_settop(L, 0);
+	lua_pushinteger(L, jobId);
+	return 1;
+}
+
+// register_async_dofile(path)
+int ModApiServer::l_register_async_dofile(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	std::string path = readParam<std::string>(L, 1);
+	CHECK_SECURE_PATH(L, path.c_str(), false);
+
+	// Find currently running mod name (only at init time)
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
+	if (!lua_isstring(L, -1))
+		return 0;
+	std::string modname = readParam<std::string>(L, -1);
+
+	getServer(L)->m_async_init_files.emplace_back(modname, path);
+	lua_pushboolean(L, true);
+	return 1;
+}
+
+// serialize_roundtrip(value)
+// Meant for unit testing the packer from Lua
+int ModApiServer::l_serialize_roundtrip(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	int top = lua_gettop(L);
+	auto *pv = script_pack(L, 1);
+	if (top != lua_gettop(L))
+		throw LuaError("stack values leaked");
+
+#ifndef NDEBUG
+	script_dump_packed(pv);
+#endif
+
+	top = lua_gettop(L);
+	script_unpack(L, pv);
+	delete pv;
+	if (top + 1 != lua_gettop(L))
+		throw LuaError("stack values leaked");
+
+	return 1;
+}
+
 void ModApiServer::Initialize(lua_State *L, int top)
 {
 	API_FCT(request_shutdown);
@@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top)
 	API_FCT(remove_player);
 	API_FCT(unban_player_or_ip);
 	API_FCT(notify_authentication_modified);
+
+	API_FCT(do_async_callback);
+	API_FCT(register_async_dofile);
+	API_FCT(serialize_roundtrip);
+}
+
+void ModApiServer::InitializeAsync(lua_State *L, int top)
+{
+	API_FCT(get_worldpath);
+	API_FCT(is_singleplayer);
+
+	API_FCT(get_current_modname);
+	API_FCT(get_modpath);
+	API_FCT(get_modnames);
 }

src/script/lua_api/l_server.h

@@ -106,6 +106,16 @@ private:
 	// notify_authentication_modified(name)
 	static int l_notify_authentication_modified(lua_State *L);
 
+	// do_async_callback(func, params, mod_origin)
+	static int l_do_async_callback(lua_State *L);
+
+	// register_async_dofile(path)
+	static int l_register_async_dofile(lua_State *L);
+
+	// serialize_roundtrip(obj)
+	static int l_serialize_roundtrip(lua_State *L);
+
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_util.cpp

@@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
 	API_FCT(cpdir);
 	API_FCT(mvdir);
 	API_FCT(get_dir_list);
+	API_FCT(safe_file_write);
+
+	API_FCT(request_insecure_environment);
 
 	API_FCT(encode_base64);
 	API_FCT(decode_base64);
@@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
 	API_FCT(colorspec_to_colorstring);
 	API_FCT(colorspec_to_bytes);
 
+	API_FCT(encode_png);
+
 	API_FCT(get_last_run_mod);
 	API_FCT(set_last_run_mod);
 

src/script/lua_api/l_util.h

@@ -129,6 +129,4 @@ public:
 	static void Initialize(lua_State *L, int top);
 	static void InitializeAsync(lua_State *L, int top);
 	static void InitializeClient(lua_State *L, int top);
-
-	static void InitializeAsync(AsyncEngine &engine);
 };

src/script/lua_api/l_vmanip.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_content.h"
 #include "common/c_converter.h"
+#include "common/c_packer.h"
 #include "emerge.h"
 #include "environment.h"
 #include "map.h"
@@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L)
 
 	LuaVoxelManip *o = checkobject(L, 1);
 	MMVManip *vm = o->vm;
+	if (vm->isOrphan())
+		return 0;
 
 	v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2));
 	v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3));
@@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg)
 	return *(LuaVoxelManip **)ud;  // unbox pointer
 }
 
+void *LuaVoxelManip::packIn(lua_State *L, int idx)
+{
+	LuaVoxelManip *o = checkobject(L, idx);
+
+	if (o->is_mapgen_vm)
+		throw LuaError("nope");
+	return o->vm->clone();
+}
+
+void LuaVoxelManip::packOut(lua_State *L, void *ptr)
+{
+	MMVManip *vm = reinterpret_cast<MMVManip*>(ptr);
+	if (!L) {
+		delete vm;
+		return;
+	}
+
+	// Associate vmanip with map if the Lua env has one
+	Environment *env = getEnv(L);
+	if (env)
+		vm->reparent(&(env->getMap()));
+
+	LuaVoxelManip *o = new LuaVoxelManip(vm, false);
+	*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+	luaL_getmetatable(L, className);
+	lua_setmetatable(L, -2);
+}
+
 void LuaVoxelManip::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L)
 
 	// Can be created from Lua (VoxelManip())
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 const char LuaVoxelManip::className[] = "VoxelManip";

src/script/lua_api/l_vmanip.h

@@ -75,5 +75,8 @@ public:
 
 	static LuaVoxelManip *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };