Fix modstore/favourites hang by adding asynchronous lua job support

src/script/lua_api/CMakeLists.txt

@@ -16,6 +16,8 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/l_async_events.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/marshall.c
 	PARENT_SCOPE)
 
 # Used by client only

src/script/lua_api/l_async_events.cpp

@@ -0,0 +1,396 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+extern "C" {
+#include "lua.h"
+#include "lauxlib.h"
+#include "lualib.h"
+int luaopen_marshal(lua_State *L);
+}
+#include <stdio.h>
+
+#include "l_async_events.h"
+#include "log.h"
+#include "filesys.h"
+#include "porting.h"
+
+//TODO replace by ShadowNinja version not yet merged to master
+static int script_error_handler(lua_State *L) {
+  lua_getfield(L, LUA_GLOBALSINDEX, "debug");
+  if (!lua_istable(L, -1)) {
+    lua_pop(L, 1);
+    return 1;
+  }
+  lua_getfield(L, -1, "traceback");
+  if (!lua_isfunction(L, -1)) {
+    lua_pop(L, 2);
+    return 1;
+  }
+  lua_pushvalue(L, 1);
+  lua_pushinteger(L, 2);
+  lua_call(L, 2, 1);
+  return 1;
+}
+
+/******************************************************************************/
+static void scriptError(const char *fmt, ...)
+{
+	va_list argp;
+	va_start(argp, fmt);
+	char buf[10000];
+	vsnprintf(buf, 10000, fmt, argp);
+	va_end(argp);
+	errorstream<<"ERROR: "<<buf;
+	fprintf(stderr,"ERROR: %s\n",buf);
+}
+
+/******************************************************************************/
+AsyncEngine::AsyncEngine() :
+	m_initDone(false),
+	m_JobIdCounter(0)
+{
+	assert(m_JobQueueMutex.Init() == 0);
+	assert(m_ResultQueueMutex.Init() == 0);
+}
+
+/******************************************************************************/
+AsyncEngine::~AsyncEngine()
+{
+	/** request all threads to stop **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Stop();
+	}
+
+
+	/** wakeup all threads **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+				i != m_WorkerThreads.end();i++) {
+		m_JobQueueCounter.Post();
+	}
+
+	/** wait for threads to finish **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Wait();
+	}
+
+	/** force kill all threads **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Kill();
+		delete *i;
+	}
+
+	m_JobQueueMutex.Lock();
+	m_JobQueue.clear();
+	m_JobQueueMutex.Unlock();
+	m_WorkerThreads.clear();
+}
+
+/******************************************************************************/
+bool AsyncEngine::registerFunction(const char* name, lua_CFunction fct) {
+
+	if (m_initDone) return false;
+	m_FunctionList[name] = fct;
+	return true;
+}
+
+/******************************************************************************/
+void AsyncEngine::Initialize(unsigned int numengines) {
+	m_initDone = true;
+
+	for (unsigned int i=0; i < numengines; i ++) {
+
+		AsyncWorkerThread* toadd = new AsyncWorkerThread(this,i);
+		m_WorkerThreads.push_back(toadd);
+		toadd->Start();
+	}
+}
+
+/******************************************************************************/
+unsigned int AsyncEngine::doAsyncJob(std::string fct, std::string params) {
+
+	m_JobQueueMutex.Lock();
+	LuaJobInfo toadd;
+	toadd.JobId = m_JobIdCounter++;
+	toadd.serializedFunction = fct;
+	toadd.serializedParams = params;
+
+	m_JobQueue.push_back(toadd);
+
+	m_JobQueueCounter.Post();
+
+	m_JobQueueMutex.Unlock();
+
+	return toadd.JobId;
+}
+
+/******************************************************************************/
+LuaJobInfo AsyncEngine::getJob() {
+
+	m_JobQueueCounter.Wait();
+	m_JobQueueMutex.Lock();
+
+	LuaJobInfo retval;
+
+	if (m_JobQueue.size() != 0) {
+		retval = m_JobQueue.front();
+		m_JobQueue.erase((m_JobQueue.begin()));
+	}
+	m_JobQueueMutex.Unlock();
+
+	return retval;
+}
+
+/******************************************************************************/
+void AsyncEngine::putJobResult(LuaJobInfo result) {
+	m_ResultQueueMutex.Lock();
+	m_ResultQueue.push_back(result);
+	m_ResultQueueMutex.Unlock();
+}
+
+/******************************************************************************/
+void AsyncEngine::Step(lua_State *L) {
+	m_ResultQueueMutex.Lock();
+	while(!m_ResultQueue.empty()) {
+
+		LuaJobInfo jobdone = m_ResultQueue.front();
+		m_ResultQueue.erase(m_ResultQueue.begin());
+		lua_getglobal(L, "engine");
+
+		lua_getfield(L, -1, "async_event_handler");
+
+		if(lua_isnil(L, -1))
+			assert("Someone managed to destroy a async callback in engine!" == 0);
+
+		luaL_checktype(L, -1, LUA_TFUNCTION);
+
+		lua_pushinteger(L, jobdone.JobId);
+		lua_pushlstring(L, jobdone.serializedResult.c_str(),
+							jobdone.serializedResult.length());
+
+		if(lua_pcall(L, 2, 0, 0)) {
+			scriptError("Async ENGINE step: %s", lua_tostring(L, -1));
+		}
+
+		lua_pop(L,1);
+	}
+	m_ResultQueueMutex.Unlock();
+}
+
+/******************************************************************************/
+void AsyncEngine::PushFinishedJobs(lua_State* L) {
+	//Result Table
+	m_ResultQueueMutex.Lock();
+
+	unsigned int index=1;
+	lua_newtable(L);
+	int top = lua_gettop(L);
+
+	while(!m_ResultQueue.empty()) {
+
+		LuaJobInfo jobdone = m_ResultQueue.front();
+		m_ResultQueue.erase(m_ResultQueue.begin());
+
+		lua_pushnumber(L,index);
+
+		lua_newtable(L);
+		int top_lvl2 = lua_gettop(L);
+
+		lua_pushstring(L,"jobid");
+		lua_pushnumber(L,jobdone.JobId);
+		lua_settable(L, top_lvl2);
+
+		lua_pushstring(L,"retval");
+		lua_pushstring(L, jobdone.serializedResult.c_str());
+		lua_settable(L, top_lvl2);
+
+		lua_settable(L, top);
+		index++;
+	}
+
+	m_ResultQueueMutex.Unlock();
+
+}
+/******************************************************************************/
+void AsyncEngine::PrepareEnvironment(lua_State* L, int top) {
+	for(std::map<std::string,lua_CFunction>::iterator i = m_FunctionList.begin();
+			i != m_FunctionList.end(); i++) {
+
+		lua_pushstring(L,i->first.c_str());
+		lua_pushcfunction(L,i->second);
+		lua_settable(L, top);
+
+	}
+}
+
+/******************************************************************************/
+int async_worker_ErrorHandler(lua_State *L) {
+	lua_getfield(L, LUA_GLOBALSINDEX, "debug");
+	if (!lua_istable(L, -1)) {
+	lua_pop(L, 1);
+	return 1;
+	}
+	lua_getfield(L, -1, "traceback");
+	if (!lua_isfunction(L, -1)) {
+	lua_pop(L, 2);
+	return 1;
+	}
+	lua_pushvalue(L, 1);
+	lua_pushinteger(L, 2);
+	lua_call(L, 2, 1);
+	return 1;
+}
+
+/******************************************************************************/
+AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobdispatcher,
+									unsigned int numthreadnumber) :
+	m_JobDispatcher(jobdispatcher),
+	m_luaerrorhandler(-1),
+	m_threadnum(numthreadnumber)
+{
+	// create luastack
+	m_LuaStack = luaL_newstate();
+
+	// load basic lua modules
+	luaL_openlibs(m_LuaStack);
+
+	// load serialization functions
+	luaopen_marshal(m_LuaStack);
+}
+
+/******************************************************************************/
+AsyncWorkerThread::~AsyncWorkerThread() {
+
+	assert(IsRunning() == false);
+	lua_close(m_LuaStack);
+}
+
+/******************************************************************************/
+void* AsyncWorkerThread::worker_thread_main() {
+
+	//register thread for error logging
+	char number[21];
+	snprintf(number,sizeof(number),"%d",m_threadnum);
+	log_register_thread(std::string("AsyncWorkerThread_") + number);
+
+	/** prepare job lua environment **/
+	lua_newtable(m_LuaStack);
+	lua_setglobal(m_LuaStack, "engine");
+
+	lua_getglobal(m_LuaStack, "engine");
+	int top = lua_gettop(m_LuaStack);
+
+	lua_pushstring(m_LuaStack, DIR_DELIM);
+	lua_setglobal(m_LuaStack, "DIR_DELIM");
+
+	lua_pushstring(m_LuaStack,
+			std::string(porting::path_share + DIR_DELIM + "builtin").c_str());
+	lua_setglobal(m_LuaStack, "SCRIPTDIR");
+
+
+	m_JobDispatcher->PrepareEnvironment(m_LuaStack,top);
+
+	std::string asyncscript =
+	porting::path_share + DIR_DELIM + "builtin"
+			+ DIR_DELIM + "async_env.lua";
+
+	lua_pushcfunction(m_LuaStack, async_worker_ErrorHandler);
+	m_luaerrorhandler = lua_gettop(m_LuaStack);
+
+	if(!runScript(asyncscript)) {
+		infostream
+			<< "AsyncWorkderThread::worker_thread_main execution of async base environment failed!"
+			<< std::endl;
+			assert("no future with broken builtin async environment scripts" == 0);
+	}
+	/** main loop **/
+	while(IsRunning()) {
+		//wait for job
+		LuaJobInfo toprocess = m_JobDispatcher->getJob();
+
+		if (!IsRunning()) { continue; }
+
+		//first push error handler
+		lua_pushcfunction(m_LuaStack, script_error_handler);
+		int errorhandler = lua_gettop(m_LuaStack);
+
+		lua_getglobal(m_LuaStack, "engine");
+		if(lua_isnil(m_LuaStack, -1))
+			assert("unable to find engine within async environment" == 0);
+
+		lua_getfield(m_LuaStack, -1, "job_processor");
+		if(lua_isnil(m_LuaStack, -1))
+			assert("Someone managed to destroy a async worker engine!" == 0);
+
+		luaL_checktype(m_LuaStack, -1, LUA_TFUNCTION);
+
+		//call it
+		lua_pushlstring(m_LuaStack,
+						toprocess.serializedFunction.c_str(),
+						toprocess.serializedFunction.length());
+		lua_pushlstring(m_LuaStack,
+						toprocess.serializedParams.c_str(),
+						toprocess.serializedParams.length());
+
+		if (!IsRunning()) { continue; }
+		if(lua_pcall(m_LuaStack, 2, 2, errorhandler)) {
+			scriptError("Async WORKER thread: %s\n", lua_tostring(m_LuaStack, -1));
+			toprocess.serializedResult="ERROR";
+		}
+		else {
+			//fetch result
+			const char *retval = lua_tostring(m_LuaStack, -2);
+			unsigned int lenght = lua_tointeger(m_LuaStack,-1);
+			toprocess.serializedResult = std::string(retval,lenght);
+		}
+
+		if (!IsRunning()) { continue; }
+		//put job result
+		m_JobDispatcher->putJobResult(toprocess);
+	}
+	log_deregister_thread();
+	return 0;
+}
+
+/******************************************************************************/
+bool AsyncWorkerThread::runScript(std::string script) {
+
+	int ret = 	luaL_loadfile(m_LuaStack, script.c_str()) ||
+				lua_pcall(m_LuaStack, 0, 0, m_luaerrorhandler);
+	if(ret){
+		errorstream<<"==== ERROR FROM LUA WHILE INITIALIZING ASYNC ENVIRONMENT ====="<<std::endl;
+		errorstream<<"Failed to load and run script from "<<std::endl;
+		errorstream<<script<<":"<<std::endl;
+		errorstream<<std::endl;
+		errorstream<<lua_tostring(m_LuaStack, -1)<<std::endl;
+		errorstream<<std::endl;
+		errorstream<<"=================== END OF ERROR FROM LUA ===================="<<std::endl;
+		lua_pop(m_LuaStack, 1); // Pop error message from stack
+		lua_pop(m_LuaStack, 1); // Pop the error handler from stack
+		return false;
+	}
+	return true;
+}
+
+/******************************************************************************/
+void* AsyncWorkerThread::worker_thread_wrapper(void* thread) {
+	return ((AsyncWorkerThread*) thread)->worker_thread_main();
+}

src/script/lua_api/l_async_events.h

@@ -0,0 +1,228 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef C_ASYNC_EVENTS_H_
+#define C_ASYNC_EVENTS_H_
+
+#include <vector>
+#include <map>
+#include <unistd.h>
+
+/******************************************************************************/
+/* Includes                                                                   */
+/******************************************************************************/
+#include "jthread/jthread.h"
+#include "jthread/jmutex.h"
+#include "jthread/jsemaphore.h"
+#include "debug.h"
+#include "lua.h"
+
+/******************************************************************************/
+/* Typedefs and macros                                                        */
+/******************************************************************************/
+#define MAINMENU_NUMBER_OF_ASYNC_THREADS                   4
+
+/******************************************************************************/
+/* forward declarations                                                       */
+/******************************************************************************/
+class AsyncEngine;
+
+/******************************************************************************/
+/* declarations                                                               */
+/******************************************************************************/
+
+/** a struct to encapsulate data required to queue a job **/
+struct LuaJobInfo {
+	/** function to be called in async environment **/
+	std::string serializedFunction;
+	/** parameter table to be passed to function **/
+	std::string serializedParams;
+	/** result of function call **/
+	std::string serializedResult;
+	/** jobid used to identify a job and match it to callback **/
+	unsigned int JobId;
+};
+
+/** class encapsulating a asynchronous working environment **/
+class AsyncWorkerThread : public JThread {
+public:
+	/**
+	 * default constructor
+	 * @param pointer to job dispatcher
+	 */
+	AsyncWorkerThread(AsyncEngine* jobdispatcher, unsigned int threadnumber);
+
+	/**
+	 * default destructor
+	 */
+	virtual ~AsyncWorkerThread();
+
+	/**
+	 * thread function
+	 */
+	void* Thread() {
+			ThreadStarted();
+			return worker_thread_wrapper(this);
+	}
+
+	/**
+	 * wait for thread to stop
+	 */
+	void Wait() {
+		while(IsRunning()) {
+			sleep(1);
+		}
+	}
+
+private:
+	/**
+	 * helper function to run a lua script
+	 * @param path of script
+	 */
+	bool runScript(std::string script);
+
+	/**
+	 * main function of thread
+	 */
+	void* worker_thread_main();
+
+	/**
+	 * static wrapper for thread creation
+	 * @param this pointer to the thread to be created
+	 */
+	static void* worker_thread_wrapper(void* thread);
+
+	/**
+	 * pointer to job dispatcher
+	 */
+	AsyncEngine* m_JobDispatcher;
+
+	/**
+	 * the lua stack to run at
+	 */
+	lua_State*   m_LuaStack;
+
+	/**
+	 * lua internal stack number of error handler
+	 */
+	int m_luaerrorhandler;
+
+	/**
+	 * thread number used for debug output
+	 */
+	unsigned int m_threadnum;
+
+};
+
+/** asynchornous thread and job management **/
+class AsyncEngine {
+	friend AsyncWorkerThread;
+public:
+	/**
+	 * default constructor
+	 */
+	AsyncEngine();
+	/**
+	 * default destructor
+	 */
+	~AsyncEngine();
+
+	/**
+	 * register function to be used within engines
+	 * @param name function name to be used within lua environment
+	 * @param fct c-function to be called
+	 */
+	bool registerFunction(const char* name, lua_CFunction fct);
+
+	/**
+	 * create async engine tasks and lock function registration
+	 * @param numengines number of async threads to be started
+	 */
+	void Initialize(unsigned int numengines);
+
+	/**
+	 * queue/run a async job
+	 * @param fct serialized lua function
+	 * @param params serialized parameters
+	 * @return jobid the job is queued
+	 */
+	unsigned int doAsyncJob(std::string fct, std::string params);
+
+	/**
+	 * engine step to process finished jobs
+	 *   the engine step is one way to pass events back, PushFinishedJobs another
+	 * @param L the lua environment to do the step in
+	 */
+	void Step(lua_State *L);
+
+
+	void PushFinishedJobs(lua_State* L);
+
+protected:
+	/**
+	 * Get a Job from queue to be processed
+	 *  this function blocks until a job is ready
+	 * @return a job to be processed
+	 */
+	LuaJobInfo getJob();
+
+	/**
+	 * put a Job result back to result queue
+	 * @param result result of completed job
+	 */
+	void putJobResult(LuaJobInfo result);
+
+	/**
+	 * initialize environment with current registred functions
+	 *  this function adds all functions registred by registerFunction to the
+	 *  passed lua stack
+	 * @param L lua stack to initialize
+	 * @param top stack position
+	 */
+	void PrepareEnvironment(lua_State* L, int top);
+
+private:
+
+	/** variable locking the engine against further modification **/
+	bool m_initDone;
+
+	/** internal store for registred functions **/
+	std::map<std::string,lua_CFunction> m_FunctionList;
+
+	/** internal counter to create job id's **/
+	unsigned int m_JobIdCounter;
+
+	/** mutex to protect job queue **/
+	JMutex m_JobQueueMutex;
+	/** job queue **/
+	std::vector<LuaJobInfo> m_JobQueue;
+
+	/** mutext to protect result queue **/
+	JMutex m_ResultQueueMutex;
+	/** result queue **/
+	std::vector<LuaJobInfo> m_ResultQueue;
+
+	/** list of current worker threads **/
+	std::vector<AsyncWorkerThread*> m_WorkerThreads;
+
+	/** counter semaphore for job dispatching **/
+	JSemaphore m_JobQueueCounter;
+};
+
+#endif /* C_ASYNC_EVENTS_H_ */

src/script/lua_api/l_internal.h

@@ -30,14 +30,16 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "common/c_internal.h"
 
 #define luamethod(class, name) {#name, class::l_##name}
-#define API_FCT(name) registerFunction(L,#name,l_##name,top)
+#define API_FCT(name) registerFunction(L, #name, l_##name,top)
+#define ASYNC_API_FCT(name) engine.registerFunction(#name, l_##name)
 
 #if (defined(WIN32) || defined(_WIN32_WCE))
 #define NO_MAP_LOCK_REQUIRED
 #else
 #include "main.h"
 #include "profiler.h"
-#define NO_MAP_LOCK_REQUIRED ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD)
+#define NO_MAP_LOCK_REQUIRED \
+	ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD)
 #endif
 
 #endif /* L_INTERNAL_H_ */

src/script/lua_api/l_mainmenu.cpp

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_mainmenu.h"
 #include "lua_api/l_internal.h"
 #include "common/c_content.h"
+#include "lua_api/l_async_events.h"
 #include "guiEngine.h"
 #include "guiMainMenu.h"
 #include "guiKeyChangeMenu.h"
@@ -200,9 +201,6 @@ int ModApiMainMenu::l_get_textlist_index(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_worlds(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<WorldSpec> worlds = getAvailableWorlds();
 
 	lua_newtable(L);
@@ -237,9 +235,6 @@ int ModApiMainMenu::l_get_worlds(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_games(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<SubgameSpec> games = getAvailableGames();
 
 	lua_newtable(L);
@@ -365,9 +360,6 @@ int ModApiMainMenu::l_get_modstore_details(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_modstore_list(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::string listtype = "local";
 
 	if (!lua_isnone(L,1)) {
@@ -421,9 +413,6 @@ int ModApiMainMenu::l_get_modstore_list(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_favorites(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::string listtype = "local";
 
 	if (!lua_isnone(L,1)) {
@@ -545,9 +534,6 @@ int ModApiMainMenu::l_get_favorites(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_delete_favorite(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<ServerListSpec> servers;
 
 	std::string listtype = "local";
@@ -599,9 +585,6 @@ int ModApiMainMenu::l_show_keys_menu(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_create_world(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	const char *name	= luaL_checkstring(L, 1);
 	int gameidx			= luaL_checkinteger(L,2) -1;
 
@@ -632,9 +615,6 @@ int ModApiMainMenu::l_create_world(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_delete_world(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	int worldidx	= luaL_checkinteger(L,1) -1;
 
 	std::vector<WorldSpec> worlds = getAvailableWorlds();
@@ -962,9 +942,6 @@ int ModApiMainMenu::l_sound_stop(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_download_file(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	const char *url    = luaL_checkstring(L, 1);
 	const char *target = luaL_checkstring(L, 2);
 
@@ -972,7 +949,7 @@ int ModApiMainMenu::l_download_file(lua_State *L)
 	std::string absolute_destination = fs::RemoveRelativePathComponents(target);
 
 	if (ModApiMainMenu::isMinetestPath(absolute_destination)) {
-		if (engine->downloadFile(url,absolute_destination)) {
+		if (GUIEngine::downloadFile(url,absolute_destination)) {
 			lua_pushboolean(L,true);
 			return 1;
 		}
@@ -990,6 +967,28 @@ int ModApiMainMenu::l_gettext(lua_State *L)
 	return 1;
 }
 
+/******************************************************************************/
+int ModApiMainMenu::l_do_async_callback(lua_State *L)
+{
+	GUIEngine* engine = getGuiEngine(L);
+
+	const char* serialized_fct_raw = luaL_checkstring(L, 1);
+	unsigned int lenght_fct = luaL_checkint(L, 2);
+
+	const char* serialized_params_raw = luaL_checkstring(L, 3);
+	unsigned int lenght_params = luaL_checkint(L, 4);
+
+	assert(serialized_fct_raw != 0);
+	assert(serialized_params_raw != 0);
+
+	std::string serialized_fct = std::string(serialized_fct_raw,lenght_fct);
+	std::string serialized_params = std::string(serialized_params_raw,lenght_params);
+
+	lua_pushinteger(L,engine->DoAsync(serialized_fct,serialized_params));
+
+	return 1;
+}
+
 /******************************************************************************/
 void ModApiMainMenu::Initialize(lua_State *L, int top)
 {
@@ -1024,4 +1023,27 @@ void ModApiMainMenu::Initialize(lua_State *L, int top)
 	API_FCT(sound_play);
 	API_FCT(sound_stop);
 	API_FCT(gettext);
+	API_FCT(do_async_callback);
+}
+
+/******************************************************************************/
+void ModApiMainMenu::InitializeAsync(AsyncEngine& engine)
+{
+
+	ASYNC_API_FCT(get_worlds);
+	ASYNC_API_FCT(get_games);
+	ASYNC_API_FCT(get_favorites);
+	ASYNC_API_FCT(get_modpath);
+	ASYNC_API_FCT(get_gamepath);
+	ASYNC_API_FCT(get_texturepath);
+	ASYNC_API_FCT(get_dirlist);
+	ASYNC_API_FCT(create_dir);
+	ASYNC_API_FCT(delete_dir);
+	ASYNC_API_FCT(copy_dir);
+	//ASYNC_API_FCT(extract_zip); //TODO remove dependency to GuiEngine
+	ASYNC_API_FCT(get_version);
+	ASYNC_API_FCT(download_file);
+	ASYNC_API_FCT(get_modstore_details);
+	ASYNC_API_FCT(get_modstore_list);
+	//ASYNC_API_FCT(gettext); (gettext lib isn't threadsafe)
 }

src/script/lua_api/l_mainmenu.h

@@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "lua_api/l_base.h"
 
+class AsyncEngine;
+
 /** Implementation of lua api support for mainmenu */
 class ModApiMainMenu : public ModApiBase {
 
@@ -125,6 +127,8 @@ private:
 
 	static int l_download_file(lua_State *L);
 
+	// async
+	static int l_do_async_callback(lua_State *L);
 
 public:
 	/**
@@ -134,6 +138,8 @@ public:
 	 */
 	static void Initialize(lua_State *L, int top);
 
+	static void InitializeAsync(AsyncEngine& engine);
+
 };
 
 #endif /* L_MAINMENU_H_ */

src/script/lua_api/l_util.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "lua_api/l_async_events.h"
 #include "debug.h"
 #include "log.h"
 #include "tool.h"
@@ -257,3 +258,18 @@ void ModApiUtil::Initialize(lua_State *L, int top)
 	API_FCT(is_yes);
 }
 
+void ModApiUtil::InitializeAsync(AsyncEngine& engine)
+{
+	ASYNC_API_FCT(debug);
+	ASYNC_API_FCT(log);
+
+	//ASYNC_API_FCT(setting_set);
+	ASYNC_API_FCT(setting_get);
+	//ASYNC_API_FCT(setting_setbool);
+	ASYNC_API_FCT(setting_getbool);
+	//ASYNC_API_FCT(setting_save);
+
+	ASYNC_API_FCT(parse_json);
+
+	ASYNC_API_FCT(is_yes);
+}

src/script/lua_api/l_util.h

@@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "lua_api/l_base.h"
 
+class AsyncEngine;
+
 class ModApiUtil : public ModApiBase {
 private:
 	/*
@@ -77,6 +79,8 @@ private:
 public:
 	static void Initialize(lua_State *L, int top);
 
+	static void InitializeAsync(AsyncEngine& engine);
+
 };
 
 #endif /* L_UTIL_H_ */

src/script/lua_api/marshall.c

@@ -0,0 +1,551 @@
+/*
+* lmarshal.c
+* A Lua library for serializing and deserializing Lua values
+* Richard Hundt <richardhundt@gmail.com>
+*
+* License: MIT
+*
+* Copyright (c) 2010 Richard Hundt
+*
+* Permission is hereby granted, free of charge, to any person
+* obtaining a copy of this software and associated documentation
+* files (the "Software"), to deal in the Software without
+* restriction, including without limitation the rights to use,
+* copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following
+* conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+* OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+
+
+#define MAR_TREF 1
+#define MAR_TVAL 2
+#define MAR_TUSR 3
+
+#define MAR_CHR 1
+#define MAR_I32 4
+#define MAR_I64 8
+
+#define MAR_MAGIC 0x8e
+#define SEEN_IDX  3
+
+typedef struct mar_Buffer {
+    size_t size;
+    size_t seek;
+    size_t head;
+    char*  data;
+} mar_Buffer;
+
+static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx);
+static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx);
+
+static void buf_init(lua_State *L, mar_Buffer *buf)
+{
+    buf->size = 128;
+    buf->seek = 0;
+    buf->head = 0;
+    if (!(buf->data = malloc(buf->size))) luaL_error(L, "Out of memory!");
+}
+
+static void buf_done(lua_State* L, mar_Buffer *buf)
+{
+    free(buf->data);
+}
+
+static int buf_write(lua_State* L, const char* str, size_t len, mar_Buffer *buf)
+{
+    if (len > UINT32_MAX) luaL_error(L, "buffer too long");
+    if (buf->size - buf->head < len) {
+        size_t new_size = buf->size << 1;
+        size_t cur_head = buf->head;
+        while (new_size - cur_head <= len) {
+            new_size = new_size << 1;
+        }
+        if (!(buf->data = realloc(buf->data, new_size))) {
+            luaL_error(L, "Out of memory!");
+        }
+        buf->size = new_size;
+    }
+    memcpy(&buf->data[buf->head], str, len);
+    buf->head += len;
+    return 0;
+}
+
+static const char* buf_read(lua_State *L, mar_Buffer *buf, size_t *len)
+{
+    if (buf->seek < buf->head) {
+        buf->seek = buf->head;
+        *len = buf->seek;
+        return buf->data;
+    }
+    *len = 0;
+    return NULL;
+}
+
+static void mar_encode_value(lua_State *L, mar_Buffer *buf, int val, size_t *idx)
+{
+    size_t l;
+    int val_type = lua_type(L, val);
+    lua_pushvalue(L, val);
+
+    buf_write(L, (void*)&val_type, MAR_CHR, buf);
+    switch (val_type) {
+    case LUA_TBOOLEAN: {
+        int int_val = lua_toboolean(L, -1);
+        buf_write(L, (void*)&int_val, MAR_CHR, buf);
+        break;
+    }
+    case LUA_TSTRING: {
+        const char *str_val = lua_tolstring(L, -1, &l);
+        buf_write(L, (void*)&l, MAR_I32, buf);
+        buf_write(L, str_val, l, buf);
+        break;
+    }
+    case LUA_TNUMBER: {
+        lua_Number num_val = lua_tonumber(L, -1);
+        buf_write(L, (void*)&num_val, MAR_I64, buf);
+        break;
+    }
+    case LUA_TTABLE: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            lua_pop(L, 1); /* pop nil */
+            if (luaL_getmetafield(L, -1, "__persist")) {
+                tag = MAR_TUSR;
+
+                lua_pushvalue(L, -2); /* self */
+                lua_call(L, 1, 1);
+                if (!lua_isfunction(L, -1)) {
+                    luaL_error(L, "__persist must return a function");
+                }
+
+                lua_remove(L, -2); /* __persist */
+
+                lua_newtable(L);
+                lua_pushvalue(L, -2); /* callback */
+                lua_rawseti(L, -2, 1);
+
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+                buf_write(L, rec_buf.data, rec_buf.head, buf);
+                buf_done(L, &rec_buf);
+                lua_pop(L, 1);
+            }
+            else {
+                tag = MAR_TVAL;
+
+                lua_pushvalue(L, -1);
+                lua_pushinteger(L, (*idx)++);
+                lua_rawset(L, SEEN_IDX);
+
+                lua_pushvalue(L, -1);
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+                lua_pop(L, 1);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+                buf_write(L, rec_buf.data,rec_buf.head, buf);
+                buf_done(L, &rec_buf);
+            }
+        }
+        break;
+    }
+    case LUA_TFUNCTION: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            int i;
+            lua_Debug ar;
+            lua_pop(L, 1); /* pop nil */
+
+            lua_pushvalue(L, -1);
+            lua_getinfo(L, ">nuS", &ar);
+            if (ar.what[0] != 'L') {
+                luaL_error(L, "attempt to persist a C function '%s'", ar.name);
+            }
+            tag = MAR_TVAL;
+            lua_pushvalue(L, -1);
+            lua_pushinteger(L, (*idx)++);
+            lua_rawset(L, SEEN_IDX);
+
+            lua_pushvalue(L, -1);
+            buf_init(L, &rec_buf);
+            lua_dump(L, (lua_Writer)buf_write, &rec_buf);
+
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+            buf_write(L, rec_buf.data, rec_buf.head, buf);
+            buf_done(L, &rec_buf);
+            lua_pop(L, 1);
+
+            lua_newtable(L);
+            for (i=1; i <= ar.nups; i++) {
+                lua_getupvalue(L, -2, i);
+                lua_rawseti(L, -2, i);
+            }
+
+            buf_init(L, &rec_buf);
+            mar_encode_table(L, &rec_buf, idx);
+
+            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+            buf_write(L, rec_buf.data, rec_buf.head, buf);
+            buf_done(L, &rec_buf);
+            lua_pop(L, 1);
+        }
+
+        break;
+    }
+    case LUA_TUSERDATA: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            lua_pop(L, 1); /* pop nil */
+            if (luaL_getmetafield(L, -1, "__persist")) {
+                tag = MAR_TUSR;
+
+                lua_pushvalue(L, -2);
+                lua_pushinteger(L, (*idx)++);
+                lua_rawset(L, SEEN_IDX);
+
+                lua_pushvalue(L, -2);
+                lua_call(L, 1, 1);
+                if (!lua_isfunction(L, -1)) {
+                    luaL_error(L, "__persist must return a function");
+                }
+                lua_newtable(L);
+                lua_pushvalue(L, -2);
+                lua_rawseti(L, -2, 1);
+                lua_remove(L, -2);
+
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+		buf_write(L, rec_buf.data, rec_buf.head, buf);
+		buf_done(L, &rec_buf);
+            }
+            else {
+                luaL_error(L, "attempt to encode userdata (no __persist hook)");
+            }
+            lua_pop(L, 1);
+        }
+        break;
+    }
+    case LUA_TNIL: break;
+    default:
+        luaL_error(L, "invalid value type (%s)", lua_typename(L, val_type));
+    }
+    lua_pop(L, 1);
+}
+
+static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx)
+{
+    lua_pushnil(L);
+    while (lua_next(L, -2) != 0) {
+        mar_encode_value(L, buf, -2, idx);
+        mar_encode_value(L, buf, -1, idx);
+        lua_pop(L, 1);
+    }
+    return 1;
+}
+
+#define mar_incr_ptr(l) \
+    if (((*p)-buf)+(l) > len) luaL_error(L, "bad code"); (*p) += (l);
+
+#define mar_next_len(l,T) \
+    if (((*p)-buf)+sizeof(T) > len) luaL_error(L, "bad code"); \
+    l = *(T*)*p; (*p) += sizeof(T);
+
+static void mar_decode_value
+    (lua_State *L, const char *buf, size_t len, const char **p, size_t *idx)
+{
+    size_t l;
+    char val_type = **p;
+    mar_incr_ptr(MAR_CHR);
+    switch (val_type) {
+    case LUA_TBOOLEAN:
+        lua_pushboolean(L, *(char*)*p);
+        mar_incr_ptr(MAR_CHR);
+        break;
+    case LUA_TNUMBER:
+        lua_pushnumber(L, *(lua_Number*)*p);
+        mar_incr_ptr(MAR_I64);
+        break;
+    case LUA_TSTRING:
+        mar_next_len(l, uint32_t);
+        lua_pushlstring(L, *p, l);
+        mar_incr_ptr(l);
+        break;
+    case LUA_TTABLE: {
+        char tag = *(char*)*p;
+        mar_incr_ptr(MAR_CHR);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else if (tag == MAR_TVAL) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_decode_table(L, *p, l, idx);
+            mar_incr_ptr(l);
+        }
+        else if (tag == MAR_TUSR) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            lua_rawgeti(L, -1, 1);
+            lua_call(L, 0, 1);
+            lua_remove(L, -2);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_incr_ptr(l);
+        }
+        else {
+            luaL_error(L, "bad encoded data");
+        }
+        break;
+    }
+    case LUA_TFUNCTION: {
+        size_t nups;
+        int i;
+        mar_Buffer dec_buf;
+        char tag = *(char*)*p;
+        mar_incr_ptr(1);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else {
+            mar_next_len(l, uint32_t);
+            dec_buf.data = (char*)*p;
+            dec_buf.size = l;
+            dec_buf.head = l;
+            dec_buf.seek = 0;
+            lua_load(L, (lua_Reader)buf_read, &dec_buf, "=marshal");
+            mar_incr_ptr(l);
+
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            nups = lua_objlen(L, -1);
+            for (i=1; i <= nups; i++) {
+                lua_rawgeti(L, -1, i);
+                lua_setupvalue(L, -3, i);
+            }
+            lua_pop(L, 1);
+            mar_incr_ptr(l);
+        }
+        break;
+    }
+    case LUA_TUSERDATA: {
+        char tag = *(char*)*p;
+        mar_incr_ptr(MAR_CHR);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else if (tag == MAR_TUSR) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            lua_rawgeti(L, -1, 1);
+            lua_call(L, 0, 1);
+            lua_remove(L, -2);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_incr_ptr(l);
+        }
+        else { /* tag == MAR_TVAL */
+            lua_pushnil(L);
+        }
+        break;
+    }
+    case LUA_TNIL:
+    case LUA_TTHREAD:
+        lua_pushnil(L);
+        break;
+    default:
+        luaL_error(L, "bad code");
+    }
+}
+
+static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx)
+{
+    const char* p;
+    p = buf;
+    while (p - buf < len) {
+        mar_decode_value(L, buf, len, &p, idx);
+        mar_decode_value(L, buf, len, &p, idx);
+        lua_settable(L, -3);
+    }
+    return 1;
+}
+
+static int mar_encode(lua_State* L)
+{
+    const unsigned char m = MAR_MAGIC;
+    size_t idx, len;
+    mar_Buffer buf;
+
+    if (lua_isnone(L, 1)) {
+        lua_pushnil(L);
+    }
+    if (lua_isnoneornil(L, 2)) {
+        lua_newtable(L);
+    }
+    else if (!lua_istable(L, 2)) {
+        luaL_error(L, "bad argument #2 to encode (expected table)");
+    }
+    lua_settop(L, 2);
+
+    len = lua_objlen(L, 2);
+    lua_newtable(L);
+    for (idx = 1; idx <= len; idx++) {
+        lua_rawgeti(L, 2, idx);
+        if (lua_isnil(L, -1)) {
+            lua_pop(L, 1);
+            continue;
+        }
+        lua_pushinteger(L, idx);
+        lua_rawset(L, SEEN_IDX);
+    }
+    lua_pushvalue(L, 1);
+
+    buf_init(L, &buf);
+    buf_write(L, (void*)&m, 1, &buf);
+
+    mar_encode_value(L, &buf, -1, &idx);
+
+    lua_pop(L, 1);
+
+    lua_pushlstring(L, buf.data, buf.head);
+
+    buf_done(L, &buf);
+
+    lua_remove(L, SEEN_IDX);
+
+    return 1;
+}
+
+static int mar_decode(lua_State* L)
+{
+    size_t l, idx, len;
+    const char *p;
+    const char *s = luaL_checklstring(L, 1, &l);
+
+    if (l < 1) luaL_error(L, "bad header");
+    if (*(unsigned char *)s++ != MAR_MAGIC) luaL_error(L, "bad magic");
+    l -= 1;
+
+    if (lua_isnoneornil(L, 2)) {
+        lua_newtable(L);
+    }
+    else if (!lua_istable(L, 2)) {
+        luaL_error(L, "bad argument #2 to decode (expected table)");
+    }
+    lua_settop(L, 2);
+
+    len = lua_objlen(L, 2);
+    lua_newtable(L);
+    for (idx = 1; idx <= len; idx++) {
+        lua_rawgeti(L, 2, idx);
+        lua_rawseti(L, SEEN_IDX, idx);
+    }
+
+    p = s;
+    mar_decode_value(L, s, l, &p, &idx);
+
+    lua_remove(L, SEEN_IDX);
+    lua_remove(L, 2);
+
+    return 1;
+}
+
+static int mar_clone(lua_State* L)
+{
+    mar_encode(L);
+    lua_replace(L, 1);
+    mar_decode(L);
+    return 1;
+}
+
+static const luaL_reg R[] =
+{
+    {"encode",      mar_encode},
+    {"decode",      mar_decode},
+    {"clone",       mar_clone},
+    {NULL,	    NULL}
+};
+
+int luaopen_marshal(lua_State *L)
+{
+    lua_newtable(L);
+    luaL_register(L, "marshal", R);
+    return 1;
+}
+
+
+
+
+