Try to preserve metatable information in serialzed data

builtin/common/serialize.lua

@@ -8,12 +8,23 @@ local next, rawget, pairs, pcall, error, type, setfenv, loadstring
 local table_concat, string_dump, string_format, string_match, math_huge
 	= table.concat, string.dump, string.format, string.match, math.huge
 
--- Recursively counts occurrences of objects (non-primitives including strings) in a table.
+local itemstack_mt
-local function count_objects(value)
+if ItemStack then
+	itemstack_mt = getmetatable(ItemStack())
+end
+local function is_itemstack(x)
+	return itemstack_mt and getmetatable(x) == itemstack_mt
+end
+
+-- Recursively
+-- (1) reads metatables from tables;
+-- (2) counts occurrences of objects (non-primitives including strings) in a table.
+local function prepare_objects(value)
 	local counts = {}
+	local type_lookup = {}
 	if value == nil then
 		-- Early return for nil; tables can't contain nil
-		return counts
+		return counts, type_lookup
 	end
 	local function count_values(val)
 		local type_ = type(val)
@@ -22,19 +33,23 @@ local function count_objects(value)
 		end
 		local count = counts[val]
 		counts[val] = (count or 0) + 1
+		local mt = getmetatable(val)
 		if type_ == "table" then
 			if not count then
 				for k, v in pairs(val) do
 					count_values(k)
 					count_values(v)
 				end
+				if mt then
+					type_lookup[val] = core.known_metatables[mt]
+				end
 			end
-		elseif type_ ~= "string" and type_ ~= "function" then
+		elseif type_ ~= "string" and type_ ~= "function" and not is_itemstack(val) then
 			error("unsupported type: " .. type_)
 		end
 	end
-	count_values(value)
+	count_values(value, {})
-	return counts
+	return counts, type_lookup
 end
 
 -- Build a "set" of Lua keywords. These can't be used as short key names.
@@ -58,6 +73,10 @@ local function dump_func(func)
 	return string_format("loadstring(%q)", string_dump(func))
 end
 
+local function dump_itemstack(item)
+	return string_format("ItemStack(%q)", item:to_string())
+end
+
 -- Serializes Lua nil, booleans, numbers, strings, tables and even functions
 -- Tables are referenced by reference, strings are referenced by value. Supports circular tables.
 local function serialize(value, write)
@@ -66,7 +85,11 @@ local function serialize(value, write)
 	local references = {}
 	-- Circular tables that must be filled using `table[key] = value` statements
 	local to_fill = {}
-	for object, count in pairs(count_objects(value)) do
+	local counts, typenames = prepare_objects(value)
+	if next(typenames) then
+		write "if not setmetatable then core={known_metatables={}}; setmetatable = function(x) return x end; end;"
+	end
+	for object, count in pairs(counts) do
 		local type_ = type(object)
 		-- Object must appear more than once. If it is a string, the reference has to be shorter than the string.
 		if count >= 2 and (type_ ~= "string" or #reference + 5 < #object) then
@@ -82,10 +105,12 @@ local function serialize(value, write)
 				write(dump_func(object))
 			elseif type_ == "string" then
 				write(quote(object))
+			elseif is_itemstack(object) then
+				write(dump_itemstack(object))
 			end
 			write(";")
 			references[object] = reference
-			if type_ == "table" then
+			if type_ ~= "string" and not is_itemstack(object) then
 				to_fill[object] = reference
 			end
 			refnum = refnum + 1
@@ -96,7 +121,7 @@ local function serialize(value, write)
 	local function use_short_key(key)
 		return not references[key] and type(key) == "string" and (not keywords[key]) and string_match(key, "^[%a_][%a%d_]*$")
 	end
-	local function dump(value)
+	local function dump(value, skip_mt)
 		-- Primitive types
 		if value == nil then
 			return write("nil")
@@ -126,12 +151,23 @@ local function serialize(value, write)
 			write(ref)
 			return write"]"
 		end
+		if (not skip_mt) and typenames[value] then
+			write "setmetatable("
+			dump(value, true)
+			write ",core.known_metatables["
+			dump(typenames[value])
+			write "] or {})"
+			return
+		end
 		if type_ == "string" then
 			return write(quote(value))
 		end
 		if type_ == "function" then
 			return write(dump_func(value))
 		end
+		if is_itemstack(value) then
+			return write(dump_itemstack(value))
+		end
 		if type_ == "table" then
 			write("{")
 			-- First write list keys:
@@ -169,6 +205,7 @@ local function serialize(value, write)
 	end
 	-- Write the statements to fill circular tables
 	for table, ref in pairs(to_fill) do
+		local typename = typenames[table]
 		for k, v in pairs(table) do
 			write("_[")
 			write(ref)
@@ -185,6 +222,13 @@ local function serialize(value, write)
 			dump(v)
 			write(";")
 		end
+		if typename then
+			write("setmetatable(_[")
+			write(ref)
+			write("],core.known_metatables[")
+			dump(typename)
+			write("] or {})")
+		end
 	end
 	write("return ")
 	dump(value)
@@ -216,7 +260,13 @@ function core.deserialize(str, safe)
 	if not func then return nil, err end
 
 	-- math.huge was serialized to inf and NaNs to nan by Lua in engine version 5.6, so we have to support this here
-	local env = {inf = math_huge, nan = 0/0}
+	local env = {
+		inf = math_huge,
+		nan = 0/0,
+		ItemStack = ItemStack or function(str) return str end,
+		setmetatable = setmetatable,
+		core = { known_metatables = core.known_metatables }
+	}
 	if safe then
 		env.loadstring = dummy_func
 	else
@@ -236,3 +286,4 @@ function core.deserialize(str, safe)
 	end
 	return nil, value_or_err
 end
+

builtin/common/tests/serialize_spec.lua

@@ -4,6 +4,7 @@ _G.setfenv = require 'busted.compatibility'.setfenv
 
 dofile("builtin/common/serialize.lua")
 dofile("builtin/common/vector.lua")
+dofile("builtin/common/metatable.lua")
 
 -- Supports circular tables; does not support table keys
 -- Correctly checks whether a mapping of references ("same") exists
@@ -40,11 +41,32 @@ local t1, t2 = {x, x, y, y}, {x, y, x, y}
 assert.same(t1, t2) -- will succeed because it only checks whether the depths match
 assert(not pcall(assert_same, t1, t2)) -- will correctly fail because it checks whether the refs match
 
+local pair_mt = {
+	__eq = function(x, y)
+		return x[1] == y[1] and x[2] == y[2]
+	end,
+}
+local function pair(x, y)
+	return setmetatable({x, y}, pair_mt)
+end
+-- Use our own serialization functions to avoid incorrectly passing test related to references.
+core.register_metatable("pair", pair_mt)
+assert.equals(pair(1, 2), pair(1, 2))
+assert.not_equals(pair(1, 2), pair(3, 4))
+
 describe("serialize", function()
 	local function assert_preserves(value)
 		local preserved_value = core.deserialize(core.serialize(value))
 		assert_same(value, preserved_value)
 	end
+	local function assert_strictly_preserves(value)
+		local preserved_value = core.deserialize(core.serialize(value))
+		assert.equals(value, preserved_value)
+	end
+	local function assert_compatibly_preserves(value)
+		local preserved_value = loadstring(core.serialize(value))()
+		assert_same(value, preserved_value)
+	end
 	it("works", function()
 		assert_preserves({cat={sound="nyan", speed=400}, dog={sound="woof"}})
 	end)
@@ -53,6 +75,10 @@ describe("serialize", function()
 		assert_preserves({escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"})
 	end)
 
+	it("handles nil", function()
+		assert_strictly_preserves(nil)
+	end)
+
 	it("handles NaN & infinities", function()
 		local nan = core.deserialize(core.serialize(0/0))
 		assert(nan ~= nan)
@@ -113,7 +139,10 @@ describe("serialize", function()
 	it("vectors work", function()
 		local v = vector.new(1, 2, 3)
 		assert_preserves({v})
-		assert_preserves(v)
+		assert_compatibly_preserves({v})
+		assert_strictly_preserves(v)
+		assert_compatibly_preserves(v)
+		assert(core.deserialize(core.serialize(v)):check())
 
 		-- abuse
 		v = vector.new(1, 2, 3)
@@ -121,6 +150,43 @@ describe("serialize", function()
 		assert_preserves(v)
 	end)
 
+	it("correctly handles typed objects with multiple references", function()
+		local x, y = pair(1, 2), pair(1, 2)
+		local t = core.deserialize(core.serialize{x, x, y})
+		assert.equals(x, t[1])
+		assert.equals(x, t[3])
+		assert(rawequal(t[1], t[2]))
+		assert(not rawequal(t[1], t[3]))
+	end)
+
+	it("correctly handles recursive typed objects with the identity function as serializer", function()
+		local mt = {
+			__eq = function(x, y)
+				return x[1] == y[1]
+			end,
+		}
+		core.register_metatable("test_recursive_typed", mt)
+		local t = setmetatable({1}, mt)
+		t[2] = t
+		assert_strictly_preserves(t)
+	end)
+
+	it("correctly handles binary trees", function()
+		local child = {pair(1, 1)}
+		local layers = 4
+		for i = 2, layers do
+			child[i] = pair(child[i-1], child[i-1])
+		end
+		local tree = child[layers]
+		assert_strictly_preserves(tree)
+		local node = core.deserialize(core.serialize(tree))
+		for i = 2, layers do
+			assert(rawequal(node[1], node[2]))
+			node = node[1]
+		end
+		assert_compatibly_preserves(tree)
+	end)
+
 	it("handles keywords as keys", function()
 		assert_preserves({["and"] = "keyword", ["for"] = "keyword"})
 	end)

builtin/common/vector.lua

@@ -12,6 +12,10 @@ vector = {}
 local metatable = {}
 vector.metatable = metatable
 
+if core and core.register_serializable then
+	core.register_serializable("__builtin:vector", metatable)
+end
+
 local xyz = {"x", "y", "z"}
 
 -- only called when rawget(v, key) returns nil

doc/lua_api.md

@@ -7655,7 +7655,7 @@ Misc.
 
 * `core.register_portable_metatable(name, mt)`:
     * Register a metatable that should be preserved when Lua data is transferred
-      between environments (via IPC or `handle_async`).
+      between environments (via IPC, `handle_async`, or `core.serialize`).
     * `name` is a string that identifies the metatable. It is recommended to
       follow the `modname:name` convention for this identifier.
     * `mt` is the metatable to register.

games/devtest/mods/unittests/itemstack_equals.lua

@@ -72,3 +72,10 @@ local function test_itemstack_equals_metadata()
 end
 
 unittests.register("test_itemstack_equals_metadata", test_itemstack_equals_metadata)
+
+local function test_itemstack_serialization_preservation()
+	local i = ItemStack("basenodes:stone 20 1000")
+	assert(i:equals(core.deserialize(core.serialize(i))))
+end
+
+unittests.register("test_itemstack_serialization_preservation", test_itemstack_serialization_preservation)