From 24f0eb6bc190476d43614eacdf7c23d7aa47b885 Mon Sep 17 00:00:00 2001 From: y5nw <37980625+y5nw@users.noreply.github.com> Date: Fri, 9 Feb 2024 17:17:16 +0100 Subject: [PATCH] Try to preserve metatable information in serialzed data --- builtin/common/serialize.lua | 71 ++++++++++++++++--- builtin/common/tests/serialize_spec.lua | 68 +++++++++++++++++- builtin/common/vector.lua | 4 ++ doc/lua_api.md | 2 +- .../mods/unittests/itemstack_equals.lua | 7 ++ 5 files changed, 140 insertions(+), 12 deletions(-) diff --git a/builtin/common/serialize.lua b/builtin/common/serialize.lua index 146128e0c..8a7974853 100644 --- a/builtin/common/serialize.lua +++ b/builtin/common/serialize.lua @@ -8,12 +8,23 @@ local next, rawget, pairs, pcall, error, type, setfenv, loadstring local table_concat, string_dump, string_format, string_match, math_huge = table.concat, string.dump, string.format, string.match, math.huge --- Recursively counts occurrences of objects (non-primitives including strings) in a table. -local function count_objects(value) +local itemstack_mt +if ItemStack then + itemstack_mt = getmetatable(ItemStack()) +end +local function is_itemstack(x) + return itemstack_mt and getmetatable(x) == itemstack_mt +end + +-- Recursively +-- (1) reads metatables from tables; +-- (2) counts occurrences of objects (non-primitives including strings) in a table. +local function prepare_objects(value) local counts = {} + local type_lookup = {} if value == nil then -- Early return for nil; tables can't contain nil - return counts + return counts, type_lookup end local function count_values(val) local type_ = type(val) @@ -22,19 +33,23 @@ local function count_objects(value) end local count = counts[val] counts[val] = (count or 0) + 1 + local mt = getmetatable(val) if type_ == "table" then if not count then for k, v in pairs(val) do count_values(k) count_values(v) end + if mt then + type_lookup[val] = core.known_metatables[mt] + end end - elseif type_ ~= "string" and type_ ~= "function" then + elseif type_ ~= "string" and type_ ~= "function" and not is_itemstack(val) then error("unsupported type: " .. type_) end end - count_values(value) - return counts + count_values(value, {}) + return counts, type_lookup end -- Build a "set" of Lua keywords. These can't be used as short key names. @@ -58,6 +73,10 @@ local function dump_func(func) return string_format("loadstring(%q)", string_dump(func)) end +local function dump_itemstack(item) + return string_format("ItemStack(%q)", item:to_string()) +end + -- Serializes Lua nil, booleans, numbers, strings, tables and even functions -- Tables are referenced by reference, strings are referenced by value. Supports circular tables. local function serialize(value, write) @@ -66,7 +85,11 @@ local function serialize(value, write) local references = {} -- Circular tables that must be filled using `table[key] = value` statements local to_fill = {} - for object, count in pairs(count_objects(value)) do + local counts, typenames = prepare_objects(value) + if next(typenames) then + write "if not setmetatable then core={known_metatables={}}; setmetatable = function(x) return x end; end;" + end + for object, count in pairs(counts) do local type_ = type(object) -- Object must appear more than once. If it is a string, the reference has to be shorter than the string. if count >= 2 and (type_ ~= "string" or #reference + 5 < #object) then @@ -82,10 +105,12 @@ local function serialize(value, write) write(dump_func(object)) elseif type_ == "string" then write(quote(object)) + elseif is_itemstack(object) then + write(dump_itemstack(object)) end write(";") references[object] = reference - if type_ == "table" then + if type_ ~= "string" and not is_itemstack(object) then to_fill[object] = reference end refnum = refnum + 1 @@ -96,7 +121,7 @@ local function serialize(value, write) local function use_short_key(key) return not references[key] and type(key) == "string" and (not keywords[key]) and string_match(key, "^[%a_][%a%d_]*$") end - local function dump(value) + local function dump(value, skip_mt) -- Primitive types if value == nil then return write("nil") @@ -126,12 +151,23 @@ local function serialize(value, write) write(ref) return write"]" end + if (not skip_mt) and typenames[value] then + write "setmetatable(" + dump(value, true) + write ",core.known_metatables[" + dump(typenames[value]) + write "] or {})" + return + end if type_ == "string" then return write(quote(value)) end if type_ == "function" then return write(dump_func(value)) end + if is_itemstack(value) then + return write(dump_itemstack(value)) + end if type_ == "table" then write("{") -- First write list keys: @@ -169,6 +205,7 @@ local function serialize(value, write) end -- Write the statements to fill circular tables for table, ref in pairs(to_fill) do + local typename = typenames[table] for k, v in pairs(table) do write("_[") write(ref) @@ -185,6 +222,13 @@ local function serialize(value, write) dump(v) write(";") end + if typename then + write("setmetatable(_[") + write(ref) + write("],core.known_metatables[") + dump(typename) + write("] or {})") + end end write("return ") dump(value) @@ -216,7 +260,13 @@ function core.deserialize(str, safe) if not func then return nil, err end -- math.huge was serialized to inf and NaNs to nan by Lua in engine version 5.6, so we have to support this here - local env = {inf = math_huge, nan = 0/0} + local env = { + inf = math_huge, + nan = 0/0, + ItemStack = ItemStack or function(str) return str end, + setmetatable = setmetatable, + core = { known_metatables = core.known_metatables } + } if safe then env.loadstring = dummy_func else @@ -236,3 +286,4 @@ function core.deserialize(str, safe) end return nil, value_or_err end + diff --git a/builtin/common/tests/serialize_spec.lua b/builtin/common/tests/serialize_spec.lua index d4e501468..5037976ad 100644 --- a/builtin/common/tests/serialize_spec.lua +++ b/builtin/common/tests/serialize_spec.lua @@ -4,6 +4,7 @@ _G.setfenv = require 'busted.compatibility'.setfenv dofile("builtin/common/serialize.lua") dofile("builtin/common/vector.lua") +dofile("builtin/common/metatable.lua") -- Supports circular tables; does not support table keys -- Correctly checks whether a mapping of references ("same") exists @@ -40,11 +41,32 @@ local t1, t2 = {x, x, y, y}, {x, y, x, y} assert.same(t1, t2) -- will succeed because it only checks whether the depths match assert(not pcall(assert_same, t1, t2)) -- will correctly fail because it checks whether the refs match +local pair_mt = { + __eq = function(x, y) + return x[1] == y[1] and x[2] == y[2] + end, +} +local function pair(x, y) + return setmetatable({x, y}, pair_mt) +end +-- Use our own serialization functions to avoid incorrectly passing test related to references. +core.register_metatable("pair", pair_mt) +assert.equals(pair(1, 2), pair(1, 2)) +assert.not_equals(pair(1, 2), pair(3, 4)) + describe("serialize", function() local function assert_preserves(value) local preserved_value = core.deserialize(core.serialize(value)) assert_same(value, preserved_value) end + local function assert_strictly_preserves(value) + local preserved_value = core.deserialize(core.serialize(value)) + assert.equals(value, preserved_value) + end + local function assert_compatibly_preserves(value) + local preserved_value = loadstring(core.serialize(value))() + assert_same(value, preserved_value) + end it("works", function() assert_preserves({cat={sound="nyan", speed=400}, dog={sound="woof"}}) end) @@ -53,6 +75,10 @@ describe("serialize", function() assert_preserves({escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"}) end) + it("handles nil", function() + assert_strictly_preserves(nil) + end) + it("handles NaN & infinities", function() local nan = core.deserialize(core.serialize(0/0)) assert(nan ~= nan) @@ -113,7 +139,10 @@ describe("serialize", function() it("vectors work", function() local v = vector.new(1, 2, 3) assert_preserves({v}) - assert_preserves(v) + assert_compatibly_preserves({v}) + assert_strictly_preserves(v) + assert_compatibly_preserves(v) + assert(core.deserialize(core.serialize(v)):check()) -- abuse v = vector.new(1, 2, 3) @@ -121,6 +150,43 @@ describe("serialize", function() assert_preserves(v) end) + it("correctly handles typed objects with multiple references", function() + local x, y = pair(1, 2), pair(1, 2) + local t = core.deserialize(core.serialize{x, x, y}) + assert.equals(x, t[1]) + assert.equals(x, t[3]) + assert(rawequal(t[1], t[2])) + assert(not rawequal(t[1], t[3])) + end) + + it("correctly handles recursive typed objects with the identity function as serializer", function() + local mt = { + __eq = function(x, y) + return x[1] == y[1] + end, + } + core.register_metatable("test_recursive_typed", mt) + local t = setmetatable({1}, mt) + t[2] = t + assert_strictly_preserves(t) + end) + + it("correctly handles binary trees", function() + local child = {pair(1, 1)} + local layers = 4 + for i = 2, layers do + child[i] = pair(child[i-1], child[i-1]) + end + local tree = child[layers] + assert_strictly_preserves(tree) + local node = core.deserialize(core.serialize(tree)) + for i = 2, layers do + assert(rawequal(node[1], node[2])) + node = node[1] + end + assert_compatibly_preserves(tree) + end) + it("handles keywords as keys", function() assert_preserves({["and"] = "keyword", ["for"] = "keyword"}) end) diff --git a/builtin/common/vector.lua b/builtin/common/vector.lua index 7a8558cbd..bbe4e6ad4 100644 --- a/builtin/common/vector.lua +++ b/builtin/common/vector.lua @@ -12,6 +12,10 @@ vector = {} local metatable = {} vector.metatable = metatable +if core and core.register_serializable then + core.register_serializable("__builtin:vector", metatable) +end + local xyz = {"x", "y", "z"} -- only called when rawget(v, key) returns nil diff --git a/doc/lua_api.md b/doc/lua_api.md index ba72e97e9..50a05f864 100644 --- a/doc/lua_api.md +++ b/doc/lua_api.md @@ -7655,7 +7655,7 @@ Misc. * `core.register_portable_metatable(name, mt)`: * Register a metatable that should be preserved when Lua data is transferred - between environments (via IPC or `handle_async`). + between environments (via IPC, `handle_async`, or `core.serialize`). * `name` is a string that identifies the metatable. It is recommended to follow the `modname:name` convention for this identifier. * `mt` is the metatable to register. diff --git a/games/devtest/mods/unittests/itemstack_equals.lua b/games/devtest/mods/unittests/itemstack_equals.lua index 561e612c4..ff74e562b 100644 --- a/games/devtest/mods/unittests/itemstack_equals.lua +++ b/games/devtest/mods/unittests/itemstack_equals.lua @@ -72,3 +72,10 @@ local function test_itemstack_equals_metadata() end unittests.register("test_itemstack_equals_metadata", test_itemstack_equals_metadata) + +local function test_itemstack_serialization_preservation() + local i = ItemStack("basenodes:stone 20 1000") + assert(i:equals(core.deserialize(core.serialize(i)))) +end + +unittests.register("test_itemstack_serialization_preservation", test_itemstack_serialization_preservation)