diff --git a/builtin/game/misc_s.lua b/builtin/game/misc_s.lua index e64134e15..9433f74bb 100644 --- a/builtin/game/misc_s.lua +++ b/builtin/game/misc_s.lua @@ -129,6 +129,7 @@ core.protocol_versions = { ["5.9.1"] = 45, ["5.10.0"] = 46, ["5.11.0"] = 47, + ["5.12.0"] = 48, } setmetatable(core.protocol_versions, {__newindex = function() diff --git a/games/devtest/mods/unittests/version.lua b/games/devtest/mods/unittests/version.lua index baf4520a4..73b4e5a11 100644 --- a/games/devtest/mods/unittests/version.lua +++ b/games/devtest/mods/unittests/version.lua @@ -35,6 +35,6 @@ unittests.register("test_protocol_version", function(player) -- The protocol version the client and server agreed on must exist in the table. local match = table.key_value_swap(core.protocol_versions)[info.protocol_version] - assert(match ~= nil) print(string.format("client proto matched: %s sent: %s", match, info.version_string)) + assert(match ~= nil) end, {player = true}) diff --git a/src/client/client.cpp b/src/client/client.cpp index 08fd2a215..031bed8d7 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -902,7 +902,6 @@ void Client::request_media(const std::vector &file_requests) FATAL_ERROR_IF(file_requests_size > 0xFFFF, "Unsupported number of file requests"); - // Packet dynamicly resized NetworkPacket pkt(TOSERVER_REQUEST_MEDIA, 2 + 0); pkt << (u16) (file_requests_size & 0xFFFF); diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index fe490dd8c..bb1930d96 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -606,10 +606,6 @@ void Client::handleCommand_DeathScreenLegacy(NetworkPacket* pkt) void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt) { - u16 num_files; - - *pkt >> num_files; - infostream << "Client: Received media announcement: packet size: " << pkt->getSize() << std::endl; @@ -619,9 +615,7 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt) "we already saw another announcement" : "all media has been received already"; errorstream << "Client: Received media announcement but " - << problem << "! " - << " files=" << num_files - << " size=" << pkt->getSize() << std::endl; + << problem << "!" << std::endl; return; } @@ -629,16 +623,36 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt) // updating content definitions sanity_check(!m_mesh_update_manager->isRunning()); - for (u16 i = 0; i < num_files; i++) { + if (m_proto_ver >= 48) { + // compressed table of media names + std::vector names; + { + std::istringstream iss(pkt->readLongString(), std::ios::binary); + std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary); + decompressZstd(iss, ss); + names = deserializeString16Array(ss); + } + + // raw hash for each media file + for (auto &name : names) { + auto sha1_raw = pkt->readRawString(20); + m_media_downloader->addFile(name, sha1_raw); + } + } else { + u16 num_files; + *pkt >> num_files; + std::string name, sha1_base64; + for (u16 i = 0; i < num_files; i++) { + *pkt >> name >> sha1_base64; - *pkt >> name >> sha1_base64; - - std::string sha1_raw = base64_decode(sha1_base64); - m_media_downloader->addFile(name, sha1_raw); + std::string sha1_raw = base64_decode(sha1_base64); + m_media_downloader->addFile(name, sha1_raw); + } } { + // Remote media servers std::string str; *pkt >> str; @@ -657,18 +671,6 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt) void Client::handleCommand_Media(NetworkPacket* pkt) { - /* - u16 command - u16 total number of file bunches - u16 index of this bunch - u32 number of files in this bunch - for each file { - u16 length of name - string name - u32 length of data - data - } - */ u16 num_bunches; u16 bunch_i; u32 num_files; @@ -695,6 +697,12 @@ void Client::handleCommand_Media(NetworkPacket* pkt) *pkt >> name; data = pkt->readLongString(); + if (m_proto_ver >= 48) { + std::istringstream iss(data, std::ios::binary); + std::ostringstream oss(std::ios::binary); + decompressZstd(iss, oss); + data = oss.str(); + } bool ok = false; if (init_phase) { @@ -729,7 +737,10 @@ void Client::handleCommand_NodeDef(NetworkPacket* pkt) // Decompress node definitions std::istringstream tmp_is(pkt->readLongString(), std::ios::binary); std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out); - decompressZlib(tmp_is, tmp_os); + if (m_proto_ver >= 48) + decompressZstd(tmp_is, tmp_os); + else + decompressZlib(tmp_is, tmp_os); // Deserialize node definitions m_nodedef->deSerialize(tmp_os, m_proto_ver); @@ -748,7 +759,10 @@ void Client::handleCommand_ItemDef(NetworkPacket* pkt) // Decompress item definitions std::istringstream tmp_is(pkt->readLongString(), std::ios::binary); std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out); - decompressZlib(tmp_is, tmp_os); + if (m_proto_ver >= 48) + decompressZstd(tmp_is, tmp_os); + else + decompressZlib(tmp_is, tmp_os); // Deserialize node definitions m_itemdef->deSerialize(tmp_os, m_proto_ver); diff --git a/src/network/networkpacket.cpp b/src/network/networkpacket.cpp index f736d877b..ca7c53f5f 100644 --- a/src/network/networkpacket.cpp +++ b/src/network/networkpacket.cpp @@ -69,6 +69,18 @@ void NetworkPacket::putRawString(const char* src, u32 len) m_read_offset += len; } +void NetworkPacket::readRawString(char *dst, u32 len) +{ + checkReadOffset(m_read_offset, len); + + if (len == 0) + return; + + memcpy(dst, &m_data[m_read_offset], len); + m_read_offset += len; +} + + NetworkPacket& NetworkPacket::operator>>(std::string& dst) { checkReadOffset(m_read_offset, 2); diff --git a/src/network/networkpacket.h b/src/network/networkpacket.h index 32d378f47..d5e687c68 100644 --- a/src/network/networkpacket.h +++ b/src/network/networkpacket.h @@ -51,6 +51,16 @@ public: putRawString(src.data(), src.size()); } + // Reads bytes from packet into string buffer + void readRawString(char *dst, u32 len); + std::string readRawString(u32 len) + { + std::string s; + s.resize(len); + readRawString(&s[0], len); + return s; + } + NetworkPacket &operator>>(std::string &dst); NetworkPacket &operator<<(std::string_view src); diff --git a/src/network/networkprotocol.cpp b/src/network/networkprotocol.cpp index 85096930f..d1cc245d7 100644 --- a/src/network/networkprotocol.cpp +++ b/src/network/networkprotocol.cpp @@ -62,10 +62,13 @@ PROTOCOL VERSION 47 Add particle blend mode "clip" [scheduled bump for 5.11.0] + PROTOCOL VERSION 48 + Add compression to some existing packets + [scheduled bump for 5.12.0] */ // Note: Also update core.protocol_versions in builtin when bumping -const u16 LATEST_PROTOCOL_VERSION = 47; +const u16 LATEST_PROTOCOL_VERSION = 48; // See also formspec [Version History] in doc/lua_api.md const u16 FORMSPEC_API_VERSION = 8; diff --git a/src/network/networkprotocol.h b/src/network/networkprotocol.h index a0fa6b96d..152534cbe 100644 --- a/src/network/networkprotocol.h +++ b/src/network/networkprotocol.h @@ -33,24 +33,28 @@ enum ToClientCommand : u16 u32 supported auth methods std::string unused (used to be username) */ + TOCLIENT_AUTH_ACCEPT = 0x03, /* Message from server to accept auth. - v3s16 player's position + v3f(0,BS/2,0) floatToInt'd + v3f unused u64 map seed f1000 recommended send interval u32 : supported auth methods for sudo mode (where the user can change their password) */ + TOCLIENT_ACCEPT_SUDO_MODE = 0x04, /* Sent to client to show it is in sudo mode now. */ + TOCLIENT_DENY_SUDO_MODE = 0x05, /* Signals client that sudo mode auth failed. */ + TOCLIENT_ACCESS_DENIED = 0x0A, /* u8 reason @@ -59,18 +63,26 @@ enum ToClientCommand : u16 */ TOCLIENT_BLOCKDATA = 0x20, + /* + v3s16 position + serialized MapBlock + */ + TOCLIENT_ADDNODE = 0x21, /* v3s16 position serialized mapnode - u8 keep_metadata // Added in protocol version 22 + u8 keep_metadata */ + TOCLIENT_REMOVENODE = 0x22, + /* + v3s16 position + */ TOCLIENT_INVENTORY = 0x27, /* - [0] u16 command - [2] serialized inventory + serialized inventory */ TOCLIENT_TIME_OF_DAY = 0x29, @@ -167,40 +179,38 @@ enum ToClientCommand : u16 TOCLIENT_MEDIA = 0x38, /* - u16 total number of texture bunches + u16 total number of bunches u16 index of this bunch u32 number of files in this bunch for each file { u16 length of name string name u32 length of data - data + data (zstd-compressed) } - u16 length of remote media server url (if applicable) - string url */ TOCLIENT_NODEDEF = 0x3a, /* - u32 length of the next item - serialized NodeDefManager + u32 length of buffer + serialized NodeDefManager (zstd-compressed) */ TOCLIENT_ANNOUNCE_MEDIA = 0x3c, /* - u32 number of files - for each texture { - u16 length of name - string name - u16 length of sha1_digest - string sha1_digest + u32 length of compressed name array + string16array names (zstd-compressed) + for each file { + char[20] sha1_digest } + u16 length of remote media server url + string url */ TOCLIENT_ITEMDEF = 0x3d, /* - u32 length of next item - serialized ItemDefManager + u32 length of buffer + serialized ItemDefManager (zstd-compressed) */ TOCLIENT_PLAY_SOUND = 0x3f, @@ -721,18 +731,16 @@ enum ToServerCommand : u16 TOSERVER_PLAYERPOS = 0x23, /* - [0] u16 command - [2] v3s32 position*100 - [2+12] v3s32 speed*100 - [2+12+12] s32 pitch*100 - [2+12+12+4] s32 yaw*100 - [2+12+12+4+4] u32 keyPressed - [2+12+12+4+4+4] u8 fov*80 - [2+12+12+4+4+4+1] u8 ceil(wanted_range / MAP_BLOCKSIZE) - [2+12+12+4+4+4+1+1] u8 camera_inverted (bool) - [2+12+12+4+4+4+1+1+1] f32 movement_speed - [2+12+12+4+4+4+1+1+1+4] f32 movement_direction - + v3s32 position*100 + v3s32 speed*100 + s32 pitch*100 + s32 yaw*100 + u32 keyPressed + u8 fov*80 + u8 ceil(wanted_range / MAP_BLOCKSIZE) + u8 camera_inverted (bool) + f32 movement_speed + f32 movement_direction */ TOSERVER_GOTBLOCKS = 0x24, diff --git a/src/server.cpp b/src/server.cpp index 16611843f..af6fed3a0 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1487,17 +1487,20 @@ void Server::SendAccessDenied(session_t peer_id, AccessDeniedCode reason, void Server::SendItemDef(session_t peer_id, IItemDefManager *itemdef, u16 protocol_version) { + auto *client = m_clients.getClientNoEx(peer_id, CS_Created); + assert(client); + NetworkPacket pkt(TOCLIENT_ITEMDEF, 0, peer_id); - /* - u16 command - u32 length of the next item - zlib-compressed serialized ItemDefManager - */ - std::ostringstream tmp_os(std::ios::binary); - itemdef->serialize(tmp_os, protocol_version); std::ostringstream tmp_os2(std::ios::binary); - compressZlib(tmp_os.str(), tmp_os2); + { + std::ostringstream tmp_os(std::ios::binary); + itemdef->serialize(tmp_os, protocol_version); + if (client->net_proto_version >= 48) + compressZstd(tmp_os.str(), tmp_os2); + else + compressZlib(tmp_os.str(), tmp_os2); + } pkt.putLongString(tmp_os2.str()); // Make data buffer @@ -1510,18 +1513,20 @@ void Server::SendItemDef(session_t peer_id, void Server::SendNodeDef(session_t peer_id, const NodeDefManager *nodedef, u16 protocol_version) { + auto *client = m_clients.getClientNoEx(peer_id, CS_Created); + assert(client); + NetworkPacket pkt(TOCLIENT_NODEDEF, 0, peer_id); - /* - u16 command - u32 length of the next item - zlib-compressed serialized NodeDefManager - */ - std::ostringstream tmp_os(std::ios::binary); - nodedef->serialize(tmp_os, protocol_version); std::ostringstream tmp_os2(std::ios::binary); - compressZlib(tmp_os.str(), tmp_os2); - + { + std::ostringstream tmp_os(std::ios::binary); + nodedef->serialize(tmp_os, protocol_version); + if (client->net_proto_version >= 48) + compressZstd(tmp_os.str(), tmp_os2); + else + compressZlib(tmp_os.str(), tmp_os2); + } pkt.putLongString(tmp_os2.str()); // Make data buffer @@ -2583,13 +2588,12 @@ bool Server::addMediaFile(const std::string &filename, } std::string sha1 = hashing::sha1(filedata); - std::string sha1_base64 = base64_encode(sha1); std::string sha1_hex = hex_encode(sha1); if (digest_to) *digest_to = sha1; // Put in list - m_media[filename] = MediaInfo(filepath, sha1_base64); + m_media[filename] = MediaInfo(filepath, sha1); verbosestream << "Server: " << sha1_hex << " is " << filename << std::endl; @@ -2651,20 +2655,48 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co }; // Make packet + auto *client = m_clients.getClientNoEx(peer_id, CS_Created); + assert(client); NetworkPacket pkt(TOCLIENT_ANNOUNCE_MEDIA, 0, peer_id); - u16 media_sent = 0; - for (const auto &i : m_media) { - if (include(i.first, i.second)) - media_sent++; - } - pkt << media_sent; + size_t media_sent = 0; + if (client->net_proto_version < 48) { + for (const auto &i : m_media) { + if (include(i.first, i.second)) + media_sent++; + } + assert(media_sent < U16_MAX); + pkt << static_cast(media_sent); + for (const auto &i : m_media) { + if (include(i.first, i.second)) + pkt << i.first << base64_encode(i.second.sha1_digest); + } + } else { + std::vector names; + for (const auto &i : m_media) { + if (include(i.first, i.second)) + names.emplace_back(i.first); + } + media_sent = names.size(); - for (const auto &i : m_media) { - if (include(i.first, i.second)) - pkt << i.first << i.second.sha1_digest; + // compressed table of media names + { + std::ostringstream oss(std::ios::binary); + auto tmp = serializeString16Array(names); + compressZstd(tmp, oss); + pkt.putLongString(oss.str()); + } + + // then the raw hash for each file + for (const auto &i : m_media) { + if (include(i.first, i.second)) { + assert(i.second.sha1_digest.size() == 20); + pkt.putRawString(i.second.sha1_digest); + } + } } + // and the remote media server(s) pkt << g_settings->get("remote_media"); Send(&pkt); @@ -2694,8 +2726,11 @@ void Server::sendRequestedMedia(session_t peer_id, auto *client = getClient(peer_id, CS_DefinitionsSent); assert(client); + const bool compress = client->net_proto_version >= 48; + infostream << "Server::sendRequestedMedia(): Sending " - << tosend.size() << " files to " << client->getName() << std::endl; + << tosend.size() << " files to " << client->getName() + << (compress ? " (compressed)" : "") << std::endl; /* Read files and prepare bunches */ @@ -2713,6 +2748,7 @@ void Server::sendRequestedMedia(session_t peer_id, // the amount of bunches quite well (at the expense of overshooting). u32 file_size_bunch_total = 0; + size_t bytes_compressed = 0, bytes_uncompressed = 0; for (const std::string &name : tosend) { auto it = m_media.find(name); @@ -2739,9 +2775,19 @@ void Server::sendRequestedMedia(session_t peer_id, if (!fs::ReadFile(m.path, data, true)) { continue; } - file_size_bunch_total += data.size(); + bytes_uncompressed += data.size(); + if (compress) { + // Zstd is very fast and can handle non-compressible data efficiently + // so we can just throw it at every file. Still we don't want to + // spend too much here, so we use the lowest compression level. + std::ostringstream oss(std::ios::binary); + compressZstd(data, oss, 1); + data = oss.str(); + } + bytes_compressed += data.size(); // Put in list + file_size_bunch_total += data.size(); file_bunches.back().emplace_back(name, m.path, std::move(data)); // Start next bunch if got enough data @@ -2756,17 +2802,6 @@ void Server::sendRequestedMedia(session_t peer_id, const u16 num_bunches = file_bunches.size(); for (u16 i = 0; i < num_bunches; i++) { auto &bunch = file_bunches[i]; - /* - u16 total number of media bunches - u16 index of this bunch - u32 number of files in this bunch - for each file { - u16 length of name - string name - u32 length of data - data - } - */ NetworkPacket pkt(TOCLIENT_MEDIA, 4 + 0, peer_id); const u32 bunch_size = bunch.size(); @@ -2784,6 +2819,14 @@ void Server::sendRequestedMedia(session_t peer_id, << " size=" << pkt.getSize() << std::endl; Send(&pkt); } + + if (compress && bytes_uncompressed != 0) { + int percent = bytes_compressed / (float)bytes_uncompressed * 100; + int diff = (int)bytes_compressed - (int)bytes_uncompressed; + infostream << "Server::sendRequestedMedia(): size after compression " + << percent << "% (" << (diff > 0 ? '+' : '-') << std::abs(diff) + << " byte)" << std::endl; + } } void Server::stepPendingDynMediaCallbacks(float dtime) @@ -4210,7 +4253,7 @@ std::unordered_map Server::getMediaList() for (auto &it : m_media) { if (it.second.no_announce) continue; - ret.emplace(base64_decode(it.second.sha1_digest), it.second.path); + ret.emplace(it.second.sha1_digest, it.second.path); } return ret; } diff --git a/src/server.h b/src/server.h index 74f192195..c9869e1dd 100644 --- a/src/server.h +++ b/src/server.h @@ -90,7 +90,7 @@ enum ClientDeletionReason { struct MediaInfo { std::string path; - std::string sha1_digest; // base64-encoded + std::string sha1_digest; // true = not announced in TOCLIENT_ANNOUNCE_MEDIA (at player join) bool no_announce; // does what it says. used by some cases of dynamic media. diff --git a/src/util/serialize.cpp b/src/util/serialize.cpp index 257991008..e7a002662 100644 --- a/src/util/serialize.cpp +++ b/src/util/serialize.cpp @@ -105,6 +105,67 @@ std::string deSerializeString32(std::istream &is) return s; } +//// +//// String Array +//// + +std::string serializeString16Array(const std::vector &array) +{ + std::string ret; + const auto &at = [&] (size_t index) { + return reinterpret_cast(&ret[index]); + }; + + if (array.size() > U32_MAX) + throw SerializationError("serializeString16Array: too many strings"); + ret.resize(4 + array.size() * 2); + writeU32(at(0), array.size()); + + // Serialize lengths next to each other + size_t total = 0; + for (u32 i = 0; i < array.size(); i++) { + auto &s = array[i]; + if (s.size() > STRING_MAX_LEN) + throw SerializationError("serializeString16Array: string too long"); + writeU16(at(4 + 2*i), s.size()); + total += s.size(); + } + + // Now the contents + ret.reserve(ret.size() + total); + for (auto &s : array) + ret.append(s); + + return ret; +} + +std::vector deserializeString16Array(std::istream &is) +{ + std::vector ret; + + u32 count = readU32(is); + if (is.gcount() != 4) + throw SerializationError("deserializeString16Array: count not read"); + ret.resize(count); + + // prepare string buffers as we read the sizes + for (auto &sbuf : ret) { + u16 size = readU16(is); + if (is.gcount() != 2) + throw SerializationError("deserializeString16Array: size not read"); + sbuf.resize(size); + } + + // now extract the strings + for (auto &sbuf : ret) { + is.read(sbuf.data(), sbuf.size()); + if (is.gcount() != (std::streamsize) sbuf.size()) + throw SerializationError("deserializeString16Array: truncated"); + } + + return ret; +} + //// //// JSON-like strings //// diff --git a/src/util/serialize.h b/src/util/serialize.h index 8247eeb3e..c2cfa601d 100644 --- a/src/util/serialize.h +++ b/src/util/serialize.h @@ -469,3 +469,10 @@ std::string serializeJsonStringIfNeeded(std::string_view s); // Parses a string serialized by serializeJsonStringIfNeeded. std::string deSerializeJsonStringIfNeeded(std::istream &is); + +// Serializes an array of strings (max 2^16 chars each) +// Output is well suited for compression :) +std::string serializeString16Array(const std::vector &array); + +// Deserializes a string array +std::vector deserializeString16Array(std::istream &is);