Clean up and compress some pre-join packets (#15881)

builtin/game/misc_s.lua

@@ -129,6 +129,7 @@ core.protocol_versions = {
 	["5.9.1"] = 45,
 	["5.10.0"] = 46,
 	["5.11.0"] = 47,
+	["5.12.0"] = 48,
 }
 
 setmetatable(core.protocol_versions, {__newindex = function()

games/devtest/mods/unittests/version.lua

@@ -35,6 +35,6 @@ unittests.register("test_protocol_version", function(player)
 
 	-- The protocol version the client and server agreed on must exist in the table.
 	local match = table.key_value_swap(core.protocol_versions)[info.protocol_version]
-	assert(match ~= nil)
 	print(string.format("client proto matched: %s sent: %s", match, info.version_string))
+	assert(match ~= nil)
 end, {player = true})

src/client/client.cpp

@@ -902,7 +902,6 @@ void Client::request_media(const std::vector<std::string> &file_requests)
 
 	FATAL_ERROR_IF(file_requests_size > 0xFFFF, "Unsupported number of file requests");
 
-	// Packet dynamicly resized
 	NetworkPacket pkt(TOSERVER_REQUEST_MEDIA, 2 + 0);
 
 	pkt << (u16) (file_requests_size & 0xFFFF);

src/network/clientpackethandler.cpp

@@ -606,10 +606,6 @@ void Client::handleCommand_DeathScreenLegacy(NetworkPacket* pkt)
 
 void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 {
-	u16 num_files;
-
-	*pkt >> num_files;
-
 	infostream << "Client: Received media announcement: packet size: "
 			<< pkt->getSize() << std::endl;
 
@@ -619,9 +615,7 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 			"we already saw another announcement" :
 			"all media has been received already";
 		errorstream << "Client: Received media announcement but "
-			<< problem << "! "
-			<< " files=" << num_files
-			<< " size=" << pkt->getSize() << std::endl;
+			<< problem << "!" << std::endl;
 		return;
 	}
 
@@ -629,16 +623,36 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 	// updating content definitions
 	sanity_check(!m_mesh_update_manager->isRunning());
 
-	for (u16 i = 0; i < num_files; i++) {
+	if (m_proto_ver >= 48) {
+		// compressed table of media names
+		std::vector<std::string> names;
+		{
+			std::istringstream iss(pkt->readLongString(), std::ios::binary);
+			std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
+			decompressZstd(iss, ss);
+			names = deserializeString16Array(ss);
+		}
+
+		// raw hash for each media file
+		for (auto &name : names) {
+			auto sha1_raw = pkt->readRawString(20);
+			m_media_downloader->addFile(name, sha1_raw);
+		}
+	} else {
+		u16 num_files;
+		*pkt >> num_files;
+
 		std::string name, sha1_base64;
+		for (u16 i = 0; i < num_files; i++) {
+			*pkt >> name >> sha1_base64;
 
-		*pkt >> name >> sha1_base64;
-
-		std::string sha1_raw = base64_decode(sha1_base64);
-		m_media_downloader->addFile(name, sha1_raw);
+			std::string sha1_raw = base64_decode(sha1_base64);
+			m_media_downloader->addFile(name, sha1_raw);
+		}
 	}
 
 	{
+		// Remote media servers
 		std::string str;
 		*pkt >> str;
 
@@ -657,18 +671,6 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 
 void Client::handleCommand_Media(NetworkPacket* pkt)
 {
-	/*
-		u16 command
-		u16 total number of file bunches
-		u16 index of this bunch
-		u32 number of files in this bunch
-		for each file {
-			u16 length of name
-			string name
-			u32 length of data
-			data
-		}
-	*/
 	u16 num_bunches;
 	u16 bunch_i;
 	u32 num_files;
@@ -695,6 +697,12 @@ void Client::handleCommand_Media(NetworkPacket* pkt)
 
 		*pkt >> name;
 		data = pkt->readLongString();
+		if (m_proto_ver >= 48) {
+			std::istringstream iss(data, std::ios::binary);
+			std::ostringstream oss(std::ios::binary);
+			decompressZstd(iss, oss);
+			data = oss.str();
+		}
 
 		bool ok = false;
 		if (init_phase) {
@@ -729,7 +737,10 @@ void Client::handleCommand_NodeDef(NetworkPacket* pkt)
 	// Decompress node definitions
 	std::istringstream tmp_is(pkt->readLongString(), std::ios::binary);
 	std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out);
-	decompressZlib(tmp_is, tmp_os);
+	if (m_proto_ver >= 48)
+		decompressZstd(tmp_is, tmp_os);
+	else
+		decompressZlib(tmp_is, tmp_os);
 
 	// Deserialize node definitions
 	m_nodedef->deSerialize(tmp_os, m_proto_ver);
@@ -748,7 +759,10 @@ void Client::handleCommand_ItemDef(NetworkPacket* pkt)
 	// Decompress item definitions
 	std::istringstream tmp_is(pkt->readLongString(), std::ios::binary);
 	std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out);
-	decompressZlib(tmp_is, tmp_os);
+	if (m_proto_ver >= 48)
+		decompressZstd(tmp_is, tmp_os);
+	else
+		decompressZlib(tmp_is, tmp_os);
 
 	// Deserialize node definitions
 	m_itemdef->deSerialize(tmp_os, m_proto_ver);

src/network/networkpacket.cpp

@@ -69,6 +69,18 @@ void NetworkPacket::putRawString(const char* src, u32 len)
 	m_read_offset += len;
 }
 
+void NetworkPacket::readRawString(char *dst, u32 len)
+{
+	checkReadOffset(m_read_offset, len);
+
+	if (len == 0)
+		return;
+
+	memcpy(dst, &m_data[m_read_offset], len);
+	m_read_offset += len;
+}
+
+
 NetworkPacket& NetworkPacket::operator>>(std::string& dst)
 {
 	checkReadOffset(m_read_offset, 2);

src/network/networkpacket.h

@@ -51,6 +51,16 @@ public:
 		putRawString(src.data(), src.size());
 	}
 
+	// Reads bytes from packet into string buffer
+	void readRawString(char *dst, u32 len);
+	std::string readRawString(u32 len)
+	{
+		std::string s;
+		s.resize(len);
+		readRawString(&s[0], len);
+		return s;
+	}
+
 	NetworkPacket &operator>>(std::string &dst);
 	NetworkPacket &operator<<(std::string_view src);
 

src/network/networkprotocol.cpp

@@ -62,10 +62,13 @@
 	PROTOCOL VERSION 47
 		Add particle blend mode "clip"
 		[scheduled bump for 5.11.0]
+	PROTOCOL VERSION 48
+		Add compression to some existing packets
+		[scheduled bump for 5.12.0]
 */
 
 // Note: Also update core.protocol_versions in builtin when bumping
-const u16 LATEST_PROTOCOL_VERSION = 47;
+const u16 LATEST_PROTOCOL_VERSION = 48;
 
 // See also formspec [Version History] in doc/lua_api.md
 const u16 FORMSPEC_API_VERSION = 8;

src/network/networkprotocol.h

@@ -33,24 +33,28 @@ enum ToClientCommand : u16
 		u32 supported auth methods
 		std::string unused (used to be username)
 	*/
+
 	TOCLIENT_AUTH_ACCEPT = 0x03,
 	/*
 		Message from server to accept auth.
 
-		v3s16 player's position + v3f(0,BS/2,0) floatToInt'd
+		v3f unused
 		u64 map seed
 		f1000 recommended send interval
 		u32 : supported auth methods for sudo mode
 		      (where the user can change their password)
 	*/
+
 	TOCLIENT_ACCEPT_SUDO_MODE = 0x04,
 	/*
 		Sent to client to show it is in sudo mode now.
 	*/
+
 	TOCLIENT_DENY_SUDO_MODE = 0x05,
 	/*
 		Signals client that sudo mode auth failed.
 	*/
+
 	TOCLIENT_ACCESS_DENIED = 0x0A,
 	/*
 		u8 reason
@@ -59,18 +63,26 @@ enum ToClientCommand : u16
 	*/
 
 	TOCLIENT_BLOCKDATA = 0x20,
+	/*
+		v3s16 position
+		serialized MapBlock
+	*/
+
 	TOCLIENT_ADDNODE = 0x21,
 	/*
 		v3s16 position
 		serialized mapnode
-		u8 keep_metadata // Added in protocol version 22
+		u8 keep_metadata
 	*/
+
 	TOCLIENT_REMOVENODE = 0x22,
+	/*
+		v3s16 position
+	*/
 
 	TOCLIENT_INVENTORY = 0x27,
 	/*
-		[0] u16 command
-		[2] serialized inventory
+		serialized inventory
 	*/
 
 	TOCLIENT_TIME_OF_DAY = 0x29,
@@ -167,40 +179,38 @@ enum ToClientCommand : u16
 
 	TOCLIENT_MEDIA = 0x38,
 	/*
-		u16 total number of texture bunches
+		u16 total number of bunches
 		u16 index of this bunch
 		u32 number of files in this bunch
 		for each file {
 			u16 length of name
 			string name
 			u32 length of data
-			data
+			data (zstd-compressed)
 		}
-		u16 length of remote media server url (if applicable)
-		string url
 	*/
 
 	TOCLIENT_NODEDEF = 0x3a,
 	/*
-		u32 length of the next item
-		serialized NodeDefManager
+		u32 length of buffer
+		serialized NodeDefManager (zstd-compressed)
 	*/
 
 	TOCLIENT_ANNOUNCE_MEDIA = 0x3c,
 	/*
-		u32 number of files
-		for each texture {
-			u16 length of name
-			string name
-			u16 length of sha1_digest
-			string sha1_digest
+		u32 length of compressed name array
+		string16array names (zstd-compressed)
+		for each file {
+			char[20] sha1_digest
 		}
+		u16 length of remote media server url
+		string url
 	*/
 
 	TOCLIENT_ITEMDEF = 0x3d,
 	/*
-		u32 length of next item
-		serialized ItemDefManager
+		u32 length of buffer
+		serialized ItemDefManager (zstd-compressed)
 	*/
 
 	TOCLIENT_PLAY_SOUND = 0x3f,
@@ -721,18 +731,16 @@ enum ToServerCommand : u16
 
 	TOSERVER_PLAYERPOS = 0x23,
 	/*
-		[0] u16 command
-		[2] v3s32 position*100
-		[2+12] v3s32 speed*100
-		[2+12+12] s32 pitch*100
-		[2+12+12+4] s32 yaw*100
-		[2+12+12+4+4] u32 keyPressed
-		[2+12+12+4+4+4] u8 fov*80
-		[2+12+12+4+4+4+1] u8 ceil(wanted_range / MAP_BLOCKSIZE)
-		[2+12+12+4+4+4+1+1] u8 camera_inverted (bool)
-		[2+12+12+4+4+4+1+1+1] f32 movement_speed
-		[2+12+12+4+4+4+1+1+1+4] f32 movement_direction
-
+		v3s32 position*100
+		v3s32 speed*100
+		s32 pitch*100
+		s32 yaw*100
+		u32 keyPressed
+		u8 fov*80
+		u8 ceil(wanted_range / MAP_BLOCKSIZE)
+		u8 camera_inverted (bool)
+		f32 movement_speed
+		f32 movement_direction
 	*/
 
 	TOSERVER_GOTBLOCKS = 0x24,

src/server.cpp

@@ -1487,17 +1487,20 @@ void Server::SendAccessDenied(session_t peer_id, AccessDeniedCode reason,
 void Server::SendItemDef(session_t peer_id,
 		IItemDefManager *itemdef, u16 protocol_version)
 {
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
+
 	NetworkPacket pkt(TOCLIENT_ITEMDEF, 0, peer_id);
 
-	/*
-		u16 command
-		u32 length of the next item
-		zlib-compressed serialized ItemDefManager
-	*/
-	std::ostringstream tmp_os(std::ios::binary);
-	itemdef->serialize(tmp_os, protocol_version);
 	std::ostringstream tmp_os2(std::ios::binary);
-	compressZlib(tmp_os.str(), tmp_os2);
+	{
+		std::ostringstream tmp_os(std::ios::binary);
+		itemdef->serialize(tmp_os, protocol_version);
+		if (client->net_proto_version >= 48)
+			compressZstd(tmp_os.str(), tmp_os2);
+		else
+			compressZlib(tmp_os.str(), tmp_os2);
+	}
 	pkt.putLongString(tmp_os2.str());
 
 	// Make data buffer
@@ -1510,18 +1513,20 @@ void Server::SendItemDef(session_t peer_id,
 void Server::SendNodeDef(session_t peer_id,
 	const NodeDefManager *nodedef, u16 protocol_version)
 {
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
+
 	NetworkPacket pkt(TOCLIENT_NODEDEF, 0, peer_id);
 
-	/*
-		u16 command
-		u32 length of the next item
-		zlib-compressed serialized NodeDefManager
-	*/
-	std::ostringstream tmp_os(std::ios::binary);
-	nodedef->serialize(tmp_os, protocol_version);
 	std::ostringstream tmp_os2(std::ios::binary);
-	compressZlib(tmp_os.str(), tmp_os2);
-
+	{
+		std::ostringstream tmp_os(std::ios::binary);
+		nodedef->serialize(tmp_os, protocol_version);
+		if (client->net_proto_version >= 48)
+			compressZstd(tmp_os.str(), tmp_os2);
+		else
+			compressZlib(tmp_os.str(), tmp_os2);
+	}
 	pkt.putLongString(tmp_os2.str());
 
 	// Make data buffer
@@ -2583,13 +2588,12 @@ bool Server::addMediaFile(const std::string &filename,
 	}
 
 	std::string sha1 = hashing::sha1(filedata);
-	std::string sha1_base64 = base64_encode(sha1);
 	std::string sha1_hex = hex_encode(sha1);
 	if (digest_to)
 		*digest_to = sha1;
 
 	// Put in list
-	m_media[filename] = MediaInfo(filepath, sha1_base64);
+	m_media[filename] = MediaInfo(filepath, sha1);
 	verbosestream << "Server: " << sha1_hex << " is " << filename
 			<< std::endl;
 
@@ -2651,20 +2655,48 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co
 	};
 
 	// Make packet
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
 	NetworkPacket pkt(TOCLIENT_ANNOUNCE_MEDIA, 0, peer_id);
 
-	u16 media_sent = 0;
-	for (const auto &i : m_media) {
-		if (include(i.first, i.second))
-			media_sent++;
-	}
-	pkt << media_sent;
+	size_t media_sent = 0;
+	if (client->net_proto_version < 48) {
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				media_sent++;
+		}
+		assert(media_sent < U16_MAX);
+		pkt << static_cast<u16>(media_sent);
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				pkt << i.first << base64_encode(i.second.sha1_digest);
+		}
+	} else {
+		std::vector<std::string> names;
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				names.emplace_back(i.first);
+		}
+		media_sent = names.size();
 
-	for (const auto &i : m_media) {
-		if (include(i.first, i.second))
-			pkt << i.first << i.second.sha1_digest;
+		// compressed table of media names
+		{
+			std::ostringstream oss(std::ios::binary);
+			auto tmp = serializeString16Array(names);
+			compressZstd(tmp, oss);
+			pkt.putLongString(oss.str());
+		}
+
+		// then the raw hash for each file
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second)) {
+				assert(i.second.sha1_digest.size() == 20);
+				pkt.putRawString(i.second.sha1_digest);
+			}
+		}
 	}
 
+	// and the remote media server(s)
 	pkt << g_settings->get("remote_media");
 	Send(&pkt);
 
@@ -2694,8 +2726,11 @@ void Server::sendRequestedMedia(session_t peer_id,
 	auto *client = getClient(peer_id, CS_DefinitionsSent);
 	assert(client);
 
+	const bool compress = client->net_proto_version >= 48;
+
 	infostream << "Server::sendRequestedMedia(): Sending "
-		<< tosend.size() << " files to " << client->getName() << std::endl;
+		<< tosend.size() << " files to " << client->getName()
+		<< (compress ? " (compressed)" : "") << std::endl;
 
 	/* Read files and prepare bunches */
 
@@ -2713,6 +2748,7 @@ void Server::sendRequestedMedia(session_t peer_id,
 	// the amount of bunches quite well (at the expense of overshooting).
 
 	u32 file_size_bunch_total = 0;
+	size_t bytes_compressed = 0, bytes_uncompressed = 0;
 	for (const std::string &name : tosend) {
 		auto it = m_media.find(name);
 
@@ -2739,9 +2775,19 @@ void Server::sendRequestedMedia(session_t peer_id,
 		if (!fs::ReadFile(m.path, data, true)) {
 			continue;
 		}
-		file_size_bunch_total += data.size();
+		bytes_uncompressed += data.size();
+		if (compress) {
+			// Zstd is very fast and can handle non-compressible data efficiently
+			// so we can just throw it at every file. Still we don't want to
+			// spend too much here, so we use the lowest compression level.
+			std::ostringstream oss(std::ios::binary);
+			compressZstd(data, oss, 1);
+			data = oss.str();
+		}
+		bytes_compressed += data.size();
 
 		// Put in list
+		file_size_bunch_total += data.size();
 		file_bunches.back().emplace_back(name, m.path, std::move(data));
 
 		// Start next bunch if got enough data
@@ -2756,17 +2802,6 @@ void Server::sendRequestedMedia(session_t peer_id,
 	const u16 num_bunches = file_bunches.size();
 	for (u16 i = 0; i < num_bunches; i++) {
 		auto &bunch = file_bunches[i];
-		/*
-			u16 total number of media bunches
-			u16 index of this bunch
-			u32 number of files in this bunch
-			for each file {
-				u16 length of name
-				string name
-				u32 length of data
-				data
-			}
-		*/
 		NetworkPacket pkt(TOCLIENT_MEDIA, 4 + 0, peer_id);
 
 		const u32 bunch_size = bunch.size();
@@ -2784,6 +2819,14 @@ void Server::sendRequestedMedia(session_t peer_id,
 				<< " size="  << pkt.getSize() << std::endl;
 		Send(&pkt);
 	}
+
+	if (compress && bytes_uncompressed != 0) {
+		int percent = bytes_compressed / (float)bytes_uncompressed * 100;
+		int diff = (int)bytes_compressed - (int)bytes_uncompressed;
+		infostream << "Server::sendRequestedMedia(): size after compression "
+			<< percent << "% (" << (diff > 0 ? '+' : '-') << std::abs(diff)
+			<< " byte)" << std::endl;
+	}
 }
 
 void Server::stepPendingDynMediaCallbacks(float dtime)
@@ -4210,7 +4253,7 @@ std::unordered_map<std::string, std::string> Server::getMediaList()
 	for (auto &it : m_media) {
 		if (it.second.no_announce)
 			continue;
-		ret.emplace(base64_decode(it.second.sha1_digest), it.second.path);
+		ret.emplace(it.second.sha1_digest, it.second.path);
 	}
 	return ret;
 }

src/server.h

@@ -90,7 +90,7 @@ enum ClientDeletionReason {
 struct MediaInfo
 {
 	std::string path;
-	std::string sha1_digest; // base64-encoded
+	std::string sha1_digest;
 	// true = not announced in TOCLIENT_ANNOUNCE_MEDIA (at player join)
 	bool no_announce;
 	// does what it says. used by some cases of dynamic media.

src/util/serialize.cpp

@@ -105,6 +105,67 @@ std::string deSerializeString32(std::istream &is)
 	return s;
 }
 
+////
+//// String Array
+////
+
+std::string serializeString16Array(const std::vector<std::string> &array)
+{
+	std::string ret;
+	const auto &at = [&] (size_t index) {
+		return reinterpret_cast<u8*>(&ret[index]);
+	};
+
+	if (array.size() > U32_MAX)
+		throw SerializationError("serializeString16Array: too many strings");
+	ret.resize(4 + array.size() * 2);
+	writeU32(at(0), array.size());
+
+	// Serialize lengths next to each other
+	size_t total = 0;
+	for (u32 i = 0; i < array.size(); i++) {
+		auto &s = array[i];
+		if (s.size() > STRING_MAX_LEN)
+			throw SerializationError("serializeString16Array: string too long");
+		writeU16(at(4 + 2*i), s.size());
+		total += s.size();
+	}
+
+	// Now the contents
+	ret.reserve(ret.size() + total);
+	for (auto &s : array)
+		ret.append(s);
+
+	return ret;
+}
+
+std::vector<std::string> deserializeString16Array(std::istream &is)
+{
+	std::vector<std::string> ret;
+
+	u32 count = readU32(is);
+	if (is.gcount() != 4)
+		throw SerializationError("deserializeString16Array: count not read");
+	ret.resize(count);
+
+	// prepare string buffers as we read the sizes
+	for (auto &sbuf : ret) {
+		u16 size = readU16(is);
+		if (is.gcount() != 2)
+			throw SerializationError("deserializeString16Array: size not read");
+		sbuf.resize(size);
+	}
+
+	// now extract the strings
+	for (auto &sbuf : ret) {
+		is.read(sbuf.data(), sbuf.size());
+		if (is.gcount() != (std::streamsize) sbuf.size())
+			throw SerializationError("deserializeString16Array: truncated");
+	}
+
+	return ret;
+}
+
 ////
 //// JSON-like strings
 ////

src/util/serialize.h

@@ -469,3 +469,10 @@ std::string serializeJsonStringIfNeeded(std::string_view s);
 
 // Parses a string serialized by serializeJsonStringIfNeeded.
 std::string deSerializeJsonStringIfNeeded(std::istream &is);
+
+// Serializes an array of strings (max 2^16 chars each)
+// Output is well suited for compression :)
+std::string serializeString16Array(const std::vector<std::string> &array);
+
+// Deserializes a string array
+std::vector<std::string> deserializeString16Array(std::istream &is);