Clean up and compress some pre-join packets (#15881)

src/network/clientpackethandler.cpp

@@ -606,10 +606,6 @@ void Client::handleCommand_DeathScreenLegacy(NetworkPacket* pkt)
 
 void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 {
-	u16 num_files;
-
-	*pkt >> num_files;
-
 	infostream << "Client: Received media announcement: packet size: "
 			<< pkt->getSize() << std::endl;
 
@@ -619,9 +615,7 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 			"we already saw another announcement" :
 			"all media has been received already";
 		errorstream << "Client: Received media announcement but "
-			<< problem << "! "
-			<< " files=" << num_files
-			<< " size=" << pkt->getSize() << std::endl;
+			<< problem << "!" << std::endl;
 		return;
 	}
 
@@ -629,16 +623,36 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 	// updating content definitions
 	sanity_check(!m_mesh_update_manager->isRunning());
 
-	for (u16 i = 0; i < num_files; i++) {
+	if (m_proto_ver >= 48) {
+		// compressed table of media names
+		std::vector<std::string> names;
+		{
+			std::istringstream iss(pkt->readLongString(), std::ios::binary);
+			std::stringstream ss(std::ios::in | std::ios::out | std::ios::binary);
+			decompressZstd(iss, ss);
+			names = deserializeString16Array(ss);
+		}
+
+		// raw hash for each media file
+		for (auto &name : names) {
+			auto sha1_raw = pkt->readRawString(20);
+			m_media_downloader->addFile(name, sha1_raw);
+		}
+	} else {
+		u16 num_files;
+		*pkt >> num_files;
+
 		std::string name, sha1_base64;
+		for (u16 i = 0; i < num_files; i++) {
+			*pkt >> name >> sha1_base64;
 
-		*pkt >> name >> sha1_base64;
-
-		std::string sha1_raw = base64_decode(sha1_base64);
-		m_media_downloader->addFile(name, sha1_raw);
+			std::string sha1_raw = base64_decode(sha1_base64);
+			m_media_downloader->addFile(name, sha1_raw);
+		}
 	}
 
 	{
+		// Remote media servers
 		std::string str;
 		*pkt >> str;
 
@@ -657,18 +671,6 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 
 void Client::handleCommand_Media(NetworkPacket* pkt)
 {
-	/*
-		u16 command
-		u16 total number of file bunches
-		u16 index of this bunch
-		u32 number of files in this bunch
-		for each file {
-			u16 length of name
-			string name
-			u32 length of data
-			data
-		}
-	*/
 	u16 num_bunches;
 	u16 bunch_i;
 	u32 num_files;
@@ -695,6 +697,12 @@ void Client::handleCommand_Media(NetworkPacket* pkt)
 
 		*pkt >> name;
 		data = pkt->readLongString();
+		if (m_proto_ver >= 48) {
+			std::istringstream iss(data, std::ios::binary);
+			std::ostringstream oss(std::ios::binary);
+			decompressZstd(iss, oss);
+			data = oss.str();
+		}
 
 		bool ok = false;
 		if (init_phase) {
@@ -729,7 +737,10 @@ void Client::handleCommand_NodeDef(NetworkPacket* pkt)
 	// Decompress node definitions
 	std::istringstream tmp_is(pkt->readLongString(), std::ios::binary);
 	std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out);
-	decompressZlib(tmp_is, tmp_os);
+	if (m_proto_ver >= 48)
+		decompressZstd(tmp_is, tmp_os);
+	else
+		decompressZlib(tmp_is, tmp_os);
 
 	// Deserialize node definitions
 	m_nodedef->deSerialize(tmp_os, m_proto_ver);
@@ -748,7 +759,10 @@ void Client::handleCommand_ItemDef(NetworkPacket* pkt)
 	// Decompress item definitions
 	std::istringstream tmp_is(pkt->readLongString(), std::ios::binary);
 	std::stringstream tmp_os(std::ios::binary | std::ios::in | std::ios::out);
-	decompressZlib(tmp_is, tmp_os);
+	if (m_proto_ver >= 48)
+		decompressZstd(tmp_is, tmp_os);
+	else
+		decompressZlib(tmp_is, tmp_os);
 
 	// Deserialize node definitions
 	m_itemdef->deSerialize(tmp_os, m_proto_ver);

src/network/networkpacket.cpp

@@ -69,6 +69,18 @@ void NetworkPacket::putRawString(const char* src, u32 len)
 	m_read_offset += len;
 }
 
+void NetworkPacket::readRawString(char *dst, u32 len)
+{
+	checkReadOffset(m_read_offset, len);
+
+	if (len == 0)
+		return;
+
+	memcpy(dst, &m_data[m_read_offset], len);
+	m_read_offset += len;
+}
+
+
 NetworkPacket& NetworkPacket::operator>>(std::string& dst)
 {
 	checkReadOffset(m_read_offset, 2);

src/network/networkpacket.h

@@ -51,6 +51,16 @@ public:
 		putRawString(src.data(), src.size());
 	}
 
+	// Reads bytes from packet into string buffer
+	void readRawString(char *dst, u32 len);
+	std::string readRawString(u32 len)
+	{
+		std::string s;
+		s.resize(len);
+		readRawString(&s[0], len);
+		return s;
+	}
+
 	NetworkPacket &operator>>(std::string &dst);
 	NetworkPacket &operator<<(std::string_view src);
 

src/network/networkprotocol.cpp

@@ -62,10 +62,13 @@
 	PROTOCOL VERSION 47
 		Add particle blend mode "clip"
 		[scheduled bump for 5.11.0]
+	PROTOCOL VERSION 48
+		Add compression to some existing packets
+		[scheduled bump for 5.12.0]
 */
 
 // Note: Also update core.protocol_versions in builtin when bumping
-const u16 LATEST_PROTOCOL_VERSION = 47;
+const u16 LATEST_PROTOCOL_VERSION = 48;
 
 // See also formspec [Version History] in doc/lua_api.md
 const u16 FORMSPEC_API_VERSION = 8;

src/network/networkprotocol.h

@@ -33,24 +33,28 @@ enum ToClientCommand : u16
 		u32 supported auth methods
 		std::string unused (used to be username)
 	*/
+
 	TOCLIENT_AUTH_ACCEPT = 0x03,
 	/*
 		Message from server to accept auth.
 
-		v3s16 player's position + v3f(0,BS/2,0) floatToInt'd
+		v3f unused
 		u64 map seed
 		f1000 recommended send interval
 		u32 : supported auth methods for sudo mode
 		      (where the user can change their password)
 	*/
+
 	TOCLIENT_ACCEPT_SUDO_MODE = 0x04,
 	/*
 		Sent to client to show it is in sudo mode now.
 	*/
+
 	TOCLIENT_DENY_SUDO_MODE = 0x05,
 	/*
 		Signals client that sudo mode auth failed.
 	*/
+
 	TOCLIENT_ACCESS_DENIED = 0x0A,
 	/*
 		u8 reason
@@ -59,18 +63,26 @@ enum ToClientCommand : u16
 	*/
 
 	TOCLIENT_BLOCKDATA = 0x20,
+	/*
+		v3s16 position
+		serialized MapBlock
+	*/
+
 	TOCLIENT_ADDNODE = 0x21,
 	/*
 		v3s16 position
 		serialized mapnode
-		u8 keep_metadata // Added in protocol version 22
+		u8 keep_metadata
 	*/
+
 	TOCLIENT_REMOVENODE = 0x22,
+	/*
+		v3s16 position
+	*/
 
 	TOCLIENT_INVENTORY = 0x27,
 	/*
-		[0] u16 command
-		[2] serialized inventory
+		serialized inventory
 	*/
 
 	TOCLIENT_TIME_OF_DAY = 0x29,
@@ -167,40 +179,38 @@ enum ToClientCommand : u16
 
 	TOCLIENT_MEDIA = 0x38,
 	/*
-		u16 total number of texture bunches
+		u16 total number of bunches
 		u16 index of this bunch
 		u32 number of files in this bunch
 		for each file {
 			u16 length of name
 			string name
 			u32 length of data
-			data
+			data (zstd-compressed)
 		}
-		u16 length of remote media server url (if applicable)
-		string url
 	*/
 
 	TOCLIENT_NODEDEF = 0x3a,
 	/*
-		u32 length of the next item
-		serialized NodeDefManager
+		u32 length of buffer
+		serialized NodeDefManager (zstd-compressed)
 	*/
 
 	TOCLIENT_ANNOUNCE_MEDIA = 0x3c,
 	/*
-		u32 number of files
-		for each texture {
-			u16 length of name
-			string name
-			u16 length of sha1_digest
-			string sha1_digest
+		u32 length of compressed name array
+		string16array names (zstd-compressed)
+		for each file {
+			char[20] sha1_digest
 		}
+		u16 length of remote media server url
+		string url
 	*/
 
 	TOCLIENT_ITEMDEF = 0x3d,
 	/*
-		u32 length of next item
-		serialized ItemDefManager
+		u32 length of buffer
+		serialized ItemDefManager (zstd-compressed)
 	*/
 
 	TOCLIENT_PLAY_SOUND = 0x3f,
@@ -721,18 +731,16 @@ enum ToServerCommand : u16
 
 	TOSERVER_PLAYERPOS = 0x23,
 	/*
-		[0] u16 command
-		[2] v3s32 position*100
-		[2+12] v3s32 speed*100
-		[2+12+12] s32 pitch*100
-		[2+12+12+4] s32 yaw*100
-		[2+12+12+4+4] u32 keyPressed
-		[2+12+12+4+4+4] u8 fov*80
-		[2+12+12+4+4+4+1] u8 ceil(wanted_range / MAP_BLOCKSIZE)
-		[2+12+12+4+4+4+1+1] u8 camera_inverted (bool)
-		[2+12+12+4+4+4+1+1+1] f32 movement_speed
-		[2+12+12+4+4+4+1+1+1+4] f32 movement_direction
-
+		v3s32 position*100
+		v3s32 speed*100
+		s32 pitch*100
+		s32 yaw*100
+		u32 keyPressed
+		u8 fov*80
+		u8 ceil(wanted_range / MAP_BLOCKSIZE)
+		u8 camera_inverted (bool)
+		f32 movement_speed
+		f32 movement_direction
 	*/
 
 	TOSERVER_GOTBLOCKS = 0x24,