Merge remote-tracking branch 'upstream/master' into Visuals-Vol-2

src/server.cpp

@@ -461,7 +461,7 @@ void Server::init()
 	m_mod_storage_database = openModStorageDatabase(m_path_world);
 	m_mod_storage_database->beginSave();
 
-	m_modmgr = std::make_unique<ServerModManager>(m_path_world);
+	m_modmgr = std::make_unique<ServerModManager>(m_path_world, m_gamespec);
 
 	// complain about mods with unsatisfied dependencies
 	if (!m_modmgr->isConsistent()) {
@@ -473,8 +473,15 @@ void Server::init()
 	EnvAutoLock envlock(this);
 
 	// Create the Map (loads map_meta.txt, overriding configured mapgen params)
-	auto startup_server_map = std::make_unique<ServerMap>(m_path_world, this,
-			m_emerge.get(), m_metrics_backend.get());
+	std::unique_ptr<ServerMap> startup_server_map;
+	try {
+		startup_server_map = std::make_unique<ServerMap>(m_path_world, this,
+				m_emerge.get(), m_metrics_backend.get());
+	} catch (DatabaseException &e) {
+		throw ServerError(std::string(
+			"Failed to initialize the map database. The world may be "
+			"corrupted or in an unsupported format.\n") + e.what());
+	}
 
 	// Initialize scripting
 	infostream << "Server: Initializing Lua" << std::endl;
@@ -560,8 +567,7 @@ void Server::start()
 {
 	init();
 
-	infostream << "Starting server on " << m_bind_addr.serializeString()
-			<< "..." << std::endl;
+	infostream << "Starting server thread..." << std::endl;
 
 	// Stop thread if already running
 	m_thread->stop();
@@ -960,6 +966,7 @@ void Server::AsyncRunStep(float dtime, bool initial_step)
 		// We'll log the amount of each
 		Profiler prof;
 
+		size_t block_count = 0;
 		std::unordered_set<v3s16> node_meta_updates;
 
 		while (!m_unsent_map_edit_queue.empty()) {
@@ -1008,6 +1015,8 @@ void Server::AsyncRunStep(float dtime, bool initial_step)
 				break;
 			}
 
+			block_count += event->modified_blocks.size();
+
 			/*
 				Set blocks not sent to far players
 			*/
@@ -1019,11 +1028,9 @@ void Server::AsyncRunStep(float dtime, bool initial_step)
 			delete event;
 		}
 
-		if (event_count >= 5) {
-			infostream << "Server: MapEditEvents:" << std::endl;
-			prof.print(infostream);
-		} else if (event_count != 0) {
-			verbosestream << "Server: MapEditEvents:" << std::endl;
+		if (event_count != 0) {
+			verbosestream << "Server: MapEditEvents modified total "
+				<< block_count << " blocks:" << std::endl;
 			prof.print(verbosestream);
 		}
 
@@ -1303,9 +1310,7 @@ void Server::ProcessData(NetworkPacket *pkt)
 		}
 
 		if (m_clients.getClientState(peer_id) < CS_Active) {
-			if (command == TOSERVER_PLAYERPOS) return;
-
-			errorstream << "Server: Got packet command "
+			warningstream << "Server: Got packet command "
 					<< static_cast<unsigned>(command)
 					<< " for peer id " << peer_id
 					<< " but client isn't active yet. Dropping packet." << std::endl;
@@ -1480,17 +1485,20 @@ void Server::SendAccessDenied(session_t peer_id, AccessDeniedCode reason,
 void Server::SendItemDef(session_t peer_id,
 		IItemDefManager *itemdef, u16 protocol_version)
 {
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
+
 	NetworkPacket pkt(TOCLIENT_ITEMDEF, 0, peer_id);
 
-	/*
-		u16 command
-		u32 length of the next item
-		zlib-compressed serialized ItemDefManager
-	*/
-	std::ostringstream tmp_os(std::ios::binary);
-	itemdef->serialize(tmp_os, protocol_version);
 	std::ostringstream tmp_os2(std::ios::binary);
-	compressZlib(tmp_os.str(), tmp_os2);
+	{
+		std::ostringstream tmp_os(std::ios::binary);
+		itemdef->serialize(tmp_os, protocol_version);
+		if (client->net_proto_version >= 48)
+			compressZstd(tmp_os.str(), tmp_os2);
+		else
+			compressZlib(tmp_os.str(), tmp_os2);
+	}
 	pkt.putLongString(tmp_os2.str());
 
 	// Make data buffer
@@ -1503,18 +1511,20 @@ void Server::SendItemDef(session_t peer_id,
 void Server::SendNodeDef(session_t peer_id,
 	const NodeDefManager *nodedef, u16 protocol_version)
 {
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
+
 	NetworkPacket pkt(TOCLIENT_NODEDEF, 0, peer_id);
 
-	/*
-		u16 command
-		u32 length of the next item
-		zlib-compressed serialized NodeDefManager
-	*/
-	std::ostringstream tmp_os(std::ios::binary);
-	nodedef->serialize(tmp_os, protocol_version);
 	std::ostringstream tmp_os2(std::ios::binary);
-	compressZlib(tmp_os.str(), tmp_os2);
-
+	{
+		std::ostringstream tmp_os(std::ios::binary);
+		nodedef->serialize(tmp_os, protocol_version);
+		if (client->net_proto_version >= 48)
+			compressZstd(tmp_os.str(), tmp_os2);
+		else
+			compressZlib(tmp_os.str(), tmp_os2);
+	}
 	pkt.putLongString(tmp_os2.str());
 
 	// Make data buffer
@@ -1559,10 +1569,6 @@ void Server::SendChatMessage(session_t peer_id, const ChatMessage &message)
 		<< static_cast<u64>(message.timestamp);
 
 	if (peer_id != PEER_ID_INEXISTENT) {
-		RemotePlayer *player = m_env->getPlayer(peer_id);
-		if (!player)
-			return;
-
 		Send(&pkt);
 	} else {
 		m_clients.sendToAll(&pkt);
@@ -1954,6 +1960,15 @@ void Server::SendSetLighting(session_t peer_id, const Lighting &lighting)
 	Send(&pkt);
 }
 
+void Server::SendCamera(session_t peer_id, Player *player)
+{
+	NetworkPacket pkt(TOCLIENT_CAMERA, 1, peer_id);
+
+	pkt << static_cast<u8>(player->allowed_camera_mode);
+
+	Send(&pkt);
+}
+
 void Server::SendTimeOfDay(session_t peer_id, u16 time, f32 time_speed)
 {
 	NetworkPacket pkt(TOCLIENT_TIME_OF_DAY, 0, peer_id);
@@ -1984,9 +1999,8 @@ void Server::SendMovePlayer(PlayerSAO *sao)
 	pkt << sao->getBasePosition() << sao->getLookPitch() << sao->getRotation().Y;
 
 	{
-		v3f pos = sao->getBasePosition();
 		verbosestream << "Server: Sending TOCLIENT_MOVE_PLAYER"
-				<< " pos=(" << pos.X << "," << pos.Y << "," << pos.Z << ")"
+				<< " pos=" << sao->getBasePosition()
 				<< " pitch=" << sao->getLookPitch()
 				<< " yaw=" << sao->getRotation().Y
 				<< std::endl;
@@ -2160,10 +2174,6 @@ void Server::SendActiveObjectRemoveAdd(RemoteClient *client, PlayerSAO *playersa
 	}
 
 	Send(&pkt);
-
-	verbosestream << "Server::SendActiveObjectRemoveAdd(): "
-		<< removed_objects.size() << " removed, " << added_objects.size()
-		<< " added, packet size is " << pkt.getSize() << std::endl;
 }
 
 void Server::SendActiveObjectMessages(session_t peer_id, const std::string &datas,
@@ -2558,6 +2568,8 @@ bool Server::addMediaFile(const std::string &filename,
 		".x", ".b3d", ".obj", ".gltf", ".glb",
 		// Translation file formats
 		".tr", ".po", ".mo",
+		// Fonts
+		".ttf", ".woff",
 		NULL
 	};
 	if (removeStringEnd(filename, supported_ext).empty()) {
@@ -2580,13 +2592,12 @@ bool Server::addMediaFile(const std::string &filename,
 	}
 
 	std::string sha1 = hashing::sha1(filedata);
-	std::string sha1_base64 = base64_encode(sha1);
 	std::string sha1_hex = hex_encode(sha1);
 	if (digest_to)
 		*digest_to = sha1;
 
 	// Put in list
-	m_media[filename] = MediaInfo(filepath, sha1_base64);
+	m_media[filename] = MediaInfo(filepath, sha1);
 	verbosestream << "Server: " << sha1_hex << " is " << filename
 			<< std::endl;
 
@@ -2634,9 +2645,9 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co
 	std::string lang_suffixes[3];
 	for (size_t i = 0; i < 3; i++) {
 		lang_suffixes[i].append(".").append(lang_code).append(translation_formats[i]);
-  }
+	}
 
-  auto include = [&] (const std::string &name, const MediaInfo &info) -> bool {
+	auto include = [&] (const std::string &name, const MediaInfo &info) -> bool {
 		if (info.no_announce)
 			return false;
 		for (size_t j = 0; j < 3; j++) {
@@ -2648,20 +2659,48 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co
 	};
 
 	// Make packet
+	auto *client = m_clients.getClientNoEx(peer_id, CS_Created);
+	assert(client);
 	NetworkPacket pkt(TOCLIENT_ANNOUNCE_MEDIA, 0, peer_id);
 
-	u16 media_sent = 0;
-	for (const auto &i : m_media) {
-		if (include(i.first, i.second))
-			media_sent++;
-	}
-	pkt << media_sent;
+	size_t media_sent = 0;
+	if (client->net_proto_version < 48) {
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				media_sent++;
+		}
+		assert(media_sent < U16_MAX);
+		pkt << static_cast<u16>(media_sent);
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				pkt << i.first << base64_encode(i.second.sha1_digest);
+		}
+	} else {
+		std::vector<std::string> names;
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second))
+				names.emplace_back(i.first);
+		}
+		media_sent = names.size();
 
-	for (const auto &i : m_media) {
-		if (include(i.first, i.second))
-			pkt << i.first << i.second.sha1_digest;
+		// compressed table of media names
+		{
+			std::ostringstream oss(std::ios::binary);
+			auto tmp = serializeString16Array(names);
+			compressZstd(tmp, oss);
+			pkt.putLongString(oss.str());
+		}
+
+		// then the raw hash for each file
+		for (const auto &i : m_media) {
+			if (include(i.first, i.second)) {
+				assert(i.second.sha1_digest.size() == 20);
+				pkt.putRawString(i.second.sha1_digest);
+			}
+		}
 	}
 
+	// and the remote media server(s)
 	pkt << g_settings->get("remote_media");
 	Send(&pkt);
 
@@ -2691,8 +2730,11 @@ void Server::sendRequestedMedia(session_t peer_id,
 	auto *client = getClient(peer_id, CS_DefinitionsSent);
 	assert(client);
 
+	const bool compress = client->net_proto_version >= 48;
+
 	infostream << "Server::sendRequestedMedia(): Sending "
-		<< tosend.size() << " files to " << client->getName() << std::endl;
+		<< tosend.size() << " files to " << client->getName()
+		<< (compress ? " (compressed)" : "") << std::endl;
 
 	/* Read files and prepare bunches */
 
@@ -2710,6 +2752,7 @@ void Server::sendRequestedMedia(session_t peer_id,
 	// the amount of bunches quite well (at the expense of overshooting).
 
 	u32 file_size_bunch_total = 0;
+	size_t bytes_compressed = 0, bytes_uncompressed = 0;
 	for (const std::string &name : tosend) {
 		auto it = m_media.find(name);
 
@@ -2736,9 +2779,19 @@ void Server::sendRequestedMedia(session_t peer_id,
 		if (!fs::ReadFile(m.path, data, true)) {
 			continue;
 		}
-		file_size_bunch_total += data.size();
+		bytes_uncompressed += data.size();
+		if (compress) {
+			// Zstd is very fast and can handle non-compressible data efficiently
+			// so we can just throw it at every file. Still we don't want to
+			// spend too much here, so we use the lowest compression level.
+			std::ostringstream oss(std::ios::binary);
+			compressZstd(data, oss, 1);
+			data = oss.str();
+		}
+		bytes_compressed += data.size();
 
 		// Put in list
+		file_size_bunch_total += data.size();
 		file_bunches.back().emplace_back(name, m.path, std::move(data));
 
 		// Start next bunch if got enough data
@@ -2753,17 +2806,6 @@ void Server::sendRequestedMedia(session_t peer_id,
 	const u16 num_bunches = file_bunches.size();
 	for (u16 i = 0; i < num_bunches; i++) {
 		auto &bunch = file_bunches[i];
-		/*
-			u16 total number of media bunches
-			u16 index of this bunch
-			u32 number of files in this bunch
-			for each file {
-				u16 length of name
-				string name
-				u32 length of data
-				data
-			}
-		*/
 		NetworkPacket pkt(TOCLIENT_MEDIA, 4 + 0, peer_id);
 
 		const u32 bunch_size = bunch.size();
@@ -2781,6 +2823,14 @@ void Server::sendRequestedMedia(session_t peer_id,
 				<< " size="  << pkt.getSize() << std::endl;
 		Send(&pkt);
 	}
+
+	if (compress && bytes_uncompressed != 0) {
+		int percent = bytes_compressed / (float)bytes_uncompressed * 100;
+		int diff = (int)bytes_compressed - (int)bytes_uncompressed;
+		infostream << "Server::sendRequestedMedia(): size after compression "
+			<< percent << "% (" << (diff > 0 ? '+' : '-') << std::abs(diff)
+			<< " byte)" << std::endl;
+	}
 }
 
 void Server::stepPendingDynMediaCallbacks(float dtime)
@@ -2920,7 +2970,7 @@ void Server::acceptAuth(session_t peer_id, bool forSudoMode)
 
 		NetworkPacket resp_pkt(TOCLIENT_AUTH_ACCEPT, 1 + 6 + 8 + 4, peer_id);
 
-		resp_pkt << v3f(0,0,0) << (u64) m_env->getServerMap().getSeed()
+		resp_pkt << v3f() << (u64) m_env->getServerMap().getSeed()
 				<< g_settings->getFloat("dedicated_server_step")
 				<< client->allowed_auth_mechs;
 
@@ -3704,7 +3754,11 @@ bool Server::dynamicAddMedia(const DynamicMediaArgs &a)
 	// Push file to existing clients
 	if (m_env) {
 		NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0);
-		pkt << raw_hash << filename << static_cast<bool>(a.ephemeral);
+		pkt << raw_hash << filename;
+		// NOTE: the meaning of a.ephemeral was accidentally inverted between proto 39 and 40,
+		// when dynamic_add_media v2 was added. As of 5.12.0 the server sends it correctly again.
+		// Compatibility code on the client-side was not added.
+		pkt << static_cast<bool>(!a.ephemeral);
 
 		NetworkPacket legacy_pkt = pkt;
 
@@ -3871,7 +3925,11 @@ std::string Server::getBuiltinLuaPath()
 
 void Server::setAsyncFatalError(const std::string &error)
 {
+	// print error right here in the thread that set it, for clearer logging
+	infostream << "setAsyncFatalError: " << error << std::endl;
+
 	m_async_fatal_error.set(error);
+
 	// make sure server steps stop happening immediately
 	if (m_thread)
 		m_thread->stop();
@@ -3898,13 +3956,13 @@ v3f Server::findSpawnPos()
 {
 	ServerMap &map = m_env->getServerMap();
 
-    std::optional<v3f> staticSpawnPoint;
+	std::optional<v3f> staticSpawnPoint;
 	if (g_settings->getV3FNoEx("static_spawnpoint", staticSpawnPoint) && staticSpawnPoint.has_value())
-    {
-       return *staticSpawnPoint * BS;
-    }
+	{
+		return *staticSpawnPoint * BS;
+	}
 
-    v3f nodeposf;
+	v3f nodeposf;
 
 	bool is_good = false;
 	// Limit spawn range to mapgen edges (determined by 'mapgen_limit')
@@ -4207,7 +4265,7 @@ std::unordered_map<std::string, std::string> Server::getMediaList()
 	for (auto &it : m_media) {
 		if (it.second.no_announce)
 			continue;
-		ret.emplace(base64_decode(it.second.sha1_digest), it.second.path);
+		ret.emplace(it.second.sha1_digest, it.second.path);
 	}
 	return ret;
 }