oss/luanti
oss/luanti
1
0
Fork 0
mirror of https://github.com/luanti-org/luanti.git synced 2025-08-11 17:51:04 +00:00
Code Wiki Activity

Switch MapBlock compression to zstd (#10788)

Browse source 
* Add zstd support.
* Rearrange serialization order
* Compress entire mapblock

Co-authored-by: sfan5 <sfan5@live.de>
This commit is contained in:
lhofhansl 2021-08-31 17:32:31 -07:00 • committed by GitHub
parent beac4a2c98
commit d1624a5521
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 494 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/main.cpp
View file

@ -38,6 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "player.h"
#include "porting.h"
#include "network/socket.h"
#include "mapblock.h"
#if USE_CURSES
#include "terminal_chat_console.h"
#endif
@ -111,6 +112,7 @@ static bool determine_subgame(GameParams *game_params);
static bool run_dedicated_server(const GameParams &game_params, const Settings &cmd_args);
static bool migrate_map_database(const GameParams &game_params, const Settings &cmd_args);
static bool recompress_map_database(const GameParams &game_params, const Settings &cmd_args, const Address &addr);
/**********************************************************************/
@ -302,6 +304,8 @@ static void set_allowed_options(OptionList *allowed_options)
_("Migrate from current auth backend to another (Only works when using minetestserver or with --server)"))));
allowed_options->insert(std::make_pair("terminal", ValueSpec(VALUETYPE_FLAG,
_("Feature an interactive terminal (Only works when using minetestserver or with --server)"))));
allowed_options->insert(std::make_pair("recompress", ValueSpec(VALUETYPE_FLAG,
_("Recompress the blocks of the given map database."))));
#ifndef SERVER
allowed_options->insert(std::make_pair("speedtests", ValueSpec(VALUETYPE_FLAG,
_("Run speed tests"))));
@ -875,7 +879,7 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings &
return false;
}
// Database migration
// Database migration/compression
if (cmd_args.exists("migrate"))
return migrate_map_database(game_params, cmd_args);
@ -885,6 +889,9 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings &
if (cmd_args.exists("migrate-auth"))
return ServerEnvironment::migrateAuthDatabase(game_params, cmd_args);
if (cmd_args.getFlag("recompress"))
return recompress_map_database(game_params, cmd_args, bind_addr);
if (cmd_args.exists("terminal")) {
#if USE_CURSES
bool name_ok = true;
@ -1034,3 +1041,67 @@ static bool migrate_map_database(const GameParams &game_params, const Settings &
return true;
}
static bool recompress_map_database(const GameParams &game_params, const Settings &cmd_args, const Address &addr)
{
Settings world_mt;
const std::string world_mt_path = game_params.world_path + DIR_DELIM + "world.mt";
if (!world_mt.readConfigFile(world_mt_path.c_str())) {
errorstream << "Cannot read world.mt at " << world_mt_path << std::endl;
return false;
}
const std::string &backend = world_mt.get("backend");
Server server(game_params.world_path, game_params.game_spec, false, addr, false);
MapDatabase *db = ServerMap::createDatabase(backend, game_params.world_path, world_mt);
u32 count = 0;
u64 last_update_time = 0;
bool &kill = *porting::signal_handler_killstatus();
const u8 serialize_as_ver = SER_FMT_VER_HIGHEST_WRITE;
// This is ok because the server doesn't actually run
std::vector<v3s16> blocks;
db->listAllLoadableBlocks(blocks);
db->beginSave();
std::istringstream iss(std::ios_base::binary);
std::ostringstream oss(std::ios_base::binary);
for (auto it = blocks.begin(); it != blocks.end(); ++it) {
if (kill) return false;
std::string data;
db->loadBlock(*it, &data);
if (data.empty()) {
errorstream << "Failed to load block " << PP(*it) << std::endl;
return false;
}
iss.str(data);
iss.clear();
MapBlock mb(nullptr, v3s16(0,0,0), &server);
u8 ver = readU8(iss);
mb.deSerialize(iss, ver, true);
oss.str("");
oss.clear();
writeU8(oss, serialize_as_ver);
mb.serialize(oss, serialize_as_ver, true, -1);
db->saveBlock(*it, oss.str());
count++;
if (count % 0xFF == 0 && porting::getTimeS() - last_update_time >= 1) {
std::cerr << " Recompressed " << count << " blocks, "
<< (100.0f * count / blocks.size()) << "% completed.\r";
db->endSave();
db->beginSave();
last_update_time = porting::getTimeS();
}
}
std::cerr << std::endl;
db->endSave();
actionstream << "Done, " << count << " blocks were recompressed." << std::endl;
return true;
}