oss/luanti
oss/luanti
1
0
Fork 0
mirror of https://github.com/luanti-org/luanti.git synced 2025-10-10 19:32:10 +00:00
Code Wiki Activity

Dynamic_Add_Media v2 (#11550)

Browse source
This commit is contained in:
sfan5 2021-09-09 16:51:35 +02:00 • committed by GitHub
parent bcb6565483
commit bbfae0cc67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 796 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/client/client.cpp
View file

@ -555,6 +555,29 @@ void Client::step(float dtime)
m_media_downloader = NULL;
}
}
{
// Acknowledge dynamic media downloads to server
std::vector<u32> done;
for (auto it = m_pending_media_downloads.begin();
it != m_pending_media_downloads.end();) {
assert(it->second->isStarted());
it->second->step(this);
if (it->second->isDone()) {
done.emplace_back(it->first);
it = m_pending_media_downloads.erase(it);
} else {
it++;
}
if (done.size() == 255) { // maximum in one packet
sendHaveMedia(done);
done.clear();
}
}
if (!done.empty())
sendHaveMedia(done);
}
/*
If the server didn't update the inventory in a while, revert
@ -770,7 +793,8 @@ void Client::request_media(const std::vector<std::string> &file_requests)
Send(&pkt);
infostream << "Client: Sending media request list to server ("
<< file_requests.size() << " files. packet size)" << std::endl;
<< file_requests.size() << " files, packet size "
<< pkt.getSize() << ")" << std::endl;
}
void Client::initLocalMapSaving(const Address &address,
@ -1295,6 +1319,19 @@ void Client::sendPlayerPos()
Send(&pkt);
}
void Client::sendHaveMedia(const std::vector<u32> &tokens)
{
NetworkPacket pkt(TOSERVER_HAVE_MEDIA, 1 + tokens.size() * 4);
sanity_check(tokens.size() < 256);
pkt << static_cast<u8>(tokens.size());
for (u32 token : tokens)
pkt << token;
Send(&pkt);
}
void Client::removeNode(v3s16 p)
{
std::map<v3s16, MapBlock*> modified_blocks;

6
src/client/client.h
View file

@ -53,6 +53,7 @@ class ISoundManager;
class NodeDefManager;
//class IWritableCraftDefManager;
class ClientMediaDownloader;
class SingleMediaDownloader;
struct MapDrawControl;
class ModChannelMgr;
class MtEventManager;
@ -245,6 +246,7 @@ public:
void sendDamage(u16 damage);
void sendRespawn();
void sendReady();
void sendHaveMedia(const std::vector<u32> &tokens);
ClientEnvironment& getEnv() { return m_env; }
ITextureSource *tsrc() { return getTextureSource(); }
@ -536,9 +538,13 @@ private:
bool m_activeobjects_received = false;
bool m_mods_loaded = false;
std::vector<std::string> m_remote_media_servers;
// Media downloader, only exists during init
ClientMediaDownloader *m_media_downloader;
// Set of media filenames pushed by server at runtime
std::unordered_set<std::string> m_media_pushed_files;
// Pending downloads of dynamic media (key: token)
std::vector<std::pair<u32, std::unique_ptr<SingleMediaDownloader>>> m_pending_media_downloads;
// time_of_day speed approximation for old protocol
bool m_time_of_day_set = false;

252
src/client/clientmedia.cpp
View file

@ -49,7 +49,6 @@ bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &file
*/
ClientMediaDownloader::ClientMediaDownloader():
m_media_cache(getMediaCacheDir()),
m_httpfetch_caller(HTTPFETCH_DISCARD)
{
}
@ -66,6 +65,12 @@ ClientMediaDownloader::~ClientMediaDownloader()
delete remote;
}
bool ClientMediaDownloader::loadMedia(Client *client, const std::string &data,
const std::string &name)
{
return client->loadMedia(data, name);
}
void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
{
assert(!m_initial_step_done); // pre-condition
@ -105,7 +110,7 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
{
assert(!m_initial_step_done); // pre-condition
#ifdef USE_CURL
#ifdef USE_CURL
if (g_settings->getBool("enable_remote_media_server")) {
infostream << "Client: Adding remote server \""
@ -117,13 +122,13 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
m_remotes.push_back(remote);
}
#else
#else
infostream << "Client: Ignoring remote server \""
<< baseurl << "\" because cURL support is not compiled in"
<< std::endl;
#endif
#endif
}
void ClientMediaDownloader::step(Client *client)
@ -172,36 +177,21 @@ void ClientMediaDownloader::initialStep(Client *client)
// Check media cache
m_uncached_count = m_files.size();
for (auto &file_it : m_files) {
std::string name = file_it.first;
const std::string &name = file_it.first;
FileStatus *filestatus = file_it.second;
const std::string &sha1 = filestatus->sha1;
std::ostringstream tmp_os(std::ios_base::binary);
bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
// If found in cache, try to load it from there
if (found_in_cache) {
bool success = checkAndLoad(name, sha1,
tmp_os.str(), true, client);
if (success) {
filestatus->received = true;
m_uncached_count--;
}
if (tryLoadFromCache(name, sha1, client)) {
filestatus->received = true;
m_uncached_count--;
}
}
assert(m_uncached_received_count == 0);
// Create the media cache dir if we are likely to write to it
if (m_uncached_count != 0) {
bool did = fs::CreateAllDirs(getMediaCacheDir());
if (!did) {
errorstream << "Client: "
<< "Could not create media cache directory: "
<< getMediaCacheDir()
<< std::endl;
}
}
if (m_uncached_count != 0)
createCacheDirs();
// If we found all files in the cache, report this fact to the server.
// If the server reported no remote servers, immediately start
@ -301,8 +291,7 @@ void ClientMediaDownloader::remoteHashSetReceived(
// available on this server, add this server
// to the available_remotes array
for(std::map<std::string, FileStatus*>::iterator
it = m_files.upper_bound(m_name_bound);
for(auto it = m_files.upper_bound(m_name_bound);
it != m_files.end(); ++it) {
FileStatus *f = it->second;
if (!f->received && sha1_set.count(f->sha1))
@ -328,8 +317,7 @@ void ClientMediaDownloader::remoteMediaReceived(
std::string name;
{
std::unordered_map<unsigned long, std::string>::iterator it =
m_remote_file_transfers.find(fetch_result.request_id);
auto it = m_remote_file_transfers.find(fetch_result.request_id);
assert(it != m_remote_file_transfers.end());
name = it->second;
m_remote_file_transfers.erase(it);
@ -398,8 +386,7 @@ void ClientMediaDownloader::startRemoteMediaTransfers()
{
bool changing_name_bound = true;
for (std::map<std::string, FileStatus*>::iterator
files_iter = m_files.upper_bound(m_name_bound);
for (auto files_iter = m_files.upper_bound(m_name_bound);
files_iter != m_files.end(); ++files_iter) {
// Abort if active fetch limit is exceeded
@ -477,19 +464,18 @@ void ClientMediaDownloader::startConventionalTransfers(Client *client)
}
}
void ClientMediaDownloader::conventionalTransferDone(
bool ClientMediaDownloader::conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client)
{
// Check that file was announced
std::map<std::string, FileStatus*>::iterator
file_iter = m_files.find(name);
auto file_iter = m_files.find(name);
if (file_iter == m_files.end()) {
errorstream << "Client: server sent media file that was"
<< "not announced, ignoring it: \"" << name << "\""
<< std::endl;
return;
return false;
}
FileStatus *filestatus = file_iter->second;
assert(filestatus != NULL);
@ -499,7 +485,7 @@ void ClientMediaDownloader::conventionalTransferDone(
errorstream << "Client: server sent media file that we already"
<< "received, ignoring it: \"" << name << "\""
<< std::endl;
return;
return true;
}
// Mark file as received, regardless of whether loading it works and
@ -512,9 +498,45 @@ void ClientMediaDownloader::conventionalTransferDone(
// Check that received file matches announced checksum
// If so, load it
checkAndLoad(name, filestatus->sha1, data, false, client);
return true;
}
bool ClientMediaDownloader::checkAndLoad(
/*
IClientMediaDownloader
*/
IClientMediaDownloader::IClientMediaDownloader():
m_media_cache(getMediaCacheDir()), m_write_to_cache(true)
{
}
void IClientMediaDownloader::createCacheDirs()
{
if (!m_write_to_cache)
return;
std::string path = getMediaCacheDir();
if (!fs::CreateAllDirs(path)) {
errorstream << "Client: Could not create media cache directory: "
<< path << std::endl;
}
}
bool IClientMediaDownloader::tryLoadFromCache(const std::string &name,
const std::string &sha1, Client *client)
{
std::ostringstream tmp_os(std::ios_base::binary);
bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
// If found in cache, try to load it from there
if (found_in_cache)
return checkAndLoad(name, sha1, tmp_os.str(), true, client);
return false;
}
bool IClientMediaDownloader::checkAndLoad(
const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache, Client *client)
{
@ -544,7 +566,7 @@ bool ClientMediaDownloader::checkAndLoad(
}
// Checksum is ok, try loading the file
bool success = client->loadMedia(data, name);
bool success = loadMedia(client, data, name);
if (!success) {
infostream << "Client: "
<< "Failed to load " << cached_or_received << " media: "
@ -559,7 +581,7 @@ bool ClientMediaDownloader::checkAndLoad(
<< std::endl;
// Update cache (unless we just loaded the file from the cache)
if (!is_from_cache)
if (!is_from_cache && m_write_to_cache)
m_media_cache.update(sha1_hex, data);
return true;
@ -587,12 +609,10 @@ std::string ClientMediaDownloader::serializeRequiredHashSet()
// Write list of hashes of files that have not been
// received (found in cache) yet
for (std::map<std::string, FileStatus*>::iterator
it = m_files.begin();
it != m_files.end(); ++it) {
if (!it->second->received) {
FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
os << it->second->sha1;
for (const auto &it : m_files) {
if (!it.second->received) {
FATAL_ERROR_IF(it.second->sha1.size() != 20, "Invalid SHA1 size");
os << it.second->sha1;
}
}
@ -628,3 +648,145 @@ void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
result.insert(data.substr(pos, 20));
}
}
/*
SingleMediaDownloader
*/
SingleMediaDownloader::SingleMediaDownloader(bool write_to_cache):
m_httpfetch_caller(HTTPFETCH_DISCARD)
{
m_write_to_cache = write_to_cache;
}
SingleMediaDownloader::~SingleMediaDownloader()
{
if (m_httpfetch_caller != HTTPFETCH_DISCARD)
httpfetch_caller_free(m_httpfetch_caller);
}
bool SingleMediaDownloader::loadMedia(Client *client, const std::string &data,
const std::string &name)
{
return client->loadMedia(data, name, true);
}
void SingleMediaDownloader::addFile(const std::string &name, const std::string &sha1)
{
assert(m_stage == STAGE_INIT); // pre-condition
assert(!name.empty());
assert(sha1.size() == 20);
FATAL_ERROR_IF(!m_file_name.empty(), "Cannot add a second file");
m_file_name = name;
m_file_sha1 = sha1;
}
void SingleMediaDownloader::addRemoteServer(const std::string &baseurl)
{
assert(m_stage == STAGE_INIT); // pre-condition
if (g_settings->getBool("enable_remote_media_server"))
m_remotes.emplace_back(baseurl);
}
void SingleMediaDownloader::step(Client *client)
{
if (m_stage == STAGE_INIT) {
m_stage = STAGE_CACHE_CHECKED;
initialStep(client);
}
// Remote media: check for completion of fetches
if (m_httpfetch_caller != HTTPFETCH_DISCARD) {
HTTPFetchResult fetch_result;
while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
remoteMediaReceived(fetch_result, client);
}
}
}
bool SingleMediaDownloader::conventionalTransferDone(const std::string &name,
const std::string &data, Client *client)
{
if (name != m_file_name)
return false;
// Mark file as received unconditionally and try to load it
m_stage = STAGE_DONE;
checkAndLoad(name, m_file_sha1, data, false, client);
return true;
}
void SingleMediaDownloader::initialStep(Client *client)
{
if (tryLoadFromCache(m_file_name, m_file_sha1, client))
m_stage = STAGE_DONE;
if (isDone())
return;
createCacheDirs();
// If the server reported no remote servers, immediately fall back to
// conventional transfer.
if (!USE_CURL || m_remotes.empty()) {
startConventionalTransfer(client);
} else {
// Otherwise start by requesting the file from the first remote media server
m_httpfetch_caller = httpfetch_caller_alloc();
m_current_remote = 0;
startRemoteMediaTransfer();
}
}
void SingleMediaDownloader::remoteMediaReceived(
const HTTPFetchResult &fetch_result, Client *client)
{
sanity_check(!isDone());
sanity_check(m_current_remote >= 0);
// If fetch succeeded, try to load it
if (fetch_result.succeeded) {
bool success = checkAndLoad(m_file_name, m_file_sha1,
fetch_result.data, false, client);
if (success) {
m_stage = STAGE_DONE;
return;
}
}
// Otherwise try the next remote server or fall back to conventional transfer
m_current_remote++;
if (m_current_remote >= (int)m_remotes.size()) {
infostream << "Client: Failed to remote-fetch \"" << m_file_name
<< "\". Requesting it the usual way." << std::endl;
m_current_remote = -1;
startConventionalTransfer(client);
} else {
startRemoteMediaTransfer();
}
}
void SingleMediaDownloader::startRemoteMediaTransfer()
{
std::string url = m_remotes.at(m_current_remote) + hex_encode(m_file_sha1);
verbosestream << "Client: Requesting remote media file "
<< "\"" << m_file_name << "\" " << "\"" << url << "\"" << std::endl;
HTTPFetchRequest fetch_request;
fetch_request.url = url;
fetch_request.caller = m_httpfetch_caller;
fetch_request.request_id = m_httpfetch_next_id;
fetch_request.timeout = g_settings->getS32("curl_file_download_timeout");
httpfetch_async(fetch_request);
m_httpfetch_next_id++;
}
void SingleMediaDownloader::startConventionalTransfer(Client *client)
{
std::vector<std::string> requests;
requests.emplace_back(m_file_name);
client->request_media(requests);
}

159
src/client/clientmedia.h
View file

@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "irrlichttypes.h"
#include "filecache.h"
#include "util/basic_macros.h"
#include <ostream>
#include <map>
#include <set>
@ -38,7 +39,62 @@ struct HTTPFetchResult;
bool clientMediaUpdateCache(const std::string &raw_hash,
const std::string &filedata);
class ClientMediaDownloader
// more of a base class than an interface but this name was most convenient...
class IClientMediaDownloader
{
public:
DISABLE_CLASS_COPY(IClientMediaDownloader)
virtual bool isStarted() const = 0;
// If this returns true, the downloader is done and can be deleted
virtual bool isDone() const = 0;
// Add a file to the list of required file (but don't fetch it yet)
virtual void addFile(const std::string &name, const std::string &sha1) = 0;
// Add a remote server to the list; ignored if not built with cURL
virtual void addRemoteServer(const std::string &baseurl) = 0;
// Steps the media downloader:
// - May load media into client by calling client->loadMedia()
// - May check media cache for files
// - May add files to media cache
// - May start remote transfers by calling httpfetch_async
// - May check for completion of current remote transfers
// - May start conventional transfers by calling client->request_media()
// - May inform server that all media has been loaded
// by calling client->received_media()
// After step has been called once, don't call addFile/addRemoteServer.
virtual void step(Client *client) = 0;
// Must be called for each file received through TOCLIENT_MEDIA
// returns true if this file belongs to this downloader
virtual bool conventionalTransferDone(const std::string &name,
const std::string &data, Client *client) = 0;
protected:
IClientMediaDownloader();
virtual ~IClientMediaDownloader() = default;
// Forwards the call to the appropriate Client method
virtual bool loadMedia(Client *client, const std::string &data,
const std::string &name) = 0;
void createCacheDirs();
bool tryLoadFromCache(const std::string &name, const std::string &sha1,
Client *client);
bool checkAndLoad(const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache, Client *client);
// Filesystem-based media cache
FileCache m_media_cache;
bool m_write_to_cache;
};
class ClientMediaDownloader : public IClientMediaDownloader
{
public:
ClientMediaDownloader();
@ -52,39 +108,29 @@ public:
return 0.0f;
}
bool isStarted() const {
bool isStarted() const override {
return m_initial_step_done;
}
// If this returns true, the downloader is done and can be deleted
bool isDone() const {
bool isDone() const override {
return m_initial_step_done &&
m_uncached_received_count == m_uncached_count;
}
// Add a file to the list of required file (but don't fetch it yet)
void addFile(const std::string &name, const std::string &sha1);
void addFile(const std::string &name, const std::string &sha1) override;
// Add a remote server to the list; ignored if not built with cURL
void addRemoteServer(const std::string &baseurl);
void addRemoteServer(const std::string &baseurl) override;
// Steps the media downloader:
// - May load media into client by calling client->loadMedia()
// - May check media cache for files
// - May add files to media cache
// - May start remote transfers by calling httpfetch_async
// - May check for completion of current remote transfers
// - May start conventional transfers by calling client->request_media()
// - May inform server that all media has been loaded
// by calling client->received_media()
// After step has been called once, don't call addFile/addRemoteServer.
void step(Client *client);
void step(Client *client) override;
// Must be called for each file received through TOCLIENT_MEDIA
void conventionalTransferDone(
bool conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client);
Client *client) override;
protected:
bool loadMedia(Client *client, const std::string &data,
const std::string &name) override;
private:
struct FileStatus {
@ -107,13 +153,9 @@ private:
void startRemoteMediaTransfers();
void startConventionalTransfers(Client *client);
bool checkAndLoad(const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache,
Client *client);
std::string serializeRequiredHashSet();
static void deSerializeHashSet(const std::string &data,
std::set<std::string> &result);
std::string serializeRequiredHashSet();
// Maps filename to file status
std::map<std::string, FileStatus*> m_files;
@ -121,9 +163,6 @@ private:
// Array of remote media servers
std::vector<RemoteServerStatus*> m_remotes;
// Filesystem-based media cache
FileCache m_media_cache;
// Has an attempt been made to load media files from the file cache?
// Have hash sets been requested from remote servers?
bool m_initial_step_done = false;
@ -149,3 +188,63 @@ private:
std::string m_name_bound = "";
};
// A media downloader that only downloads a single file.
// It does/doesn't do several things the normal downloader does:
// - won't fetch hash sets from remote servers
// - will mark loaded media as coming from file push
// - writing to file cache is optional
class SingleMediaDownloader : public IClientMediaDownloader
{
public:
SingleMediaDownloader(bool write_to_cache);
~SingleMediaDownloader();
bool isStarted() const override {
return m_stage > STAGE_INIT;
}
bool isDone() const override {
return m_stage >= STAGE_DONE;
}
void addFile(const std::string &name, const std::string &sha1) override;
void addRemoteServer(const std::string &baseurl) override;
void step(Client *client) override;
bool conventionalTransferDone(const std::string &name,
const std::string &data, Client *client) override;
protected:
bool loadMedia(Client *client, const std::string &data,
const std::string &name) override;
private:
void initialStep(Client *client);
void remoteMediaReceived(const HTTPFetchResult &fetch_result, Client *client);
void startRemoteMediaTransfer();
void startConventionalTransfer(Client *client);
enum Stage {
STAGE_INIT,
STAGE_CACHE_CHECKED, // we have tried to load the file from cache
STAGE_DONE
};
// Information about the one file we want to fetch
std::string m_file_name;
std::string m_file_sha1;
s32 m_current_remote;
// Array of remote media servers
std::vector<std::string> m_remotes;
enum Stage m_stage = STAGE_INIT;
// Status of remote transfers
unsigned long m_httpfetch_caller;
unsigned long m_httpfetch_next_id = 0;
};