diff --git a/games/devtest/mods/testtranslations/init.lua b/games/devtest/mods/testtranslations/init.lua index a2de998a21..79af80a9f7 100644 --- a/games/devtest/mods/testtranslations/init.lua +++ b/games/devtest/mods/testtranslations/init.lua @@ -5,12 +5,18 @@ local function send_compare(name, text) core.get_translated_string("", text), text, core.get_translated_string("fr", text))) end +local function send_multilang_compare(name, text) + core.chat_send_player(name, ("%s | %s | %s | %s"):format( + core.get_translated_string("", text), text, + core.get_translated_string("fr_CH:fr", text), core.get_translated_string("fr:fr_CH", text))) +end + core.register_chatcommand("testtranslations", { params = "", description = "Test translations", privs = {}, func = function(name, param) - core.chat_send_player(name, "Please ensure your locale is set to \"fr\"") + core.chat_send_player(name, "Please ensure your locale is set to \"fr_CH\"") core.chat_send_player(name, "Untranslated | Client-side translation | Server-side translation (fr)") send_compare(name, S("Testing .tr files: untranslated")) send_compare(name, S("Testing .po files: untranslated")) @@ -22,5 +28,9 @@ core.register_chatcommand("testtranslations", { send_compare(name, NS("@1: .po singular", "@1: .po plural", i, tostring(i))) send_compare(name, NS("@1: .mo singular", "@1: .mo plural", i, tostring(i))) end + core.chat_send_player(name, "Untranslated | Client-side translations" .. + " | Server-side translation (fr_CH:fr) | Server-side translation (fr:fr_CH)") + send_multilang_compare(name, S("Testing translation with multiple languages")) + send_multilang_compare(name, S("Testing French-only translation: untranslated")) end }) diff --git a/games/devtest/mods/testtranslations/locale/translation_po.fr.po b/games/devtest/mods/testtranslations/locale/translation_po.fr.po index 5aefc0f414..e2f8a4e573 100644 --- a/games/devtest/mods/testtranslations/locale/translation_po.fr.po +++ b/games/devtest/mods/testtranslations/locale/translation_po.fr.po @@ -20,3 +20,11 @@ msgstr "Testing fuzzy .po entry: translated (wrong)" msgid "Testing .po without context: untranslated" msgstr "Testing .po without context: translated" + +msgctxt "testtranslations" +msgid "Testing translation with multiple languages" +msgstr "Testing translation: translated in standard French" + +msgctxt "testtranslations" +msgid "Testing French-only translation: untranslated" +msgstr "Testing French-only translation: translated" diff --git a/games/devtest/mods/testtranslations/locale/translation_po.fr_CH.po b/games/devtest/mods/testtranslations/locale/translation_po.fr_CH.po new file mode 100644 index 0000000000..a2cc8b1216 --- /dev/null +++ b/games/devtest/mods/testtranslations/locale/translation_po.fr_CH.po @@ -0,0 +1,3 @@ +msgctxt "testtranslations" +msgid "Testing translation with multiple languages" +msgstr "Testing translation: translated in Swiss French" diff --git a/games/devtest/mods/testtranslations/test_locale/translation_po.de.po b/games/devtest/mods/testtranslations/test_locale/translation_po.de.po index 9a64805a62..5aa6b086cc 100644 --- a/games/devtest/mods/testtranslations/test_locale/translation_po.de.po +++ b/games/devtest/mods/testtranslations/test_locale/translation_po.de.po @@ -40,3 +40,9 @@ msgstr[1] "" msgctxt "context" msgid "With context" msgstr "Has context" + +msgid "In multiple languages" +msgstr "In standard German" + +msgid "In one language" +msgstr "Only in standard German" diff --git a/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po b/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po new file mode 100644 index 0000000000..8145fbcdcf --- /dev/null +++ b/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po @@ -0,0 +1,2 @@ +msgid "In multiple languages" +msgstr "In Swiss German" diff --git a/src/gettext.cpp b/src/gettext.cpp index 7cb4a7763b..b9af09f1d4 100644 --- a/src/gettext.cpp +++ b/src/gettext.cpp @@ -6,6 +6,7 @@ #include #include #include "gettext.h" +#include "util/langcode.h" #include "util/string.h" #include "porting.h" #include "log.h" @@ -146,6 +147,10 @@ static void MSVC_LocaleWorkaround(int argc, char* argv[]) #endif +static std::string configured_locale; +static std::vector effective_locale; +static std::string effective_locale_string; + /******************************************************************************/ void init_gettext(const char *path, const std::string &configured_language, int argc, char *argv[]) @@ -223,6 +228,17 @@ void init_gettext(const char *path, const std::string &configured_language, setlocale(LC_ALL, ""); #endif // if USE_GETTEXT + // Set up locale for in-game translations + configured_locale = configured_language; + if (configured_locale.empty()) { + if (auto lang = getenv("LANGUAGE"); lang && *lang) + configured_locale = lang; + else + configured_locale = getenv("LANG"); + } + effective_locale = parse_language_list(utf8_to_wide(configured_locale)); + effective_locale_string = wide_to_utf8(str_join(effective_locale, L":")); + /* no matter what locale is used we need number format to be "C" */ /* to ensure formspec parameters are evaluated correctly! */ @@ -230,3 +246,13 @@ void init_gettext(const char *path, const std::string &configured_language, infostream << "Message locale is now set to: " << setlocale(LC_ALL, 0) << std::endl; } + +const std::vector &get_effective_locale() +{ + return effective_locale; +} + +const std::string &get_client_language_code() +{ + return effective_locale_string; +} diff --git a/src/gettext.h b/src/gettext.h index a58dc93025..5dc983f810 100644 --- a/src/gettext.h +++ b/src/gettext.h @@ -4,6 +4,7 @@ #pragma once +#include #include "config.h" // for USE_GETTEXT #include "porting.h" #include "util/string.h" @@ -103,3 +104,15 @@ inline std::string fmtgettext(const char *format, Args&&... args) return buf; } + +/** + * Returns the effective locale setting for in-game translations. + * @return A vector of language codes based on priority. + */ +const std::vector &get_effective_locale(); + +/** + * Returns the effective locale setting for in-game translations. + * @return A string of the expanded language code. + */ +const std::string &get_client_language_code(); diff --git a/src/gui/guiEngine.cpp b/src/gui/guiEngine.cpp index 99a755d41f..fba12cd163 100644 --- a/src/gui/guiEngine.cpp +++ b/src/gui/guiEngine.cpp @@ -229,42 +229,46 @@ std::string findLocaleFileInMods(const std::string &path, const std::string &fil /******************************************************************************/ Translations *GUIEngine::getContentTranslations(const std::string &path, - const std::string &domain, const std::string &lang_code) + const std::string &domain, const std::vector &lang) { - if (domain.empty() || lang_code.empty()) + if (domain.empty() || lang.empty()) return nullptr; - std::string filename_no_ext = domain + "." + lang_code; - std::string key = path + DIR_DELIM "locale" DIR_DELIM + filename_no_ext; + std::string key = path + DIR_DELIM "locale" DIR_DELIM + domain; if (key == m_last_translations_key) return &m_last_translations; - std::string trans_path = key; - - switch (getContentType(path)) { - case ContentType::GAME: - trans_path = findLocaleFileInMods(path + DIR_DELIM "mods" DIR_DELIM, - filename_no_ext); - break; - case ContentType::MODPACK: - trans_path = findLocaleFileInMods(path, filename_no_ext); - break; - default: - trans_path = findLocaleFileWithExtension(trans_path); - break; + Translations translations; + for (const auto &lang_code: lang) { + auto utf8_lang_code = wide_to_utf8(lang_code); + std::string filename_no_ext = domain + "." + utf8_lang_code; + auto trans_path = key + "." + utf8_lang_code; + switch (getContentType(path)) { + case ContentType::GAME: + trans_path = findLocaleFileInMods(path + DIR_DELIM "mods" DIR_DELIM, + filename_no_ext); + break; + case ContentType::MODPACK: + trans_path = findLocaleFileInMods(path, filename_no_ext); + break; + default: + trans_path = findLocaleFileWithExtension(trans_path); + break; + } + if (trans_path.empty()) + continue; + std::string data; + if (fs::ReadFile(trans_path, data)) { + translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data); + } } - if (trans_path.empty()) + if (translations.empty()) return nullptr; m_last_translations_key = key; - m_last_translations = {}; - - std::string data; - if (fs::ReadFile(trans_path, data)) { - m_last_translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data); - } + m_last_translations = std::move(translations); return &m_last_translations; } diff --git a/src/gui/guiEngine.h b/src/gui/guiEngine.h index da0700319d..e1ae45ed4f 100644 --- a/src/gui/guiEngine.h +++ b/src/gui/guiEngine.h @@ -156,7 +156,7 @@ public: * change with the next call to `getContentTranslations`. * */ Translations *getContentTranslations(const std::string &path, - const std::string &domain, const std::string &lang_code); + const std::string &domain, const std::vector &lang_code); private: std::string m_last_translations_key; diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index 4df8a3ae3b..1c2fa05aad 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -142,13 +142,7 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt) << m_recommended_send_interval<getTranslationLanguage(lang_code); - string = wide_to_utf8(translate_string(utf8_to_wide(string), translations)); + string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations)); lua_pushstring(L, string.c_str()); return 1; } diff --git a/src/script/lua_api/l_mainmenu.cpp b/src/script/lua_api/l_mainmenu.cpp index 4b878ee1cd..4326400a6d 100644 --- a/src/script/lua_api/l_mainmenu.cpp +++ b/src/script/lua_api/l_mainmenu.cpp @@ -527,12 +527,10 @@ int ModApiMainMenu::l_get_content_translation(lua_State *L) std::string path = luaL_checkstring(L, 1); std::string domain = luaL_checkstring(L, 2); std::string string = luaL_checkstring(L, 3); - std::string lang = gettext("LANG_CODE"); - if (lang == "LANG_CODE") - lang = ""; + auto lang = get_effective_locale(); auto *translations = engine->getContentTranslations(path, domain, lang); - string = wide_to_utf8(translate_string(utf8_to_wide(string), translations)); + string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations)); lua_pushstring(L, string.c_str()); return 1; } @@ -879,10 +877,7 @@ int ModApiMainMenu::l_download_file(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_language(lua_State *L) { - std::string lang = gettext("LANG_CODE"); - if (lang == "LANG_CODE") - lang = ""; - + std::string lang = get_client_language_code(); lua_pushstring(L, lang.c_str()); return 1; } diff --git a/src/server.cpp b/src/server.cpp index a4328fb5a9..4b65f0256d 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2685,22 +2685,18 @@ void Server::fillMediaCache() infostream << "Server: " << m_media.size() << " media files collected" << std::endl; } -void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code) +void Server::sendMediaAnnouncement(session_t peer_id, const std::string &langstring) { - std::string translation_formats[3] = { ".tr", ".po", ".mo" }; - std::string lang_suffixes[3]; - for (size_t i = 0; i < 3; i++) { - lang_suffixes[i].append(".").append(lang_code).append(translation_formats[i]); - } + std::unordered_set langs; + for (const auto &lang_code: str_split(langstring, ':')) + langs.insert(lang_code); auto include = [&] (const std::string &name, const MediaInfo &info) -> bool { if (info.no_announce) return false; - for (size_t j = 0; j < 3; j++) { - if (str_ends_with(name, translation_formats[j]) && !str_ends_with(name, lang_suffixes[j])) { - return false; - } - } + if (auto filelang = Translations::getFileLanguage(name); + !filelang.empty() && langs.find(std::string(filelang)) == langs.end()) + return false; return true; }; @@ -4288,28 +4284,33 @@ void Server::broadcastModChannelMessage(const std::string &channel, } } -Translations *Server::getTranslationLanguage(const std::string &lang_code) +Translations *Server::getTranslationLanguage(const std::string &lang) { - if (lang_code.empty()) + if (lang.empty()) return nullptr; - auto it = server_translations.find(lang_code); - if (it != server_translations.end()) - return &it->second; // Already loaded + std::unordered_set load_langs; - // [] will create an entry - auto *translations = &server_translations[lang_code]; + for (const auto &lang_code: str_split(lang, ':')) { + if (loaded_translations.find(lang_code) == loaded_translations.end()) { + load_langs.insert(lang_code); + loaded_translations.insert(lang_code); + } + } + + if (load_langs.empty()) + return &server_translations; for (const auto &i : m_media) { - if (Translations::getFileLanguage(i.first) == lang_code) { + if (load_langs.find(std::string(Translations::getFileLanguage(i.first))) != load_langs.end()) { std::string data; if (fs::ReadFile(i.second.path, data, true)) { - translations->loadTranslation(i.first, data); + server_translations.loadTranslation(i.first, data); } } } - return translations; + return &server_translations; } std::unordered_map Server::getMediaList() diff --git a/src/server.h b/src/server.h index cc50f912a6..6915d01b71 100644 --- a/src/server.h +++ b/src/server.h @@ -713,7 +713,8 @@ private: // Craft definition manager IWritableCraftDefManager *m_craftdef; - std::unordered_map server_translations; + Translations server_translations; + std::unordered_set loaded_translations; ModIPCStore m_ipcstore; diff --git a/src/translation.cpp b/src/translation.cpp index 71469507d1..8ca5efb10a 100644 --- a/src/translation.cpp +++ b/src/translation.cpp @@ -36,45 +36,60 @@ void Translations::clear() m_plural_translations.clear(); } +const std::wstring &Translations::getTranslation(const std::vector &langlist, + const std::wstring &textdomain, const std::wstring &s) const +{ + auto basekey = L"|" + textdomain + L"|" + s; + for (const auto &lang: langlist) { + auto it = m_translations.find(lang + basekey); + if (it != m_translations.end()) + return it->second; + } + return s; +} + const std::wstring &Translations::getTranslation( const std::wstring &textdomain, const std::wstring &s) const { - std::wstring key = textdomain + L"|" + s; - auto it = m_translations.find(key); - if (it != m_translations.end()) - return it->second; + return getTranslation(get_effective_locale(), textdomain, s); +} + +const std::wstring &Translations::getPluralTranslation(const std::vector &langlist, + const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const +{ + auto basekey = L"|" + textdomain + L"|" + s; + for (const auto &lang: langlist) { + auto it = m_plural_translations.find(lang + basekey); + if (it != m_plural_translations.end()) { + auto n = (*(it->second.first))(number); + const std::vector &v = it->second.second; + if (n < v.size()) { + if (v[n].empty()) + return s; + return v[n]; + } + } + } return s; } const std::wstring &Translations::getPluralTranslation( const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const { - std::wstring key = textdomain + L"|" + s; - auto it = m_plural_translations.find(key); - if (it != m_plural_translations.end()) { - auto n = (*(it->second.first))(number); - const std::vector &v = it->second.second; - if (n < v.size()) { - if (v[n].empty()) - return s; - return v[n]; - } - } - return s; + return getPluralTranslation(get_effective_locale(), textdomain, s, number); } - -void Translations::addTranslation( - const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated) +void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain, + const std::wstring &original, const std::wstring &translated) { - std::wstring key = textdomain + L"|" + original; + std::wstring key = lang + L"|" + textdomain + L"|" + original; if (!translated.empty()) { m_translations.emplace(std::move(key), std::move(translated)); } } -void Translations::addPluralTranslation( - const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated) +void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain, + const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated) { static bool warned = false; if (!plural) { @@ -86,12 +101,12 @@ void Translations::addPluralTranslation( errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl; return; } - std::wstring key = textdomain + L"|" + original; + std::wstring key = lang + L"|" + textdomain + L"|" + original; m_plural_translations.emplace(std::move(key), std::pair(plural, translated)); } -void Translations::loadTrTranslation(const std::string &data) +void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data) { std::istringstream is(data); std::string textdomain_narrow; @@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data) } } - addTranslation(textdomain, word1.str(), word2.str()); + addTranslation(lang, textdomain, word1.str(), word2.str()); } } @@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str) return result; } -void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map &entry) +void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::map &entry) { // Process an entry from a PO file and add it to the translation table // Assumes that entry[L"msgid"] is always defined @@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl; return; } - addTranslation(textdomain, original, translated->second); + addTranslation(lang, textdomain, original, translated->second); } else { std::vector translations; for (int i = 0; ; i++) { @@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl break; translations.push_back(translated->second); } - addPluralTranslation(textdomain, plural_form, original, translations); - addPluralTranslation(textdomain, plural_form, plural->second, translations); + addPluralTranslation(lang, textdomain, plural_form, original, translations); + addPluralTranslation(lang, textdomain, plural_form, plural->second, translations); } } @@ -415,7 +431,7 @@ std::optional> Translations::parsePoLine(c return std::pair(prefix, s); } -void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data) +void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data) { std::istringstream is(data); std::string line; @@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std: } } } else { - loadPoEntry(wbasefilename, plural, last_entry); + loadPoEntry(lang, wbasefilename, plural, last_entry); } } last_entry.clear(); @@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std: if (last_entry.find(L"msgid") != last_entry.end()) { if (!skip_last && !last_entry[L"msgid"].empty()) - loadPoEntry(wbasefilename, plural, last_entry); + loadPoEntry(lang, wbasefilename, plural, last_entry); } else if (!last_entry.empty()) { errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl; } } -void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated) +void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated) { std::wstring textdomain = L""; size_t found; @@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl found = noriginal.find('\0'); if (found != std::string::npos) { std::vector translations = str_split(utf8_to_wide(translated), L'\0'); - addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations); - addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations); + addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations); + addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations); } else { - addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated)); + addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated)); } } @@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0) } } -void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data) +void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data) { size_t length = data.length(); std::wstring wbasefilename = utf8_to_wide(basefilename); @@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std: } } } else { - loadMoEntry(wbasefilename, plural_form, original, translated); + loadMoEntry(lang, wbasefilename, plural_form, original, translated); } } @@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std: void Translations::loadTranslation(const std::string &filename, const std::string &data) { + auto lang = utf8_to_wide(getFileLanguage(filename)); const char *trExtension[] = { ".tr", NULL }; const char *poExtension[] = { ".po", NULL }; const char *moExtension[] = { ".mo", NULL }; if (!removeStringEnd(filename, trExtension).empty()) { - loadTrTranslation(data); + loadTrTranslation(lang, data); } else if (!removeStringEnd(filename, poExtension).empty()) { std::string basefilename = str_split(filename, '.')[0]; - loadPoTranslation(basefilename, data); + loadPoTranslation(lang, basefilename, data); } else if (!removeStringEnd(filename, moExtension).empty()) { std::string basefilename = str_split(filename, '.')[0]; - loadMoTranslation(basefilename, data); + loadMoTranslation(lang, basefilename, data); } else { errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl; } diff --git a/src/translation.h b/src/translation.h index c5494dd789..9f8947165f 100644 --- a/src/translation.h +++ b/src/translation.h @@ -20,37 +20,50 @@ class Translations public: void loadTranslation(const std::string &filename, const std::string &data); void clear(); - const std::wstring &getTranslation( + const std::wstring &getTranslation(const std::vector &lang, const std::wstring &textdomain, const std::wstring &s) const; - const std::wstring &getPluralTranslation(const std::wstring &textdomain, + const std::wstring &getPluralTranslation(const std::vector &lang, + const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const; static const std::string_view getFileLanguage(const std::string &filename); static inline bool isTranslationFile(const std::string &filename) { return getFileLanguage(filename) != ""; } - // for testing inline size_t size() { return m_translations.size() + m_plural_translations.size()/2; } + inline bool empty() + { + return size() == 0; + } + +#ifndef SERVER + const std::wstring &getTranslation( + const std::wstring &textdomain, const std::wstring &s) const; + const std::wstring &getPluralTranslation(const std::wstring &textdomain, + const std::wstring &s, unsigned long int number) const; +#endif private: std::unordered_map m_translations; std::unordered_map>> m_plural_translations; - void addTranslation(const std::wstring &textdomain, const std::wstring &original, + void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated); - void addPluralTranslation(const std::wstring &textdomain, + void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated); std::wstring unescapeC(const std::wstring &str); std::optional> parsePoLine(const std::string &line); bool inEscape(const std::wstring &str, size_t pos); - void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map &entry); - void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated); - void loadTrTranslation(const std::string &data); - void loadPoTranslation(const std::string &basefilename, const std::string &data); - void loadMoTranslation(const std::string &basefilename, const std::string &data); + void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::map &entry); + void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated); + void loadTrTranslation(const std::wstring &lang, const std::string &data); + void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data); + void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data); }; diff --git a/src/unittest/CMakeLists.txt b/src/unittest/CMakeLists.txt index c31be6e4e4..c175d65cf5 100644 --- a/src/unittest/CMakeLists.txt +++ b/src/unittest/CMakeLists.txt @@ -19,6 +19,7 @@ set (UNITTEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/test_irrptr.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_irr_matrix4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_irr_rotation.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test_langcode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_logging.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_lbmmanager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_lua.cpp diff --git a/src/unittest/test_langcode.cpp b/src/unittest/test_langcode.cpp new file mode 100644 index 0000000000..e131bc636c --- /dev/null +++ b/src/unittest/test_langcode.cpp @@ -0,0 +1,16 @@ +// Luanti +// SPDX-License-Identifier: LGPL-2.1-or-later + +#include "util/langcode.h" +#include "catch.h" + +TEST_CASE("test langcode") +{ + SECTION("test language list") + { + CHECK(expand_language_list(L"de_DE@euro.UTF-8:fr") == L"de_DE@euro:de_DE:de:fr"); + CHECK(expand_language_list(L"zh_HK:yue_HK:zh_TW") == L"zh_HK:yue_HK:yue:zh_TW:zh"); + CHECK(expand_language_list(L"de_DE:fr:de_CH:en:de:de_AT") == L"de_DE:fr:de_CH:en:de:de_AT"); + CHECK(expand_language_list(L".UTF-8:de:.ISO-8859-1:fr:.GB2312") == L"de:fr"); + } +} diff --git a/src/unittest/test_translations.cpp b/src/unittest/test_translations.cpp index 96f5acb9b6..6f022146e0 100644 --- a/src/unittest/test_translations.cpp +++ b/src/unittest/test_translations.cpp @@ -9,8 +9,11 @@ #define CONTEXT L"context" #define TEXTDOMAIN_PO L"translation_po" #define TEST_PO_NAME "translation_po.de.po" +#define SECONDARY_PO_NAME "translation_po.de_CH.po" #define TEST_MO_NAME "translation_mo.de.mo" +const std::vector lang {L"de"}; + static std::string read_translation_file(const std::string &filename) { auto gamespec = findSubgame("devtest"); @@ -86,17 +89,17 @@ TEST_CASE("test translations") Translations translations; translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME)); - CHECK(translations.size() == 5); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations"); - CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); + CHECK(translations.size() == 7); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations"); + CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context"); } SECTION("MO file parser") @@ -105,8 +108,19 @@ TEST_CASE("test translations") translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME)); CHECK(translations.size() == 2); - CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); - CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result"); - CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result"); + CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context"); + CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result"); + CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result"); + } + + SECTION("Translation fallback") + { + Translations translations; + translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME)); + translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME)); + + CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German"); + CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German"); + CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German"); } } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 0cbf0eaa15..205ae645d4 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -11,6 +11,7 @@ set(util_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/guid.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hashing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/langcode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp ${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp diff --git a/src/util/langcode.cpp b/src/util/langcode.cpp new file mode 100644 index 0000000000..e5e5ccd7f3 --- /dev/null +++ b/src/util/langcode.cpp @@ -0,0 +1,47 @@ +// Luanti +// SPDX-License-Identifier: LGPL-2.1-or-later + +#include +#include "util/string.h" + +std::vector parse_language_list(const std::wstring &lang) +{ + std::unordered_map added_by; + std::vector> expanded; + + for (const auto &name: str_split(lang, L':')) { + auto pos = name.find(L'.'); // strip encoding information + const auto realname = pos == name.npos ? name : name.substr(0, pos); + if (realname.empty()) + continue; + + std::vector basenames = {}; + auto base = realname; + do { + if (added_by[base] == base) + break; + added_by[base] = realname; + basenames.push_back(base); + + pos = base.find_last_of(L"_@"); + base = base.substr(0, pos); + } while (pos != base.npos); + if (!basenames.empty()) + expanded.push_back(std::move(basenames)); + } + + std::vector langlist; + for (auto &basenames: expanded) + { + auto first = basenames.front(); + for (auto &&name: basenames) + if (added_by[name] == first) + langlist.push_back(std::move(name)); + } + return langlist; +} + +std::wstring expand_language_list(const std::wstring &lang) +{ + return str_join(parse_language_list(lang), L":"); +} diff --git a/src/util/langcode.h b/src/util/langcode.h new file mode 100644 index 0000000000..800a0dda0b --- /dev/null +++ b/src/util/langcode.h @@ -0,0 +1,9 @@ +// Luanti +// SPDX-License-Identifier: LGPL-2.1-or-later + +#pragma once +#include +#include + +std::vector parse_language_list(const std::wstring &lang); +std::wstring expand_language_list(const std::wstring &lang); diff --git a/src/util/string.cpp b/src/util/string.cpp index aeec51cb80..1c2646ef36 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -7,6 +7,7 @@ #include "numeric.h" #include "log.h" +#include "gettext.h" #include "hex.h" #include "porting.h" #include "translation.h" @@ -654,9 +655,9 @@ std::string wrap_rows(std::string_view from, unsigned row_len, bool has_color_co */ static void translate_all(std::wstring_view s, size_t &i, - Translations *translations, std::wstring &res); + const std::vector &lang, Translations *translations, std::wstring &res); -static void translate_string(std::wstring_view s, Translations *translations, +static void translate_string(std::wstring_view s, const std::vector &lang, Translations *translations, const std::wstring &textdomain, size_t &i, std::wstring &res, bool use_plural, unsigned long int number) { @@ -716,7 +717,7 @@ static void translate_string(std::wstring_view s, Translations *translations, if (arg_number >= 10) { errorstream << "Ignoring too many arguments to translation" << std::endl; std::wstring arg; - translate_all(s, i, translations, arg); + translate_all(s, i, lang, translations, arg); args.push_back(arg); continue; } @@ -724,7 +725,7 @@ static void translate_string(std::wstring_view s, Translations *translations, output += std::to_wstring(arg_number); ++arg_number; std::wstring arg; - translate_all(s, i, translations, arg); + translate_all(s, i, lang, translations, arg); args.push_back(std::move(arg)); } else { // This is an escape sequence *inside* the template string to translate itself. @@ -739,10 +740,10 @@ static void translate_string(std::wstring_view s, Translations *translations, if (translations != nullptr) { if (use_plural) toutput = translations->getPluralTranslation( - textdomain, output, number); + lang, textdomain, output, number); else toutput = translations->getTranslation( - textdomain, output); + lang, textdomain, output); } else { toutput = output; } @@ -780,7 +781,7 @@ static void translate_string(std::wstring_view s, Translations *translations, } static void translate_all(std::wstring_view s, size_t &i, - Translations *translations, std::wstring &res) + const std::vector &lang, Translations *translations, std::wstring &res) { res.clear(); res.reserve(s.length()); @@ -858,7 +859,7 @@ static void translate_all(std::wstring_view s, size_t &i, } } std::wstring translated; - translate_string(s, translations, textdomain, i, translated, use_plural, number); + translate_string(s, lang, translations, textdomain, i, translated, use_plural, number); res.append(translated); } else { // Another escape sequence, such as colors. Preserve it. @@ -868,17 +869,17 @@ static void translate_all(std::wstring_view s, size_t &i, } // Translate string server side -std::wstring translate_string(std::wstring_view s, Translations *translations) +std::wstring translate_string(std::wstring_view s, const std::vector &lang, Translations *translations) { size_t i = 0; std::wstring res; - translate_all(s, i, translations, res); + translate_all(s, i, lang, translations, res); return res; } std::wstring translate_string(std::wstring_view s) { - return translate_string(s, g_client_translations); + return translate_string(s, get_effective_locale(), g_client_translations); } static const std::array disallowed_dir_names = { diff --git a/src/util/string.h b/src/util/string.h index 78881a9a4c..8707f56294 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -665,7 +665,7 @@ std::vector > split(const std::basic_string &s, T delim) } [[nodiscard]] -std::wstring translate_string(std::wstring_view s, Translations *translations); +std::wstring translate_string(std::wstring_view s, const std::vector &lang, Translations *translations); [[nodiscard]] std::wstring translate_string(std::wstring_view s); @@ -756,11 +756,12 @@ inline const std::string duration_to_string(int sec) * * @return A std::string */ +template [[nodiscard]] -inline std::string str_join(const std::vector &list, - std::string_view delimiter) +inline std::basic_string str_join(const std::vector> &list, + std::basic_string_view delimiter) { - std::ostringstream oss; + std::basic_ostringstream oss; bool first = true; for (const auto &part : list) { if (!first) @@ -771,6 +772,13 @@ inline std::string str_join(const std::vector &list, return oss.str(); } +template +inline std::basic_string str_join(const std::vector> &list, + const T *delimiter) +{ + return str_join(list, std::basic_string_view(delimiter)); +} + #if IS_CLIENT_BUILD /** * Create a UTF8 std::string from an core::stringw.