diff --git a/games/devtest/mods/testtranslations/test_locale/translation_po.de.po b/games/devtest/mods/testtranslations/test_locale/translation_po.de.po index 9a64805a62..5aa6b086cc 100644 --- a/games/devtest/mods/testtranslations/test_locale/translation_po.de.po +++ b/games/devtest/mods/testtranslations/test_locale/translation_po.de.po @@ -40,3 +40,9 @@ msgstr[1] "" msgctxt "context" msgid "With context" msgstr "Has context" + +msgid "In multiple languages" +msgstr "In standard German" + +msgid "In one language" +msgstr "Only in standard German" diff --git a/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po b/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po new file mode 100644 index 0000000000..8145fbcdcf --- /dev/null +++ b/games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po @@ -0,0 +1,2 @@ +msgid "In multiple languages" +msgstr "In Swiss German" diff --git a/src/translation.cpp b/src/translation.cpp index 71469507d1..8ca5efb10a 100644 --- a/src/translation.cpp +++ b/src/translation.cpp @@ -36,45 +36,60 @@ void Translations::clear() m_plural_translations.clear(); } +const std::wstring &Translations::getTranslation(const std::vector &langlist, + const std::wstring &textdomain, const std::wstring &s) const +{ + auto basekey = L"|" + textdomain + L"|" + s; + for (const auto &lang: langlist) { + auto it = m_translations.find(lang + basekey); + if (it != m_translations.end()) + return it->second; + } + return s; +} + const std::wstring &Translations::getTranslation( const std::wstring &textdomain, const std::wstring &s) const { - std::wstring key = textdomain + L"|" + s; - auto it = m_translations.find(key); - if (it != m_translations.end()) - return it->second; + return getTranslation(get_effective_locale(), textdomain, s); +} + +const std::wstring &Translations::getPluralTranslation(const std::vector &langlist, + const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const +{ + auto basekey = L"|" + textdomain + L"|" + s; + for (const auto &lang: langlist) { + auto it = m_plural_translations.find(lang + basekey); + if (it != m_plural_translations.end()) { + auto n = (*(it->second.first))(number); + const std::vector &v = it->second.second; + if (n < v.size()) { + if (v[n].empty()) + return s; + return v[n]; + } + } + } return s; } const std::wstring &Translations::getPluralTranslation( const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const { - std::wstring key = textdomain + L"|" + s; - auto it = m_plural_translations.find(key); - if (it != m_plural_translations.end()) { - auto n = (*(it->second.first))(number); - const std::vector &v = it->second.second; - if (n < v.size()) { - if (v[n].empty()) - return s; - return v[n]; - } - } - return s; + return getPluralTranslation(get_effective_locale(), textdomain, s, number); } - -void Translations::addTranslation( - const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated) +void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain, + const std::wstring &original, const std::wstring &translated) { - std::wstring key = textdomain + L"|" + original; + std::wstring key = lang + L"|" + textdomain + L"|" + original; if (!translated.empty()) { m_translations.emplace(std::move(key), std::move(translated)); } } -void Translations::addPluralTranslation( - const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated) +void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain, + const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated) { static bool warned = false; if (!plural) { @@ -86,12 +101,12 @@ void Translations::addPluralTranslation( errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl; return; } - std::wstring key = textdomain + L"|" + original; + std::wstring key = lang + L"|" + textdomain + L"|" + original; m_plural_translations.emplace(std::move(key), std::pair(plural, translated)); } -void Translations::loadTrTranslation(const std::string &data) +void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data) { std::istringstream is(data); std::string textdomain_narrow; @@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data) } } - addTranslation(textdomain, word1.str(), word2.str()); + addTranslation(lang, textdomain, word1.str(), word2.str()); } } @@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str) return result; } -void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map &entry) +void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::map &entry) { // Process an entry from a PO file and add it to the translation table // Assumes that entry[L"msgid"] is always defined @@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl; return; } - addTranslation(textdomain, original, translated->second); + addTranslation(lang, textdomain, original, translated->second); } else { std::vector translations; for (int i = 0; ; i++) { @@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl break; translations.push_back(translated->second); } - addPluralTranslation(textdomain, plural_form, original, translations); - addPluralTranslation(textdomain, plural_form, plural->second, translations); + addPluralTranslation(lang, textdomain, plural_form, original, translations); + addPluralTranslation(lang, textdomain, plural_form, plural->second, translations); } } @@ -415,7 +431,7 @@ std::optional> Translations::parsePoLine(c return std::pair(prefix, s); } -void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data) +void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data) { std::istringstream is(data); std::string line; @@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std: } } } else { - loadPoEntry(wbasefilename, plural, last_entry); + loadPoEntry(lang, wbasefilename, plural, last_entry); } } last_entry.clear(); @@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std: if (last_entry.find(L"msgid") != last_entry.end()) { if (!skip_last && !last_entry[L"msgid"].empty()) - loadPoEntry(wbasefilename, plural, last_entry); + loadPoEntry(lang, wbasefilename, plural, last_entry); } else if (!last_entry.empty()) { errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl; } } -void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated) +void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated) { std::wstring textdomain = L""; size_t found; @@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl found = noriginal.find('\0'); if (found != std::string::npos) { std::vector translations = str_split(utf8_to_wide(translated), L'\0'); - addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations); - addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations); + addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations); + addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations); } else { - addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated)); + addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated)); } } @@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0) } } -void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data) +void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data) { size_t length = data.length(); std::wstring wbasefilename = utf8_to_wide(basefilename); @@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std: } } } else { - loadMoEntry(wbasefilename, plural_form, original, translated); + loadMoEntry(lang, wbasefilename, plural_form, original, translated); } } @@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std: void Translations::loadTranslation(const std::string &filename, const std::string &data) { + auto lang = utf8_to_wide(getFileLanguage(filename)); const char *trExtension[] = { ".tr", NULL }; const char *poExtension[] = { ".po", NULL }; const char *moExtension[] = { ".mo", NULL }; if (!removeStringEnd(filename, trExtension).empty()) { - loadTrTranslation(data); + loadTrTranslation(lang, data); } else if (!removeStringEnd(filename, poExtension).empty()) { std::string basefilename = str_split(filename, '.')[0]; - loadPoTranslation(basefilename, data); + loadPoTranslation(lang, basefilename, data); } else if (!removeStringEnd(filename, moExtension).empty()) { std::string basefilename = str_split(filename, '.')[0]; - loadMoTranslation(basefilename, data); + loadMoTranslation(lang, basefilename, data); } else { errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl; } diff --git a/src/translation.h b/src/translation.h index c5494dd789..b883a4b239 100644 --- a/src/translation.h +++ b/src/translation.h @@ -20,9 +20,10 @@ class Translations public: void loadTranslation(const std::string &filename, const std::string &data); void clear(); - const std::wstring &getTranslation( + const std::wstring &getTranslation(const std::vector &lang, const std::wstring &textdomain, const std::wstring &s) const; - const std::wstring &getPluralTranslation(const std::wstring &textdomain, + const std::wstring &getPluralTranslation(const std::vector &lang, + const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const; static const std::string_view getFileLanguage(const std::string &filename); static inline bool isTranslationFile(const std::string &filename) @@ -35,22 +36,31 @@ public: return m_translations.size() + m_plural_translations.size()/2; } +#ifndef SERVER + const std::wstring &getTranslation( + const std::wstring &textdomain, const std::wstring &s) const; + const std::wstring &getPluralTranslation(const std::wstring &textdomain, + const std::wstring &s, unsigned long int number) const; +#endif + private: std::unordered_map m_translations; std::unordered_map>> m_plural_translations; - void addTranslation(const std::wstring &textdomain, const std::wstring &original, + void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated); - void addPluralTranslation(const std::wstring &textdomain, + void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector &translated); std::wstring unescapeC(const std::wstring &str); std::optional> parsePoLine(const std::string &line); bool inEscape(const std::wstring &str, size_t pos); - void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map &entry); - void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated); - void loadTrTranslation(const std::string &data); - void loadPoTranslation(const std::string &basefilename, const std::string &data); - void loadMoTranslation(const std::string &basefilename, const std::string &data); + void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::map &entry); + void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename, + const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated); + void loadTrTranslation(const std::wstring &lang, const std::string &data); + void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data); + void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data); }; diff --git a/src/unittest/test_translations.cpp b/src/unittest/test_translations.cpp index 5bab3e15cc..4f14336286 100644 --- a/src/unittest/test_translations.cpp +++ b/src/unittest/test_translations.cpp @@ -9,8 +9,11 @@ #define CONTEXT L"context" #define TEXTDOMAIN_PO L"translation_po" #define TEST_PO_NAME "translation_po.de.po" +#define SECONDARY_PO_NAME "translation_po.de_CH.po" #define TEST_MO_NAME "translation_mo.de.mo" +const std::vector lang {L"de"}; + static std::string read_translation_file(const std::string &filename) { auto gamespec = findSubgame("devtest"); @@ -67,17 +70,17 @@ TEST_CASE("test translations") Translations translations; translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME)); - CHECK(translations.size() == 5); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult"); - CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated"); - CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations"); - CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); + CHECK(translations.size() == 7); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult"); + CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated"); + CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations"); + CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context"); } SECTION("MO file parser") @@ -86,8 +89,19 @@ TEST_CASE("test translations") translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME)); CHECK(translations.size() == 2); - CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); - CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result"); - CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result"); + CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context"); + CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result"); + CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result"); + } + + SECTION("Translation fallback") + { + Translations translations; + translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME)); + translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME)); + + CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German"); + CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German"); + CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German"); } }