oss/luanti
oss/luanti
1
0
Fork 0
mirror of https://github.com/luanti-org/luanti.git synced 2025-09-30 19:22:14 +00:00
Code Wiki Activity

Expand Translations class to include language information

Browse source
This commit is contained in:
y5nw 2025-02-24 01:20:42 +01:00 • committed by y5nw
parent e7c6525d47
commit 027a2a67b7
5 changed files with 114 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

6
games/devtest/mods/testtranslations/test_locale/translation_po.de.po
View file

@ -40,3 +40,9 @@ msgstr[1] ""
msgctxt "context" msgctxt "context"
msgid "With context" msgid "With context"
msgstr "Has context" msgstr "Has context"
msgid "In multiple languages"
msgstr "In standard German"
msgid "In one language"
msgstr "Only in standard German"

2
games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po Normal file
View file

@ -0,0 +1,2 @@
msgid "In multiple languages"
msgstr "In Swiss German"

100
src/translation.cpp
View file

@ -36,45 +36,60 @@ void Translations::clear()
m_plural_translations.clear(); m_plural_translations.clear();
} }
const std::wstring &Translations::getTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s) const
{
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_translations.find(lang + basekey);
if (it != m_translations.end())
return it->second;
}
return s;
}
const std::wstring &Translations::getTranslation( const std::wstring &Translations::getTranslation(
const std::wstring &textdomain, const std::wstring &s) const const std::wstring &textdomain, const std::wstring &s) const
{ {
std::wstring key = textdomain + L"|" + s; return getTranslation(get_effective_locale(), textdomain, s);
auto it = m_translations.find(key); }
if (it != m_translations.end())
return it->second; const std::wstring &Translations::getPluralTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_plural_translations.find(lang + basekey);
if (it != m_plural_translations.end()) {
auto n = (*(it->second.first))(number);
const std::vector<std::wstring> &v = it->second.second;
if (n < v.size()) {
if (v[n].empty())
return s;
return v[n];
}
}
}
return s; return s;
} }
const std::wstring &Translations::getPluralTranslation( const std::wstring &Translations::getPluralTranslation(
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{ {
std::wstring key = textdomain + L"|" + s; return getPluralTranslation(get_effective_locale(), textdomain, s, number);
auto it = m_plural_translations.find(key);
if (it != m_plural_translations.end()) {
auto n = (*(it->second.first))(number);
const std::vector<std::wstring> &v = it->second.second;
if (n < v.size()) {
if (v[n].empty())
return s;
return v[n];
}
}
return s;
} }
void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain,
void Translations::addTranslation( const std::wstring &original, const std::wstring &translated)
const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated)
{ {
std::wstring key = textdomain + L"|" + original; std::wstring key = lang + L"|" + textdomain + L"|" + original;
if (!translated.empty()) { if (!translated.empty()) {
m_translations.emplace(std::move(key), std::move(translated)); m_translations.emplace(std::move(key), std::move(translated));
} }
} }
void Translations::addPluralTranslation( void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated) const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
{ {
static bool warned = false; static bool warned = false;
if (!plural) { if (!plural) {
@ -86,12 +101,12 @@ void Translations::addPluralTranslation(
errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl; errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl;
return; return;
} }
std::wstring key = textdomain + L"|" + original; std::wstring key = lang + L"|" + textdomain + L"|" + original;
m_plural_translations.emplace(std::move(key), std::pair(plural, translated)); m_plural_translations.emplace(std::move(key), std::pair(plural, translated));
} }
void Translations::loadTrTranslation(const std::string &data) void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data)
{ {
std::istringstream is(data); std::istringstream is(data);
std::string textdomain_narrow; std::string textdomain_narrow;
@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data)
} }
} }
addTranslation(textdomain, word1.str(), word2.str()); addTranslation(lang, textdomain, word1.str(), word2.str());
} }
} }
@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str)
return result; return result;
} }
void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry) void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
{ {
// Process an entry from a PO file and add it to the translation table // Process an entry from a PO file and add it to the translation table
// Assumes that entry[L"msgid"] is always defined // Assumes that entry[L"msgid"] is always defined
@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl; errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl;
return; return;
} }
addTranslation(textdomain, original, translated->second); addTranslation(lang, textdomain, original, translated->second);
} else { } else {
std::vector<std::wstring> translations; std::vector<std::wstring> translations;
for (int i = 0; ; i++) { for (int i = 0; ; i++) {
@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
break; break;
translations.push_back(translated->second); translations.push_back(translated->second);
} }
addPluralTranslation(textdomain, plural_form, original, translations); addPluralTranslation(lang, textdomain, plural_form, original, translations);
addPluralTranslation(textdomain, plural_form, plural->second, translations); addPluralTranslation(lang, textdomain, plural_form, plural->second, translations);
} }
} }
@ -415,7 +431,7 @@ std::optional<std::pair<std::wstring, std::wstring>> Translations::parsePoLine(c
return std::pair(prefix, s); return std::pair(prefix, s);
} }
void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data) void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{ {
std::istringstream is(data); std::istringstream is(data);
std::string line; std::string line;
@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
} }
} }
} else { } else {
loadPoEntry(wbasefilename, plural, last_entry); loadPoEntry(lang, wbasefilename, plural, last_entry);
} }
} }
last_entry.clear(); last_entry.clear();
@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
if (last_entry.find(L"msgid") != last_entry.end()) { if (last_entry.find(L"msgid") != last_entry.end()) {
if (!skip_last && !last_entry[L"msgid"].empty()) if (!skip_last && !last_entry[L"msgid"].empty())
loadPoEntry(wbasefilename, plural, last_entry); loadPoEntry(lang, wbasefilename, plural, last_entry);
} else if (!last_entry.empty()) { } else if (!last_entry.empty()) {
errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl; errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl;
} }
} }
void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated) void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
{ {
std::wstring textdomain = L""; std::wstring textdomain = L"";
size_t found; size_t found;
@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl
found = noriginal.find('\0'); found = noriginal.find('\0');
if (found != std::string::npos) { if (found != std::string::npos) {
std::vector<std::wstring> translations = str_split(utf8_to_wide(translated), L'\0'); std::vector<std::wstring> translations = str_split(utf8_to_wide(translated), L'\0');
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations); addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations); addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
} else { } else {
addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated)); addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
} }
} }
@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0)
} }
} }
void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data) void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{ {
size_t length = data.length(); size_t length = data.length();
std::wstring wbasefilename = utf8_to_wide(basefilename); std::wstring wbasefilename = utf8_to_wide(basefilename);
@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
} }
} }
} else { } else {
loadMoEntry(wbasefilename, plural_form, original, translated); loadMoEntry(lang, wbasefilename, plural_form, original, translated);
} }
} }
@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
void Translations::loadTranslation(const std::string &filename, const std::string &data) void Translations::loadTranslation(const std::string &filename, const std::string &data)
{ {
auto lang = utf8_to_wide(getFileLanguage(filename));
const char *trExtension[] = { ".tr", NULL }; const char *trExtension[] = { ".tr", NULL };
const char *poExtension[] = { ".po", NULL }; const char *poExtension[] = { ".po", NULL };
const char *moExtension[] = { ".mo", NULL }; const char *moExtension[] = { ".mo", NULL };
if (!removeStringEnd(filename, trExtension).empty()) { if (!removeStringEnd(filename, trExtension).empty()) {
loadTrTranslation(data); loadTrTranslation(lang, data);
} else if (!removeStringEnd(filename, poExtension).empty()) { } else if (!removeStringEnd(filename, poExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0]; std::string basefilename = str_split(filename, '.')[0];
loadPoTranslation(basefilename, data); loadPoTranslation(lang, basefilename, data);
} else if (!removeStringEnd(filename, moExtension).empty()) { } else if (!removeStringEnd(filename, moExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0]; std::string basefilename = str_split(filename, '.')[0];
loadMoTranslation(basefilename, data); loadMoTranslation(lang, basefilename, data);
} else { } else {
errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl; errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl;
} }

28
src/translation.h
View file

@ -20,9 +20,10 @@ class Translations
public: public:
void loadTranslation(const std::string &filename, const std::string &data); void loadTranslation(const std::string &filename, const std::string &data);
void clear(); void clear();
const std::wstring &getTranslation( const std::wstring &getTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain, const std::wstring &s) const; const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain, const std::wstring &getPluralTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const; const std::wstring &s, unsigned long int number) const;
static const std::string_view getFileLanguage(const std::string &filename); static const std::string_view getFileLanguage(const std::string &filename);
static inline bool isTranslationFile(const std::string &filename) static inline bool isTranslationFile(const std::string &filename)
@ -35,22 +36,31 @@ public:
return m_translations.size() + m_plural_translations.size()/2; return m_translations.size() + m_plural_translations.size()/2;
} }
#ifndef SERVER
const std::wstring &getTranslation(
const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const;
#endif
private: private:
std::unordered_map<std::wstring, std::wstring> m_translations; std::unordered_map<std::wstring, std::wstring> m_translations;
std::unordered_map<std::wstring, std::pair<GettextPluralForm::Ptr, std::vector<std::wstring>>> m_plural_translations; std::unordered_map<std::wstring, std::pair<GettextPluralForm::Ptr, std::vector<std::wstring>>> m_plural_translations;
void addTranslation(const std::wstring &textdomain, const std::wstring &original, void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original,
const std::wstring &translated); const std::wstring &translated);
void addPluralTranslation(const std::wstring &textdomain, void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const GettextPluralForm::Ptr &plural, const GettextPluralForm::Ptr &plural,
const std::wstring &original, const std::wstring &original,
std::vector<std::wstring> &translated); std::vector<std::wstring> &translated);
std::wstring unescapeC(const std::wstring &str); std::wstring unescapeC(const std::wstring &str);
std::optional<std::pair<std::wstring, std::wstring>> parsePoLine(const std::string &line); std::optional<std::pair<std::wstring, std::wstring>> parsePoLine(const std::string &line);
bool inEscape(const std::wstring &str, size_t pos); bool inEscape(const std::wstring &str, size_t pos);
void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry); void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated); const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
void loadTrTranslation(const std::string &data); void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
void loadPoTranslation(const std::string &basefilename, const std::string &data); const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
void loadMoTranslation(const std::string &basefilename, const std::string &data); void loadTrTranslation(const std::wstring &lang, const std::string &data);
void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
}; };

42
src/unittest/test_translations.cpp
View file

@ -9,8 +9,11 @@
#define CONTEXT L"context" #define CONTEXT L"context"
#define TEXTDOMAIN_PO L"translation_po" #define TEXTDOMAIN_PO L"translation_po"
#define TEST_PO_NAME "translation_po.de.po" #define TEST_PO_NAME "translation_po.de.po"
#define SECONDARY_PO_NAME "translation_po.de_CH.po"
#define TEST_MO_NAME "translation_mo.de.mo" #define TEST_MO_NAME "translation_mo.de.mo"
const std::vector<std::wstring> lang {L"de"};
static std::string read_translation_file(const std::string &filename) static std::string read_translation_file(const std::string &filename)
{ {
auto gamespec = findSubgame("devtest"); auto gamespec = findSubgame("devtest");
@ -67,17 +70,17 @@ TEST_CASE("test translations")
Translations translations; Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME)); translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
CHECK(translations.size() == 5); CHECK(translations.size() == 7);
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar"); CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated"); CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy"); CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult"); CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order"); CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result"); CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result"); CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated"); CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations"); CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
} }
SECTION("MO file parser") SECTION("MO file parser")
@ -86,8 +89,19 @@ TEST_CASE("test translations")
translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME)); translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME));
CHECK(translations.size() == 2); CHECK(translations.size() == 2);
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context"); CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result"); CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result"); CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result");
}
SECTION("Translation fallback")
{
Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME));
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German");
CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German");
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German");
} }
} }