oss/luanti
oss/luanti
1
0
Fork 0
mirror of https://github.com/luanti-org/luanti.git synced 2025-09-30 19:22:14 +00:00
Code Wiki Activity

Expand Translations class to include language information

Browse source
This commit is contained in:
y5nw 2025-02-24 01:20:42 +01:00 • committed by y5nw
parent e7c6525d47
commit 027a2a67b7
5 changed files with 114 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

6
games/devtest/mods/testtranslations/test_locale/translation_po.de.po
View file

@ -40,3 +40,9 @@ msgstr[1] ""
msgctxt "context"
msgid "With context"
msgstr "Has context"
msgid "In multiple languages"
msgstr "In standard German"
msgid "In one language"
msgstr "Only in standard German"

2
games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po Normal file
View file

@ -0,0 +1,2 @@
msgid "In multiple languages"
msgstr "In Swiss German"

80
src/translation.cpp
View file

@ -36,21 +36,30 @@ void Translations::clear()
m_plural_translations.clear();
}
const std::wstring &Translations::getTranslation(
const std::wstring &Translations::getTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s) const
{
std::wstring key = textdomain + L"|" + s;
auto it = m_translations.find(key);
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_translations.find(lang + basekey);
if (it != m_translations.end())
return it->second;
}
return s;
}
const std::wstring &Translations::getPluralTranslation(
const std::wstring &Translations::getTranslation(
const std::wstring &textdomain, const std::wstring &s) const
{
return getTranslation(get_effective_locale(), textdomain, s);
}
const std::wstring &Translations::getPluralTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{
std::wstring key = textdomain + L"|" + s;
auto it = m_plural_translations.find(key);
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_plural_translations.find(lang + basekey);
if (it != m_plural_translations.end()) {
auto n = (*(it->second.first))(number);
const std::vector<std::wstring> &v = it->second.second;
@ -60,21 +69,27 @@ const std::wstring &Translations::getPluralTranslation(
return v[n];
}
}
}
return s;
}
void Translations::addTranslation(
const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated)
const std::wstring &Translations::getPluralTranslation(
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{
std::wstring key = textdomain + L"|" + original;
return getPluralTranslation(get_effective_locale(), textdomain, s, number);
}
void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain,
const std::wstring &original, const std::wstring &translated)
{
std::wstring key = lang + L"|" + textdomain + L"|" + original;
if (!translated.empty()) {
m_translations.emplace(std::move(key), std::move(translated));
}
}
void Translations::addPluralTranslation(
const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
{
static bool warned = false;
if (!plural) {
@ -86,12 +101,12 @@ void Translations::addPluralTranslation(
errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl;
return;
}
std::wstring key = textdomain + L"|" + original;
std::wstring key = lang + L"|" + textdomain + L"|" + original;
m_plural_translations.emplace(std::move(key), std::pair(plural, translated));
}
void Translations::loadTrTranslation(const std::string &data)
void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data)
{
std::istringstream is(data);
std::string textdomain_narrow;
@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data)
}
}
addTranslation(textdomain, word1.str(), word2.str());
addTranslation(lang, textdomain, word1.str(), word2.str());
}
}
@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str)
return result;
}
void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
{
// Process an entry from a PO file and add it to the translation table
// Assumes that entry[L"msgid"] is always defined
@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl;
return;
}
addTranslation(textdomain, original, translated->second);
addTranslation(lang, textdomain, original, translated->second);
} else {
std::vector<std::wstring> translations;
for (int i = 0; ; i++) {
@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
break;
translations.push_back(translated->second);
}
addPluralTranslation(textdomain, plural_form, original, translations);
addPluralTranslation(textdomain, plural_form, plural->second, translations);
addPluralTranslation(lang, textdomain, plural_form, original, translations);
addPluralTranslation(lang, textdomain, plural_form, plural->second, translations);
}
}
@ -415,7 +431,7 @@ std::optional<std::pair<std::wstring, std::wstring>> Translations::parsePoLine(c
return std::pair(prefix, s);
}
void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data)
void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{
std::istringstream is(data);
std::string line;
@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
}
}
} else {
loadPoEntry(wbasefilename, plural, last_entry);
loadPoEntry(lang, wbasefilename, plural, last_entry);
}
}
last_entry.clear();
@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
if (last_entry.find(L"msgid") != last_entry.end()) {
if (!skip_last && !last_entry[L"msgid"].empty())
loadPoEntry(wbasefilename, plural, last_entry);
loadPoEntry(lang, wbasefilename, plural, last_entry);
} else if (!last_entry.empty()) {
errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl;
}
}
void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
{
std::wstring textdomain = L"";
size_t found;
@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl
found = noriginal.find('\0');
if (found != std::string::npos) {
std::vector<std::wstring> translations = str_split(utf8_to_wide(translated), L'\0');
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
} else {
addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
}
}
@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0)
}
}
void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data)
void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{
size_t length = data.length();
std::wstring wbasefilename = utf8_to_wide(basefilename);
@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
}
}
} else {
loadMoEntry(wbasefilename, plural_form, original, translated);
loadMoEntry(lang, wbasefilename, plural_form, original, translated);
}
}
@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
void Translations::loadTranslation(const std::string &filename, const std::string &data)
{
auto lang = utf8_to_wide(getFileLanguage(filename));
const char *trExtension[] = { ".tr", NULL };
const char *poExtension[] = { ".po", NULL };
const char *moExtension[] = { ".mo", NULL };
if (!removeStringEnd(filename, trExtension).empty()) {
loadTrTranslation(data);
loadTrTranslation(lang, data);
} else if (!removeStringEnd(filename, poExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0];
loadPoTranslation(basefilename, data);
loadPoTranslation(lang, basefilename, data);
} else if (!removeStringEnd(filename, moExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0];
loadMoTranslation(basefilename, data);
loadMoTranslation(lang, basefilename, data);
} else {
errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl;
}

28
src/translation.h
View file

@ -20,9 +20,10 @@ class Translations
public:
void loadTranslation(const std::string &filename, const std::string &data);
void clear();
const std::wstring &getTranslation(
const std::wstring &getTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain,
const std::wstring &getPluralTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const;
static const std::string_view getFileLanguage(const std::string &filename);
static inline bool isTranslationFile(const std::string &filename)
@ -35,22 +36,31 @@ public:
return m_translations.size() + m_plural_translations.size()/2;
}
#ifndef SERVER
const std::wstring &getTranslation(
const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const;
#endif
private:
std::unordered_map<std::wstring, std::wstring> m_translations;
std::unordered_map<std::wstring, std::pair<GettextPluralForm::Ptr, std::vector<std::wstring>>> m_plural_translations;
void addTranslation(const std::wstring &textdomain, const std::wstring &original,
void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original,
const std::wstring &translated);
void addPluralTranslation(const std::wstring &textdomain,
void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const GettextPluralForm::Ptr &plural,
const std::wstring &original,
std::vector<std::wstring> &translated);
std::wstring unescapeC(const std::wstring &str);
std::optional<std::pair<std::wstring, std::wstring>> parsePoLine(const std::string &line);
bool inEscape(const std::wstring &str, size_t pos);
void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
void loadTrTranslation(const std::string &data);
void loadPoTranslation(const std::string &basefilename, const std::string &data);
void loadMoTranslation(const std::string &basefilename, const std::string &data);
void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
void loadTrTranslation(const std::wstring &lang, const std::string &data);
void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
};

42
src/unittest/test_translations.cpp
View file

@ -9,8 +9,11 @@
#define CONTEXT L"context"
#define TEXTDOMAIN_PO L"translation_po"
#define TEST_PO_NAME "translation_po.de.po"
#define SECONDARY_PO_NAME "translation_po.de_CH.po"
#define TEST_MO_NAME "translation_mo.de.mo"
const std::vector<std::wstring> lang {L"de"};
static std::string read_translation_file(const std::string &filename)
{
auto gamespec = findSubgame("devtest");
@ -67,17 +70,17 @@ TEST_CASE("test translations")
Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
CHECK(translations.size() == 5);
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
CHECK(translations.size() == 7);
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
}
SECTION("MO file parser")
@ -86,8 +89,19 @@ TEST_CASE("test translations")
translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME));
CHECK(translations.size() == 2);
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result");
CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result");
}
SECTION("Translation fallback")
{
Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME));
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German");
CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German");
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German");
}
}