oss/luanti
oss/luanti
1
0
Fork 0
mirror of https://github.com/luanti-org/luanti.git synced 2025-09-30 19:22:14 +00:00
Code Wiki Activity

Merge 6fd76674ac into 421835a30e

Browse source
This commit is contained in:
y5nw 2025-09-26 23:56:21 +03:00 • committed by GitHub
commit e064e758a0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 337 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

12
games/devtest/mods/testtranslations/init.lua
View file

@ -5,12 +5,18 @@ local function send_compare(name, text)
core.get_translated_string("", text), text, core.get_translated_string("fr", text)))
end
local function send_multilang_compare(name, text)
core.chat_send_player(name, ("%s | %s | %s | %s"):format(
core.get_translated_string("", text), text,
core.get_translated_string("fr_CH:fr", text), core.get_translated_string("fr:fr_CH", text)))
end
core.register_chatcommand("testtranslations", {
params = "",
description = "Test translations",
privs = {},
func = function(name, param)
core.chat_send_player(name, "Please ensure your locale is set to \"fr\"")
core.chat_send_player(name, "Please ensure your locale is set to \"fr_CH\"")
core.chat_send_player(name, "Untranslated | Client-side translation | Server-side translation (fr)")
send_compare(name, S("Testing .tr files: untranslated"))
send_compare(name, S("Testing .po files: untranslated"))
@ -22,5 +28,9 @@ core.register_chatcommand("testtranslations", {
send_compare(name, NS("@1: .po singular", "@1: .po plural", i, tostring(i)))
send_compare(name, NS("@1: .mo singular", "@1: .mo plural", i, tostring(i)))
end
core.chat_send_player(name, "Untranslated | Client-side translations" ..
" | Server-side translation (fr_CH:fr) | Server-side translation (fr:fr_CH)")
send_multilang_compare(name, S("Testing translation with multiple languages"))
send_multilang_compare(name, S("Testing French-only translation: untranslated"))
end
})

8
games/devtest/mods/testtranslations/locale/translation_po.fr.po
View file

@ -20,3 +20,11 @@ msgstr "Testing fuzzy .po entry: translated (wrong)"
msgid "Testing .po without context: untranslated"
msgstr "Testing .po without context: translated"
msgctxt "testtranslations"
msgid "Testing translation with multiple languages"
msgstr "Testing translation: translated in standard French"
msgctxt "testtranslations"
msgid "Testing French-only translation: untranslated"
msgstr "Testing French-only translation: translated"

3
games/devtest/mods/testtranslations/locale/translation_po.fr_CH.po Normal file
View file

@ -0,0 +1,3 @@
msgctxt "testtranslations"
msgid "Testing translation with multiple languages"
msgstr "Testing translation: translated in Swiss French"

6
games/devtest/mods/testtranslations/test_locale/translation_po.de.po
View file

@ -40,3 +40,9 @@ msgstr[1] ""
msgctxt "context"
msgid "With context"
msgstr "Has context"
msgid "In multiple languages"
msgstr "In standard German"
msgid "In one language"
msgstr "Only in standard German"

2
games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po Normal file
View file

@ -0,0 +1,2 @@
msgid "In multiple languages"
msgstr "In Swiss German"

26
src/gettext.cpp
View file

@ -6,6 +6,7 @@
#include <cstring>
#include <iostream>
#include "gettext.h"
#include "util/langcode.h"
#include "util/string.h"
#include "porting.h"
#include "log.h"
@ -146,6 +147,10 @@ static void MSVC_LocaleWorkaround(int argc, char* argv[])
#endif
static std::string configured_locale;
static std::vector<std::wstring> effective_locale;
static std::string effective_locale_string;
/******************************************************************************/
void init_gettext(const char *path, const std::string &configured_language,
int argc, char *argv[])
@ -223,6 +228,17 @@ void init_gettext(const char *path, const std::string &configured_language,
setlocale(LC_ALL, "");
#endif // if USE_GETTEXT
// Set up locale for in-game translations
configured_locale = configured_language;
if (configured_locale.empty()) {
if (auto lang = getenv("LANGUAGE"); lang && *lang)
configured_locale = lang;
else
configured_locale = getenv("LANG");
}
effective_locale = parse_language_list(utf8_to_wide(configured_locale));
effective_locale_string = wide_to_utf8(str_join(effective_locale, L":"));
/* no matter what locale is used we need number format to be "C" */
/* to ensure formspec parameters are evaluated correctly! */
@ -230,3 +246,13 @@ void init_gettext(const char *path, const std::string &configured_language,
infostream << "Message locale is now set to: "
<< setlocale(LC_ALL, 0) << std::endl;
}
const std::vector<std::wstring> &get_effective_locale()
{
return effective_locale;
}
const std::string &get_client_language_code()
{
return effective_locale_string;
}

13
src/gettext.h
View file

@ -4,6 +4,7 @@
#pragma once
#include <vector>
#include "config.h" // for USE_GETTEXT
#include "porting.h"
#include "util/string.h"
@ -103,3 +104,15 @@ inline std::string fmtgettext(const char *format, Args&&... args)
return buf;
}
/**
* Returns the effective locale setting for in-game translations.
* @return A vector of language codes based on priority.
*/
const std::vector<std::wstring> &get_effective_locale();
/**
* Returns the effective locale setting for in-game translations.
* @return A string of the expanded language code.
*/
const std::string &get_client_language_code();

52
src/gui/guiEngine.cpp
View file

@ -229,42 +229,46 @@ std::string findLocaleFileInMods(const std::string &path, const std::string &fil
/******************************************************************************/
Translations *GUIEngine::getContentTranslations(const std::string &path,
const std::string &domain, const std::string &lang_code)
const std::string &domain, const std::vector<std::wstring> &lang)
{
if (domain.empty() || lang_code.empty())
if (domain.empty() || lang.empty())
return nullptr;
std::string filename_no_ext = domain + "." + lang_code;
std::string key = path + DIR_DELIM "locale" DIR_DELIM + filename_no_ext;
std::string key = path + DIR_DELIM "locale" DIR_DELIM + domain;
if (key == m_last_translations_key)
return &m_last_translations;
std::string trans_path = key;
switch (getContentType(path)) {
case ContentType::GAME:
trans_path = findLocaleFileInMods(path + DIR_DELIM "mods" DIR_DELIM,
filename_no_ext);
break;
case ContentType::MODPACK:
trans_path = findLocaleFileInMods(path, filename_no_ext);
break;
default:
trans_path = findLocaleFileWithExtension(trans_path);
break;
Translations translations;
for (const auto &lang_code: lang) {
auto utf8_lang_code = wide_to_utf8(lang_code);
std::string filename_no_ext = domain + "." + utf8_lang_code;
auto trans_path = key + "." + utf8_lang_code;
switch (getContentType(path)) {
case ContentType::GAME:
trans_path = findLocaleFileInMods(path + DIR_DELIM "mods" DIR_DELIM,
filename_no_ext);
break;
case ContentType::MODPACK:
trans_path = findLocaleFileInMods(path, filename_no_ext);
break;
default:
trans_path = findLocaleFileWithExtension(trans_path);
break;
}
if (trans_path.empty())
continue;
std::string data;
if (fs::ReadFile(trans_path, data)) {
translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data);
}
}
if (trans_path.empty())
if (translations.empty())
return nullptr;
m_last_translations_key = key;
m_last_translations = {};
std::string data;
if (fs::ReadFile(trans_path, data)) {
m_last_translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data);
}
m_last_translations = std::move(translations);
return &m_last_translations;
}

2
src/gui/guiEngine.h
View file

@ -156,7 +156,7 @@ public:
* change with the next call to `getContentTranslations`.
* */
Translations *getContentTranslations(const std::string &path,
const std::string &domain, const std::string &lang_code);
const std::string &domain, const std::vector<std::wstring> &lang_code);
private:
std::string m_last_translations_key;

8
src/network/clientpackethandler.cpp
View file

@ -142,13 +142,7 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt)
<< m_recommended_send_interval<<std::endl;
// Reply to server
/*~ DO NOT TRANSLATE THIS LITERALLY!
This is a special string which needs to contain the translation's
language code (e.g. "de" for German). */
std::string lang = gettext("LANG_CODE");
if (lang == "LANG_CODE")
lang.clear();
std::string lang = get_client_language_code();
NetworkPacket resp_pkt(TOSERVER_INIT2, sizeof(u16) + lang.size());
resp_pkt << lang;
Send(&resp_pkt);

5
src/script/lua_api/l_client.cpp
View file

@ -177,10 +177,7 @@ int ModApiClient::l_get_language(lua_State *L)
#else
char *locale = setlocale(LC_MESSAGES, NULL);
#endif
std::string lang = gettext("LANG_CODE");
if (lang == "LANG_CODE")
lang.clear();
std::string lang = get_client_language_code();
lua_pushstring(L, locale);
lua_pushstring(L, lang.c_str());
return 2;

3
src/script/lua_api/l_env.cpp
View file

@ -1376,9 +1376,10 @@ int ModApiEnv::l_get_translated_string(lua_State * L)
std::string lang_code = luaL_checkstring(L, 1);
std::string string = luaL_checkstring(L, 2);
auto lang = str_split(utf8_to_wide(lang_code), L':');
auto *translations = getServer(L)->getTranslationLanguage(lang_code);
string = wide_to_utf8(translate_string(utf8_to_wide(string), translations));
string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations));
lua_pushstring(L, string.c_str());
return 1;
}

11
src/script/lua_api/l_mainmenu.cpp
View file

@ -527,12 +527,10 @@ int ModApiMainMenu::l_get_content_translation(lua_State *L)
std::string path = luaL_checkstring(L, 1);
std::string domain = luaL_checkstring(L, 2);
std::string string = luaL_checkstring(L, 3);
std::string lang = gettext("LANG_CODE");
if (lang == "LANG_CODE")
lang = "";
auto lang = get_effective_locale();
auto *translations = engine->getContentTranslations(path, domain, lang);
string = wide_to_utf8(translate_string(utf8_to_wide(string), translations));
string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations));
lua_pushstring(L, string.c_str());
return 1;
}
@ -879,10 +877,7 @@ int ModApiMainMenu::l_download_file(lua_State *L)
/******************************************************************************/
int ModApiMainMenu::l_get_language(lua_State *L)
{
std::string lang = gettext("LANG_CODE");
if (lang == "LANG_CODE")
lang = "";
std::string lang = get_client_language_code();
lua_pushstring(L, lang.c_str());
return 1;
}

43
src/server.cpp
View file

@ -2685,22 +2685,18 @@ void Server::fillMediaCache()
infostream << "Server: " << m_media.size() << " media files collected" << std::endl;
}
void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code)
void Server::sendMediaAnnouncement(session_t peer_id, const std::string &langstring)
{
std::string translation_formats[3] = { ".tr", ".po", ".mo" };
std::string lang_suffixes[3];
for (size_t i = 0; i < 3; i++) {
lang_suffixes[i].append(".").append(lang_code).append(translation_formats[i]);
}
std::unordered_set<std::string> langs;
for (const auto &lang_code: str_split(langstring, ':'))
langs.insert(lang_code);
auto include = [&] (const std::string &name, const MediaInfo &info) -> bool {
if (info.no_announce)
return false;
for (size_t j = 0; j < 3; j++) {
if (str_ends_with(name, translation_formats[j]) && !str_ends_with(name, lang_suffixes[j])) {
return false;
}
}
if (auto filelang = Translations::getFileLanguage(name);
!filelang.empty() && langs.find(std::string(filelang)) == langs.end())
return false;
return true;
};
@ -4288,28 +4284,33 @@ void Server::broadcastModChannelMessage(const std::string &channel,
}
}
Translations *Server::getTranslationLanguage(const std::string &lang_code)
Translations *Server::getTranslationLanguage(const std::string &lang)
{
if (lang_code.empty())
if (lang.empty())
return nullptr;
auto it = server_translations.find(lang_code);
if (it != server_translations.end())
return &it->second; // Already loaded
std::unordered_set<std::string> load_langs;
// [] will create an entry
auto *translations = &server_translations[lang_code];
for (const auto &lang_code: str_split(lang, ':')) {
if (loaded_translations.find(lang_code) == loaded_translations.end()) {
load_langs.insert(lang_code);
loaded_translations.insert(lang_code);
}
}
if (load_langs.empty())
return &server_translations;
for (const auto &i : m_media) {
if (Translations::getFileLanguage(i.first) == lang_code) {
if (load_langs.find(std::string(Translations::getFileLanguage(i.first))) != load_langs.end()) {
std::string data;
if (fs::ReadFile(i.second.path, data, true)) {
translations->loadTranslation(i.first, data);
server_translations.loadTranslation(i.first, data);
}
}
}
return translations;
return &server_translations;
}
std::unordered_map<std::string, std::string> Server::getMediaList()

3
src/server.h
View file

@ -713,7 +713,8 @@ private:
// Craft definition manager
IWritableCraftDefManager *m_craftdef;
std::unordered_map<std::string, Translations> server_translations;
Translations server_translations;
std::unordered_set<std::string> loaded_translations;
ModIPCStore m_ipcstore;

100
src/translation.cpp
View file

@ -36,45 +36,60 @@ void Translations::clear()
m_plural_translations.clear();
}
const std::wstring &Translations::getTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s) const
{
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_translations.find(lang + basekey);
if (it != m_translations.end())
return it->second;
}
return s;
}
const std::wstring &Translations::getTranslation(
const std::wstring &textdomain, const std::wstring &s) const
{
std::wstring key = textdomain + L"|" + s;
auto it = m_translations.find(key);
if (it != m_translations.end())
return it->second;
return getTranslation(get_effective_locale(), textdomain, s);
}
const std::wstring &Translations::getPluralTranslation(const std::vector<std::wstring> &langlist,
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{
auto basekey = L"|" + textdomain + L"|" + s;
for (const auto &lang: langlist) {
auto it = m_plural_translations.find(lang + basekey);
if (it != m_plural_translations.end()) {
auto n = (*(it->second.first))(number);
const std::vector<std::wstring> &v = it->second.second;
if (n < v.size()) {
if (v[n].empty())
return s;
return v[n];
}
}
}
return s;
}
const std::wstring &Translations::getPluralTranslation(
const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
{
std::wstring key = textdomain + L"|" + s;
auto it = m_plural_translations.find(key);
if (it != m_plural_translations.end()) {
auto n = (*(it->second.first))(number);
const std::vector<std::wstring> &v = it->second.second;
if (n < v.size()) {
if (v[n].empty())
return s;
return v[n];
}
}
return s;
return getPluralTranslation(get_effective_locale(), textdomain, s, number);
}
void Translations::addTranslation(
const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated)
void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain,
const std::wstring &original, const std::wstring &translated)
{
std::wstring key = textdomain + L"|" + original;
std::wstring key = lang + L"|" + textdomain + L"|" + original;
if (!translated.empty()) {
m_translations.emplace(std::move(key), std::move(translated));
}
}
void Translations::addPluralTranslation(
const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
{
static bool warned = false;
if (!plural) {
@ -86,12 +101,12 @@ void Translations::addPluralTranslation(
errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl;
return;
}
std::wstring key = textdomain + L"|" + original;
std::wstring key = lang + L"|" + textdomain + L"|" + original;
m_plural_translations.emplace(std::move(key), std::pair(plural, translated));
}
void Translations::loadTrTranslation(const std::string &data)
void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data)
{
std::istringstream is(data);
std::string textdomain_narrow;
@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data)
}
}
addTranslation(textdomain, word1.str(), word2.str());
addTranslation(lang, textdomain, word1.str(), word2.str());
}
}
@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str)
return result;
}
void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
{
// Process an entry from a PO file and add it to the translation table
// Assumes that entry[L"msgid"] is always defined
@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl;
return;
}
addTranslation(textdomain, original, translated->second);
addTranslation(lang, textdomain, original, translated->second);
} else {
std::vector<std::wstring> translations;
for (int i = 0; ; i++) {
@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
break;
translations.push_back(translated->second);
}
addPluralTranslation(textdomain, plural_form, original, translations);
addPluralTranslation(textdomain, plural_form, plural->second, translations);
addPluralTranslation(lang, textdomain, plural_form, original, translations);
addPluralTranslation(lang, textdomain, plural_form, plural->second, translations);
}
}
@ -415,7 +431,7 @@ std::optional<std::pair<std::wstring, std::wstring>> Translations::parsePoLine(c
return std::pair(prefix, s);
}
void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data)
void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{
std::istringstream is(data);
std::string line;
@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
}
}
} else {
loadPoEntry(wbasefilename, plural, last_entry);
loadPoEntry(lang, wbasefilename, plural, last_entry);
}
}
last_entry.clear();
@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
if (last_entry.find(L"msgid") != last_entry.end()) {
if (!skip_last && !last_entry[L"msgid"].empty())
loadPoEntry(wbasefilename, plural, last_entry);
loadPoEntry(lang, wbasefilename, plural, last_entry);
} else if (!last_entry.empty()) {
errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl;
}
}
void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
{
std::wstring textdomain = L"";
size_t found;
@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl
found = noriginal.find('\0');
if (found != std::string::npos) {
std::vector<std::wstring> translations = str_split(utf8_to_wide(translated), L'\0');
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
} else {
addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
}
}
@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0)
}
}
void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data)
void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
{
size_t length = data.length();
std::wstring wbasefilename = utf8_to_wide(basefilename);
@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
}
}
} else {
loadMoEntry(wbasefilename, plural_form, original, translated);
loadMoEntry(lang, wbasefilename, plural_form, original, translated);
}
}
@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
void Translations::loadTranslation(const std::string &filename, const std::string &data)
{
auto lang = utf8_to_wide(getFileLanguage(filename));
const char *trExtension[] = { ".tr", NULL };
const char *poExtension[] = { ".po", NULL };
const char *moExtension[] = { ".mo", NULL };
if (!removeStringEnd(filename, trExtension).empty()) {
loadTrTranslation(data);
loadTrTranslation(lang, data);
} else if (!removeStringEnd(filename, poExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0];
loadPoTranslation(basefilename, data);
loadPoTranslation(lang, basefilename, data);
} else if (!removeStringEnd(filename, moExtension).empty()) {
std::string basefilename = str_split(filename, '.')[0];
loadMoTranslation(basefilename, data);
loadMoTranslation(lang, basefilename, data);
} else {
errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl;
}

33
src/translation.h
View file

@ -20,37 +20,50 @@ class Translations
public:
void loadTranslation(const std::string &filename, const std::string &data);
void clear();
const std::wstring &getTranslation(
const std::wstring &getTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain,
const std::wstring &getPluralTranslation(const std::vector<std::wstring> &lang,
const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const;
static const std::string_view getFileLanguage(const std::string &filename);
static inline bool isTranslationFile(const std::string &filename)
{
return getFileLanguage(filename) != "";
}
// for testing
inline size_t size()
{
return m_translations.size() + m_plural_translations.size()/2;
}
inline bool empty()
{
return size() == 0;
}
#ifndef SERVER
const std::wstring &getTranslation(
const std::wstring &textdomain, const std::wstring &s) const;
const std::wstring &getPluralTranslation(const std::wstring &textdomain,
const std::wstring &s, unsigned long int number) const;
#endif
private:
std::unordered_map<std::wstring, std::wstring> m_translations;
std::unordered_map<std::wstring, std::pair<GettextPluralForm::Ptr, std::vector<std::wstring>>> m_plural_translations;
void addTranslation(const std::wstring &textdomain, const std::wstring &original,
void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original,
const std::wstring &translated);
void addPluralTranslation(const std::wstring &textdomain,
void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
const GettextPluralForm::Ptr &plural,
const std::wstring &original,
std::vector<std::wstring> &translated);
std::wstring unescapeC(const std::wstring &str);
std::optional<std::pair<std::wstring, std::wstring>> parsePoLine(const std::string &line);
bool inEscape(const std::wstring &str, size_t pos);
void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
void loadTrTranslation(const std::string &data);
void loadPoTranslation(const std::string &basefilename, const std::string &data);
void loadMoTranslation(const std::string &basefilename, const std::string &data);
void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
void loadTrTranslation(const std::wstring &lang, const std::string &data);
void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
};

1
src/unittest/CMakeLists.txt
View file

@ -19,6 +19,7 @@ set (UNITTEST_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/test_irrptr.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_irr_matrix4.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_irr_rotation.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_langcode.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_logging.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_lbmmanager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_lua.cpp

16
src/unittest/test_langcode.cpp Normal file
View file

@ -0,0 +1,16 @@
// Luanti
// SPDX-License-Identifier: LGPL-2.1-or-later
#include "util/langcode.h"
#include "catch.h"
TEST_CASE("test langcode")
{
SECTION("test language list")
{
CHECK(expand_language_list(L"de_DE@euro.UTF-8:fr") == L"de_DE@euro:de_DE:de:fr");
CHECK(expand_language_list(L"zh_HK:yue_HK:zh_TW") == L"zh_HK:yue_HK:yue:zh_TW:zh");
CHECK(expand_language_list(L"de_DE:fr:de_CH:en:de:de_AT") == L"de_DE:fr:de_CH:en:de:de_AT");
CHECK(expand_language_list(L".UTF-8:de:.ISO-8859-1:fr:.GB2312") == L"de:fr");
}
}

42
src/unittest/test_translations.cpp
View file

@ -9,8 +9,11 @@
#define CONTEXT L"context"
#define TEXTDOMAIN_PO L"translation_po"
#define TEST_PO_NAME "translation_po.de.po"
#define SECONDARY_PO_NAME "translation_po.de_CH.po"
#define TEST_MO_NAME "translation_mo.de.mo"
const std::vector<std::wstring> lang {L"de"};
static std::string read_translation_file(const std::string &filename)
{
auto gamespec = findSubgame("devtest");
@ -86,17 +89,17 @@ TEST_CASE("test translations")
Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
CHECK(translations.size() == 5);
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
CHECK(translations.size() == 7);
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
}
SECTION("MO file parser")
@ -105,8 +108,19 @@ TEST_CASE("test translations")
translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME));
CHECK(translations.size() == 2);
CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result");
CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result");
CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result");
}
SECTION("Translation fallback")
{
Translations translations;
translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME));
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German");
CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German");
CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German");
}
}

1
src/util/CMakeLists.txt
View file

@ -11,6 +11,7 @@ set(util_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/guid.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hashing.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp
${CMAKE_CURRENT_SOURCE_DIR}/langcode.cpp
${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp
${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp
${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp

47
src/util/langcode.cpp Normal file
View file

@ -0,0 +1,47 @@
// Luanti
// SPDX-License-Identifier: LGPL-2.1-or-later
#include <unordered_map>
#include "util/string.h"
std::vector<std::wstring> parse_language_list(const std::wstring &lang)
{
std::unordered_map<std::wstring, std::wstring> added_by;
std::vector<std::vector<std::wstring>> expanded;
for (const auto &name: str_split(lang, L':')) {
auto pos = name.find(L'.'); // strip encoding information
const auto realname = pos == name.npos ? name : name.substr(0, pos);
if (realname.empty())
continue;
std::vector<std::wstring> basenames = {};
auto base = realname;
do {
if (added_by[base] == base)
break;
added_by[base] = realname;
basenames.push_back(base);
pos = base.find_last_of(L"_@");
base = base.substr(0, pos);
} while (pos != base.npos);
if (!basenames.empty())
expanded.push_back(std::move(basenames));
}
std::vector<std::wstring> langlist;
for (auto &basenames: expanded)
{
auto first = basenames.front();
for (auto &&name: basenames)
if (added_by[name] == first)
langlist.push_back(std::move(name));
}
return langlist;
}
std::wstring expand_language_list(const std::wstring &lang)
{
return str_join(parse_language_list(lang), L":");
}

9
src/util/langcode.h Normal file
View file

@ -0,0 +1,9 @@
// Luanti
// SPDX-License-Identifier: LGPL-2.1-or-later
#pragma once
#include <vector>
#include <string>
std::vector<std::wstring> parse_language_list(const std::wstring &lang);
std::wstring expand_language_list(const std::wstring &lang);

23
src/util/string.cpp
View file

@ -7,6 +7,7 @@
#include "numeric.h"
#include "log.h"
#include "gettext.h"
#include "hex.h"
#include "porting.h"
#include "translation.h"
@ -654,9 +655,9 @@ std::string wrap_rows(std::string_view from, unsigned row_len, bool has_color_co
*/
static void translate_all(std::wstring_view s, size_t &i,
Translations *translations, std::wstring &res);
const std::vector<std::wstring> &lang, Translations *translations, std::wstring &res);
static void translate_string(std::wstring_view s, Translations *translations,
static void translate_string(std::wstring_view s, const std::vector<std::wstring> &lang, Translations *translations,
const std::wstring &textdomain, size_t &i, std::wstring &res,
bool use_plural, unsigned long int number)
{
@ -716,7 +717,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
if (arg_number >= 10) {
errorstream << "Ignoring too many arguments to translation" << std::endl;
std::wstring arg;
translate_all(s, i, translations, arg);
translate_all(s, i, lang, translations, arg);
args.push_back(arg);
continue;
}
@ -724,7 +725,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
output += std::to_wstring(arg_number);
++arg_number;
std::wstring arg;
translate_all(s, i, translations, arg);
translate_all(s, i, lang, translations, arg);
args.push_back(std::move(arg));
} else {
// This is an escape sequence *inside* the template string to translate itself.
@ -739,10 +740,10 @@ static void translate_string(std::wstring_view s, Translations *translations,
if (translations != nullptr) {
if (use_plural)
toutput = translations->getPluralTranslation(
textdomain, output, number);
lang, textdomain, output, number);
else
toutput = translations->getTranslation(
textdomain, output);
lang, textdomain, output);
} else {
toutput = output;
}
@ -780,7 +781,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
}
static void translate_all(std::wstring_view s, size_t &i,
Translations *translations, std::wstring &res)
const std::vector<std::wstring> &lang, Translations *translations, std::wstring &res)
{
res.clear();
res.reserve(s.length());
@ -858,7 +859,7 @@ static void translate_all(std::wstring_view s, size_t &i,
}
}
std::wstring translated;
translate_string(s, translations, textdomain, i, translated, use_plural, number);
translate_string(s, lang, translations, textdomain, i, translated, use_plural, number);
res.append(translated);
} else {
// Another escape sequence, such as colors. Preserve it.
@ -868,17 +869,17 @@ static void translate_all(std::wstring_view s, size_t &i,
}
// Translate string server side
std::wstring translate_string(std::wstring_view s, Translations *translations)
std::wstring translate_string(std::wstring_view s, const std::vector<std::wstring> &lang, Translations *translations)
{
size_t i = 0;
std::wstring res;
translate_all(s, i, translations, res);
translate_all(s, i, lang, translations, res);
return res;
}
std::wstring translate_string(std::wstring_view s)
{
return translate_string(s, g_client_translations);
return translate_string(s, get_effective_locale(), g_client_translations);
}
static const std::array<std::wstring_view, 30> disallowed_dir_names = {

16
src/util/string.h
View file

@ -665,7 +665,7 @@ std::vector<std::basic_string<T> > split(const std::basic_string<T> &s, T delim)
}
[[nodiscard]]
std::wstring translate_string(std::wstring_view s, Translations *translations);
std::wstring translate_string(std::wstring_view s, const std::vector<std::wstring> &lang, Translations *translations);
[[nodiscard]]
std::wstring translate_string(std::wstring_view s);
@ -756,11 +756,12 @@ inline const std::string duration_to_string(int sec)
*
* @return A std::string
*/
template<typename T>
[[nodiscard]]
inline std::string str_join(const std::vector<std::string> &list,
std::string_view delimiter)
inline std::basic_string<T> str_join(const std::vector<std::basic_string<T>> &list,
std::basic_string_view<T> delimiter)
{
std::ostringstream oss;
std::basic_ostringstream<T> oss;
bool first = true;
for (const auto &part : list) {
if (!first)
@ -771,6 +772,13 @@ inline std::string str_join(const std::vector<std::string> &list,
return oss.str();
}
template<typename T>
inline std::basic_string<T> str_join(const std::vector<std::basic_string<T>> &list,
const T *delimiter)
{
return str_join(list, std::basic_string_view<T>(delimiter));
}
#if IS_CLIENT_BUILD
/**
* Create a UTF8 std::string from an core::stringw.