Merge 6fd76674ac into 421835a30e

games/devtest/mods/testtranslations/init.lua

@@ -5,12 +5,18 @@ local function send_compare(name, text)
 			core.get_translated_string("", text), text, core.get_translated_string("fr", text)))
 end
 
+local function send_multilang_compare(name, text)
+	core.chat_send_player(name, ("%s | %s | %s | %s"):format(
+			core.get_translated_string("", text), text,
+			core.get_translated_string("fr_CH:fr", text), core.get_translated_string("fr:fr_CH", text)))
+end
+
 core.register_chatcommand("testtranslations", {
 	params = "",
 	description = "Test translations",
 	privs = {},
 	func = function(name, param)
-		core.chat_send_player(name, "Please ensure your locale is set to \"fr\"")
+		core.chat_send_player(name, "Please ensure your locale is set to \"fr_CH\"")
 		core.chat_send_player(name, "Untranslated | Client-side translation | Server-side translation (fr)")
 		send_compare(name, S("Testing .tr files: untranslated"))
 		send_compare(name, S("Testing .po files: untranslated"))
@@ -22,5 +28,9 @@ core.register_chatcommand("testtranslations", {
 			send_compare(name, NS("@1: .po singular", "@1: .po plural", i, tostring(i)))
 			send_compare(name, NS("@1: .mo singular", "@1: .mo plural", i, tostring(i)))
 		end
+		core.chat_send_player(name, "Untranslated | Client-side translations" ..
+				" | Server-side translation (fr_CH:fr) | Server-side translation (fr:fr_CH)")
+		send_multilang_compare(name, S("Testing translation with multiple languages"))
+		send_multilang_compare(name, S("Testing French-only translation: untranslated"))
 	end
 })

games/devtest/mods/testtranslations/locale/translation_po.fr.po

@@ -20,3 +20,11 @@ msgstr "Testing fuzzy .po entry: translated (wrong)"
 
 msgid "Testing .po without context: untranslated"
 msgstr "Testing .po without context: translated"
+
+msgctxt "testtranslations"
+msgid "Testing translation with multiple languages"
+msgstr "Testing translation: translated in standard French"
+
+msgctxt "testtranslations"
+msgid "Testing French-only translation: untranslated"
+msgstr "Testing French-only translation: translated"

games/devtest/mods/testtranslations/locale/translation_po.fr_CH.po

@@ -0,0 +1,3 @@
+msgctxt "testtranslations"
+msgid "Testing translation with multiple languages"
+msgstr "Testing translation: translated in Swiss French"

games/devtest/mods/testtranslations/test_locale/translation_po.de.po

@@ -40,3 +40,9 @@ msgstr[1] ""
 msgctxt "context"
 msgid "With context"
 msgstr "Has context"
+
+msgid "In multiple languages"
+msgstr "In standard German"
+
+msgid "In one language"
+msgstr "Only in standard German"

games/devtest/mods/testtranslations/test_locale/translation_po.de_CH.po

@@ -0,0 +1,2 @@
+msgid "In multiple languages"
+msgstr "In Swiss German"

src/gettext.cpp

@@ -6,6 +6,7 @@
 #include <cstring>
 #include <iostream>
 #include "gettext.h"
+#include "util/langcode.h"
 #include "util/string.h"
 #include "porting.h"
 #include "log.h"
@@ -146,6 +147,10 @@ static void MSVC_LocaleWorkaround(int argc, char* argv[])
 
 #endif
 
+static std::string configured_locale;
+static std::vector<std::wstring> effective_locale;
+static std::string effective_locale_string;
+
 /******************************************************************************/
 void init_gettext(const char *path, const std::string &configured_language,
 	int argc, char *argv[])
@@ -223,6 +228,17 @@ void init_gettext(const char *path, const std::string &configured_language,
 	setlocale(LC_ALL, "");
 #endif // if USE_GETTEXT
 
+	// Set up locale for in-game translations
+	configured_locale = configured_language;
+	if (configured_locale.empty()) {
+		if (auto lang = getenv("LANGUAGE"); lang && *lang)
+			configured_locale = lang;
+		else
+			configured_locale = getenv("LANG");
+	}
+	effective_locale = parse_language_list(utf8_to_wide(configured_locale));
+	effective_locale_string = wide_to_utf8(str_join(effective_locale, L":"));
+
 	/* no matter what locale is used we need number format to be "C" */
 	/* to ensure formspec parameters are evaluated correctly!        */
 
@@ -230,3 +246,13 @@ void init_gettext(const char *path, const std::string &configured_language,
 	infostream << "Message locale is now set to: "
 			<< setlocale(LC_ALL, 0) << std::endl;
 }
+
+const std::vector<std::wstring> &get_effective_locale()
+{
+	return effective_locale;
+}
+
+const std::string &get_client_language_code()
+{
+	return effective_locale_string;
+}

src/gettext.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <vector>
 #include "config.h" // for USE_GETTEXT
 #include "porting.h"
 #include "util/string.h"
@@ -103,3 +104,15 @@ inline std::string fmtgettext(const char *format, Args&&... args)
 
 	return buf;
 }
+
+/**
+ * Returns the effective locale setting for in-game translations.
+ * @return A vector of language codes based on priority.
+ */
+const std::vector<std::wstring> &get_effective_locale();
+
+/**
+ * Returns the effective locale setting for in-game translations.
+ * @return A string of the expanded language code.
+ */
+const std::string &get_client_language_code();

src/gui/guiEngine.cpp

@@ -229,19 +229,21 @@ std::string findLocaleFileInMods(const std::string &path, const std::string &fil
 
 /******************************************************************************/
 Translations *GUIEngine::getContentTranslations(const std::string &path,
-		const std::string &domain, const std::string &lang_code)
+		const std::string &domain, const std::vector<std::wstring> &lang)
 {
-	if (domain.empty() || lang_code.empty())
+	if (domain.empty() || lang.empty())
 		return nullptr;
 
-	std::string filename_no_ext = domain + "." + lang_code;
+	std::string key = path + DIR_DELIM "locale" DIR_DELIM + domain;
-	std::string key = path + DIR_DELIM "locale" DIR_DELIM + filename_no_ext;
 
 	if (key == m_last_translations_key)
 		return &m_last_translations;
 
-	std::string trans_path = key;
+	Translations translations;
-
+	for (const auto &lang_code: lang) {
+		auto utf8_lang_code = wide_to_utf8(lang_code);
+		std::string filename_no_ext = domain + "." + utf8_lang_code;
+		auto trans_path = key + "." + utf8_lang_code;
 		switch (getContentType(path)) {
 		case ContentType::GAME:
 			trans_path = findLocaleFileInMods(path + DIR_DELIM "mods" DIR_DELIM,
@@ -254,17 +256,19 @@ Translations *GUIEngine::getContentTranslations(const std::string &path,
 			trans_path = findLocaleFileWithExtension(trans_path);
 			break;
 		}
-
 		if (trans_path.empty())
+			continue;
+		std::string data;
+		if (fs::ReadFile(trans_path, data)) {
+			translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data);
+		}
+	}
+
+	if (translations.empty())
 		return nullptr;
 
 	m_last_translations_key = key;
-	m_last_translations = {};
+	m_last_translations = std::move(translations);
-
-	std::string data;
-	if (fs::ReadFile(trans_path, data)) {
-		m_last_translations.loadTranslation(fs::GetFilenameFromPath(trans_path.c_str()), data);
-	}
 
 	return &m_last_translations;
 }

src/gui/guiEngine.h

@@ -156,7 +156,7 @@ public:
 	 * change with the next call to `getContentTranslations`.
 	 * */
 	Translations *getContentTranslations(const std::string &path,
-			const std::string &domain, const std::string &lang_code);
+			const std::string &domain, const std::vector<std::wstring> &lang_code);
 
 private:
 	std::string m_last_translations_key;

src/network/clientpackethandler.cpp

@@ -142,13 +142,7 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt)
 					<< m_recommended_send_interval<<std::endl;
 
 	// Reply to server
-	/*~ DO NOT TRANSLATE THIS LITERALLY!
+	std::string lang = get_client_language_code();
-	This is a special string which needs to contain the translation's
-	language code (e.g. "de" for German). */
-	std::string lang = gettext("LANG_CODE");
-	if (lang == "LANG_CODE")
-		lang.clear();
-
 	NetworkPacket resp_pkt(TOSERVER_INIT2, sizeof(u16) + lang.size());
 	resp_pkt << lang;
 	Send(&resp_pkt);

src/script/lua_api/l_client.cpp

@@ -177,10 +177,7 @@ int ModApiClient::l_get_language(lua_State *L)
 #else
 	char *locale = setlocale(LC_MESSAGES, NULL);
 #endif
-	std::string lang = gettext("LANG_CODE");
+	std::string lang = get_client_language_code();
-	if (lang == "LANG_CODE")
-		lang.clear();
-
 	lua_pushstring(L, locale);
 	lua_pushstring(L, lang.c_str());
 	return 2;

src/script/lua_api/l_env.cpp

@@ -1376,9 +1376,10 @@ int ModApiEnv::l_get_translated_string(lua_State * L)
 
 	std::string lang_code = luaL_checkstring(L, 1);
 	std::string string = luaL_checkstring(L, 2);
+	auto lang = str_split(utf8_to_wide(lang_code), L':');
 
 	auto *translations = getServer(L)->getTranslationLanguage(lang_code);
-	string = wide_to_utf8(translate_string(utf8_to_wide(string), translations));
+	string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations));
 	lua_pushstring(L, string.c_str());
 	return 1;
 }

src/script/lua_api/l_mainmenu.cpp

@@ -527,12 +527,10 @@ int ModApiMainMenu::l_get_content_translation(lua_State *L)
 	std::string path = luaL_checkstring(L, 1);
 	std::string domain = luaL_checkstring(L, 2);
 	std::string string = luaL_checkstring(L, 3);
-	std::string lang = gettext("LANG_CODE");
+	auto lang = get_effective_locale();
-	if (lang == "LANG_CODE")
-		lang = "";
 
 	auto *translations = engine->getContentTranslations(path, domain, lang);
-	string = wide_to_utf8(translate_string(utf8_to_wide(string), translations));
+	string = wide_to_utf8(translate_string(utf8_to_wide(string), lang, translations));
 	lua_pushstring(L, string.c_str());
 	return 1;
 }
@@ -879,10 +877,7 @@ int ModApiMainMenu::l_download_file(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_language(lua_State *L)
 {
-	std::string lang = gettext("LANG_CODE");
+	std::string lang = get_client_language_code();
-	if (lang == "LANG_CODE")
-		lang = "";
-
 	lua_pushstring(L, lang.c_str());
 	return 1;
 }

src/server.cpp

@@ -2685,22 +2685,18 @@ void Server::fillMediaCache()
 	infostream << "Server: " << m_media.size() << " media files collected" << std::endl;
 }
 
-void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code)
+void Server::sendMediaAnnouncement(session_t peer_id, const std::string &langstring)
 {
-	std::string translation_formats[3] = { ".tr", ".po", ".mo" };
+	std::unordered_set<std::string> langs;
-	std::string lang_suffixes[3];
+	for (const auto &lang_code: str_split(langstring, ':'))
-	for (size_t i = 0; i < 3; i++) {
+		langs.insert(lang_code);
-		lang_suffixes[i].append(".").append(lang_code).append(translation_formats[i]);
-	}
 
 	auto include = [&] (const std::string &name, const MediaInfo &info) -> bool {
 		if (info.no_announce)
 			return false;
-		for (size_t j = 0; j < 3; j++) {
+		if (auto filelang = Translations::getFileLanguage(name);
-			if (str_ends_with(name, translation_formats[j]) && !str_ends_with(name, lang_suffixes[j])) {
+				!filelang.empty() && langs.find(std::string(filelang)) == langs.end())
 			return false;
-			}
-		}
 		return true;
 	};
 
@@ -4288,28 +4284,33 @@ void Server::broadcastModChannelMessage(const std::string &channel,
 	}
 }
 
-Translations *Server::getTranslationLanguage(const std::string &lang_code)
+Translations *Server::getTranslationLanguage(const std::string &lang)
 {
-	if (lang_code.empty())
+	if (lang.empty())
 		return nullptr;
 
-	auto it = server_translations.find(lang_code);
+	std::unordered_set<std::string> load_langs;
-	if (it != server_translations.end())
-		return &it->second; // Already loaded
 
-	// [] will create an entry
+	for (const auto &lang_code: str_split(lang, ':')) {
-	auto *translations = &server_translations[lang_code];
+		if (loaded_translations.find(lang_code) == loaded_translations.end()) {
+			load_langs.insert(lang_code);
+			loaded_translations.insert(lang_code);
+		}
+	}
+
+	if (load_langs.empty())
+		return &server_translations;
 
 	for (const auto &i : m_media) {
-		if (Translations::getFileLanguage(i.first) == lang_code) {
+		if (load_langs.find(std::string(Translations::getFileLanguage(i.first))) != load_langs.end()) {
 			std::string data;
 			if (fs::ReadFile(i.second.path, data, true)) {
-				translations->loadTranslation(i.first, data);
+				server_translations.loadTranslation(i.first, data);
 			}
 		}
 	}
 
-	return translations;
+	return &server_translations;
 }
 
 std::unordered_map<std::string, std::string> Server::getMediaList()

src/server.h

@@ -713,7 +713,8 @@ private:
 	// Craft definition manager
 	IWritableCraftDefManager *m_craftdef;
 
-	std::unordered_map<std::string, Translations> server_translations;
+	Translations server_translations;
+	std::unordered_set<std::string> loaded_translations;
 
 	ModIPCStore m_ipcstore;
 

src/translation.cpp

@@ -36,21 +36,30 @@ void Translations::clear()
 	m_plural_translations.clear();
 }
 
-const std::wstring &Translations::getTranslation(
+const std::wstring &Translations::getTranslation(const std::vector<std::wstring> &langlist,
 		const std::wstring &textdomain, const std::wstring &s) const
 {
-	std::wstring key = textdomain + L"|" + s;
+	auto basekey = L"|" + textdomain + L"|" + s;
-	auto it = m_translations.find(key);
+	for (const auto &lang: langlist) {
+		auto it = m_translations.find(lang + basekey);
 		if (it != m_translations.end())
 			return it->second;
+	}
 	return s;
 }
 
-const std::wstring &Translations::getPluralTranslation(
+const std::wstring &Translations::getTranslation(
+		const std::wstring &textdomain, const std::wstring &s) const
+{
+	return getTranslation(get_effective_locale(), textdomain, s);
+}
+
+const std::wstring &Translations::getPluralTranslation(const std::vector<std::wstring> &langlist,
 		const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
 {
-	std::wstring key = textdomain + L"|" + s;
+	auto basekey = L"|" + textdomain + L"|" + s;
-	auto it = m_plural_translations.find(key);
+	for (const auto &lang: langlist) {
+		auto it = m_plural_translations.find(lang + basekey);
 		if (it != m_plural_translations.end()) {
 			auto n = (*(it->second.first))(number);
 			const std::vector<std::wstring> &v = it->second.second;
@@ -60,21 +69,27 @@ const std::wstring &Translations::getPluralTranslation(
 				return v[n];
 			}
 		}
+	}
 	return s;
 }
 
-
+const std::wstring &Translations::getPluralTranslation(
-void Translations::addTranslation(
+		const std::wstring &textdomain, const std::wstring &s, unsigned long int number) const
-		const std::wstring &textdomain, const std::wstring &original, const std::wstring &translated)
 {
-	std::wstring key = textdomain + L"|" + original;
+	return getPluralTranslation(get_effective_locale(), textdomain, s, number);
+}
+
+void Translations::addTranslation(const std::wstring &lang, const std::wstring &textdomain,
+		const std::wstring &original, const std::wstring &translated)
+{
+	std::wstring key = lang + L"|" + textdomain + L"|" + original;
 	if (!translated.empty()) {
 		m_translations.emplace(std::move(key), std::move(translated));
 	}
 }
 
-void Translations::addPluralTranslation(
+void Translations::addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
-		const std::wstring &textdomain, const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
+		const GettextPluralForm::Ptr &plural, const std::wstring &original, std::vector<std::wstring> &translated)
 {
 	static bool warned = false;
 	if (!plural) {
@@ -86,12 +101,12 @@ void Translations::addPluralTranslation(
 		errorstream << "Translations: incorrect number of plural translations (expected " << plural->size() << ", got " << translated.size() << ")" << std::endl;
 		return;
 	}
-	std::wstring key = textdomain + L"|" + original;
+	std::wstring key = lang + L"|" + textdomain + L"|" + original;
 	m_plural_translations.emplace(std::move(key), std::pair(plural, translated));
 }
 
 
-void Translations::loadTrTranslation(const std::string &data)
+void Translations::loadTrTranslation(const std::wstring &lang, const std::string &data)
 {
 	std::istringstream is(data);
 	std::string textdomain_narrow;
@@ -191,7 +206,7 @@ void Translations::loadTrTranslation(const std::string &data)
 			}
 		}
 
-		addTranslation(textdomain, word1.str(), word2.str());
+		addTranslation(lang, textdomain, word1.str(), word2.str());
 	}
 }
 
@@ -318,7 +333,8 @@ std::wstring Translations::unescapeC(const std::wstring &str)
 	return result;
 }
 
-void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
+void Translations::loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
+		const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry)
 {
 	// Process an entry from a PO file and add it to the translation table
 	// Assumes that entry[L"msgid"] is always defined
@@ -338,7 +354,7 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
 			errorstream << "Could not load translation: entry for msgid \"" << wide_to_utf8(original) << "\" does not contain a msgstr field" << std::endl;
 			return;
 		}
-		addTranslation(textdomain, original, translated->second);
+		addTranslation(lang, textdomain, original, translated->second);
 	} else {
 		std::vector<std::wstring> translations;
 		for (int i = 0; ; i++) {
@@ -347,8 +363,8 @@ void Translations::loadPoEntry(const std::wstring &basefilename, const GettextPl
 				break;
 			translations.push_back(translated->second);
 		}
-		addPluralTranslation(textdomain, plural_form, original, translations);
+		addPluralTranslation(lang, textdomain, plural_form, original, translations);
-		addPluralTranslation(textdomain, plural_form, plural->second, translations);
+		addPluralTranslation(lang, textdomain, plural_form, plural->second, translations);
 	}
 }
 
@@ -415,7 +431,7 @@ std::optional<std::pair<std::wstring, std::wstring>> Translations::parsePoLine(c
 	return std::pair(prefix, s);
 }
 
-void Translations::loadPoTranslation(const std::string &basefilename, const std::string &data)
+void Translations::loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
 {
 	std::istringstream is(data);
 	std::string line;
@@ -481,7 +497,7 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
 							}
 						}
 					} else {
-						loadPoEntry(wbasefilename, plural, last_entry);
+						loadPoEntry(lang, wbasefilename, plural, last_entry);
 					}
 				}
 				last_entry.clear();
@@ -505,13 +521,14 @@ void Translations::loadPoTranslation(const std::string &basefilename, const std:
 
 	if (last_entry.find(L"msgid") != last_entry.end()) {
 		if (!skip_last && !last_entry[L"msgid"].empty())
-			loadPoEntry(wbasefilename, plural, last_entry);
+			loadPoEntry(lang, wbasefilename, plural, last_entry);
 	} else if (!last_entry.empty()) {
 		errorstream << "Unable to parse po file: Last entry has no \"msgid\" field" << std::endl;
 	}
 }
 
-void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
+void Translations::loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
+		const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated)
 {
 	std::wstring textdomain = L"";
 	size_t found;
@@ -527,10 +544,10 @@ void Translations::loadMoEntry(const std::wstring &basefilename, const GettextPl
 	found = noriginal.find('\0');
 	if (found != std::string::npos) {
 		std::vector<std::wstring> translations = str_split(utf8_to_wide(translated), L'\0');
-		addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
+		addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(0, found)), translations);
-		addPluralTranslation(textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
+		addPluralTranslation(lang, textdomain, plural_form, utf8_to_wide(noriginal.substr(found + 1)), translations);
 	} else {
-		addTranslation(textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
+		addTranslation(lang, textdomain, utf8_to_wide(noriginal), utf8_to_wide(translated));
 	}
 }
 
@@ -549,7 +566,7 @@ inline u32 readVarEndian(bool is_be, std::string_view data, size_t pos = 0)
 	}
 }
 
-void Translations::loadMoTranslation(const std::string &basefilename, const std::string &data)
+void Translations::loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data)
 {
 	size_t length = data.length();
 	std::wstring wbasefilename = utf8_to_wide(basefilename);
@@ -619,7 +636,7 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
 				}
 			}
 		} else {
-			loadMoEntry(wbasefilename, plural_form, original, translated);
+			loadMoEntry(lang, wbasefilename, plural_form, original, translated);
 		}
 	}
 
@@ -628,17 +645,18 @@ void Translations::loadMoTranslation(const std::string &basefilename, const std:
 
 void Translations::loadTranslation(const std::string &filename, const std::string &data)
 {
+	auto lang = utf8_to_wide(getFileLanguage(filename));
 	const char *trExtension[] = { ".tr", NULL };
 	const char *poExtension[] = { ".po", NULL };
 	const char *moExtension[] = { ".mo", NULL };
 	if (!removeStringEnd(filename, trExtension).empty()) {
-		loadTrTranslation(data);
+		loadTrTranslation(lang, data);
 	} else if (!removeStringEnd(filename, poExtension).empty()) {
 		std::string basefilename = str_split(filename, '.')[0];
-		loadPoTranslation(basefilename, data);
+		loadPoTranslation(lang, basefilename, data);
 	} else if (!removeStringEnd(filename, moExtension).empty()) {
 		std::string basefilename = str_split(filename, '.')[0];
-		loadMoTranslation(basefilename, data);
+		loadMoTranslation(lang, basefilename, data);
 	} else {
 		errorstream << "loadTranslation called with invalid filename: \"" << filename << "\"" << std::endl;
 	}

src/translation.h

@@ -20,37 +20,50 @@ class Translations
 public:
 	void loadTranslation(const std::string &filename, const std::string &data);
 	void clear();
-	const std::wstring &getTranslation(
+	const std::wstring &getTranslation(const std::vector<std::wstring> &lang,
 			const std::wstring &textdomain, const std::wstring &s) const;
-	const std::wstring &getPluralTranslation(const std::wstring &textdomain,
+	const std::wstring &getPluralTranslation(const std::vector<std::wstring> &lang,
+			const std::wstring &textdomain,
 			const std::wstring &s, unsigned long int number) const;
 	static const std::string_view getFileLanguage(const std::string &filename);
 	static inline bool isTranslationFile(const std::string &filename)
 	{
 		return getFileLanguage(filename) != "";
 	}
-	// for testing
 	inline size_t size()
 	{
 		return m_translations.size() + m_plural_translations.size()/2;
 	}
+	inline bool empty()
+	{
+		return size() == 0;
+	}
+
+#ifndef SERVER
+	const std::wstring &getTranslation(
+			const std::wstring &textdomain, const std::wstring &s) const;
+	const std::wstring &getPluralTranslation(const std::wstring &textdomain,
+			const std::wstring &s, unsigned long int number) const;
+#endif
 
 private:
 	std::unordered_map<std::wstring, std::wstring> m_translations;
 	std::unordered_map<std::wstring, std::pair<GettextPluralForm::Ptr, std::vector<std::wstring>>> m_plural_translations;
 
-	void addTranslation(const std::wstring &textdomain, const std::wstring &original,
+	void addTranslation(const std::wstring &lang, const std::wstring &textdomain, const std::wstring &original,
 			const std::wstring &translated);
-	void addPluralTranslation(const std::wstring &textdomain,
+	void addPluralTranslation(const std::wstring &lang, const std::wstring &textdomain,
 			const GettextPluralForm::Ptr &plural,
 			const std::wstring &original,
 			std::vector<std::wstring> &translated);
 	std::wstring unescapeC(const std::wstring &str);
 	std::optional<std::pair<std::wstring, std::wstring>> parsePoLine(const std::string &line);
 	bool inEscape(const std::wstring &str, size_t pos);
-	void loadPoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
+	void loadPoEntry(const std::wstring &lang, const std::wstring &basefilename,
-	void loadMoEntry(const std::wstring &basefilename, const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
+			const GettextPluralForm::Ptr &plural_form, const std::map<std::wstring, std::wstring> &entry);
-	void loadTrTranslation(const std::string &data);
+	void loadMoEntry(const std::wstring &lang, const std::wstring &basefilename,
-	void loadPoTranslation(const std::string &basefilename, const std::string &data);
+			const GettextPluralForm::Ptr &plural_form, const std::string &original, const std::string &translated);
-	void loadMoTranslation(const std::string &basefilename, const std::string &data);
+	void loadTrTranslation(const std::wstring &lang, const std::string &data);
+	void loadPoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
+	void loadMoTranslation(const std::wstring &lang, const std::string &basefilename, const std::string &data);
 };

src/unittest/CMakeLists.txt

@@ -19,6 +19,7 @@ set (UNITTEST_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/test_irrptr.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_irr_matrix4.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_irr_rotation.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/test_langcode.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_logging.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_lbmmanager.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_lua.cpp

src/unittest/test_langcode.cpp

@@ -0,0 +1,16 @@
+// Luanti
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "util/langcode.h"
+#include "catch.h"
+
+TEST_CASE("test langcode")
+{
+	SECTION("test language list")
+	{
+		CHECK(expand_language_list(L"de_DE@euro.UTF-8:fr") == L"de_DE@euro:de_DE:de:fr");
+		CHECK(expand_language_list(L"zh_HK:yue_HK:zh_TW") == L"zh_HK:yue_HK:yue:zh_TW:zh");
+		CHECK(expand_language_list(L"de_DE:fr:de_CH:en:de:de_AT") == L"de_DE:fr:de_CH:en:de:de_AT");
+		CHECK(expand_language_list(L".UTF-8:de:.ISO-8859-1:fr:.GB2312") == L"de:fr");
+	}
+}

src/unittest/test_translations.cpp

@@ -9,8 +9,11 @@
 #define CONTEXT L"context"
 #define TEXTDOMAIN_PO L"translation_po"
 #define TEST_PO_NAME "translation_po.de.po"
+#define SECONDARY_PO_NAME "translation_po.de_CH.po"
 #define TEST_MO_NAME "translation_mo.de.mo"
 
+const std::vector<std::wstring> lang {L"de"};
+
 static std::string read_translation_file(const std::string &filename)
 {
 		auto gamespec = findSubgame("devtest");
@@ -86,17 +89,17 @@ TEST_CASE("test translations")
 		Translations translations;
 		translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
 
-		CHECK(translations.size() == 5);
+		CHECK(translations.size() == 7);
-		CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"foo") == L"bar");
+		CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"foo") == L"bar");
-		CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
+		CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Untranslated") == L"Untranslated");
-		CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
+		CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Fuzzy") == L"Fuzzy");
-		CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
+		CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Multi\\line\nstring") == L"Multi\\\"li\\ne\nresult");
-		CHECK(translations.getTranslation(TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
+		CHECK(translations.getTranslation(lang, TEXTDOMAIN_PO, L"Wrong order") == L"Wrong order");
-		CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
+		CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Plural form", 1) == L"Singular result");
-		CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
+		CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Singular form", 0) == L"Plural result");
-		CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
+		CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translation", 1) == L"Partially translated");
-		CHECK(translations.getPluralTranslation(TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
+		CHECK(translations.getPluralTranslation(lang, TEXTDOMAIN_PO, L"Partial translations", 2) == L"Partial translations");
-		CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
+		CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
 	}
 
 	SECTION("MO file parser")
@@ -105,8 +108,19 @@ TEST_CASE("test translations")
 		translations.loadTranslation(TEST_MO_NAME, read_translation_file(TEST_MO_NAME));
 
 		CHECK(translations.size() == 2);
-		CHECK(translations.getTranslation(CONTEXT, L"With context") == L"Has context");
+		CHECK(translations.getTranslation(lang, CONTEXT, L"With context") == L"Has context");
-		CHECK(translations.getPluralTranslation(CONTEXT, L"Plural form", 1) == L"Singular result");
+		CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Plural form", 1) == L"Singular result");
-		CHECK(translations.getPluralTranslation(CONTEXT, L"Singular form", 0) == L"Plural result");
+		CHECK(translations.getPluralTranslation(lang, CONTEXT, L"Singular form", 0) == L"Plural result");
+	}
+
+	SECTION("Translation fallback")
+	{
+		Translations translations;
+		translations.loadTranslation(TEST_PO_NAME, read_translation_file(TEST_PO_NAME));
+		translations.loadTranslation(SECONDARY_PO_NAME, read_translation_file(SECONDARY_PO_NAME));
+
+		CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In Swiss German");
+		CHECK(translations.getTranslation({L"de", L"de_CH"}, TEXTDOMAIN_PO, L"In multiple languages") == L"In standard German");
+		CHECK(translations.getTranslation({L"de_CH", L"de"}, TEXTDOMAIN_PO, L"In one language") == L"Only in standard German");
 	}
 }

src/util/CMakeLists.txt

@@ -11,6 +11,7 @@ set(util_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/guid.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/hashing.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/langcode.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp

src/util/langcode.cpp

@@ -0,0 +1,47 @@
+// Luanti
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include <unordered_map>
+#include "util/string.h"
+
+std::vector<std::wstring> parse_language_list(const std::wstring &lang)
+{
+	std::unordered_map<std::wstring, std::wstring> added_by;
+	std::vector<std::vector<std::wstring>> expanded;
+
+	for (const auto &name: str_split(lang, L':')) {
+		auto pos = name.find(L'.'); // strip encoding information
+		const auto realname = pos == name.npos ? name : name.substr(0, pos);
+		if (realname.empty())
+			continue;
+
+		std::vector<std::wstring> basenames = {};
+		auto base = realname;
+		do {
+			if (added_by[base] == base)
+				break;
+			added_by[base] = realname;
+			basenames.push_back(base);
+
+			pos = base.find_last_of(L"_@");
+			base = base.substr(0, pos);
+		} while (pos != base.npos);
+		if (!basenames.empty())
+			expanded.push_back(std::move(basenames));
+	}
+
+	std::vector<std::wstring> langlist;
+	for (auto &basenames: expanded)
+	{
+		auto first = basenames.front();
+		for (auto &&name: basenames)
+			if (added_by[name] == first)
+				langlist.push_back(std::move(name));
+	}
+	return langlist;
+}
+
+std::wstring expand_language_list(const std::wstring &lang)
+{
+	return str_join(parse_language_list(lang), L":");
+}

src/util/langcode.h

@@ -0,0 +1,9 @@
+// Luanti
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#pragma once
+#include <vector>
+#include <string>
+
+std::vector<std::wstring> parse_language_list(const std::wstring &lang);
+std::wstring expand_language_list(const std::wstring &lang);

src/util/string.cpp

@@ -7,6 +7,7 @@
 #include "numeric.h"
 #include "log.h"
 
+#include "gettext.h"
 #include "hex.h"
 #include "porting.h"
 #include "translation.h"
@@ -654,9 +655,9 @@ std::string wrap_rows(std::string_view from, unsigned row_len, bool has_color_co
  */
 
 static void translate_all(std::wstring_view s, size_t &i,
-		Translations *translations, std::wstring &res);
+		const std::vector<std::wstring> &lang, Translations *translations, std::wstring &res);
 
-static void translate_string(std::wstring_view s, Translations *translations,
+static void translate_string(std::wstring_view s, const std::vector<std::wstring> &lang, Translations *translations,
 		const std::wstring &textdomain, size_t &i, std::wstring &res,
 		bool use_plural, unsigned long int number)
 {
@@ -716,7 +717,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
 			if (arg_number >= 10) {
 				errorstream << "Ignoring too many arguments to translation" << std::endl;
 				std::wstring arg;
-				translate_all(s, i, translations, arg);
+				translate_all(s, i, lang, translations, arg);
 				args.push_back(arg);
 				continue;
 			}
@@ -724,7 +725,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
 			output += std::to_wstring(arg_number);
 			++arg_number;
 			std::wstring arg;
-			translate_all(s, i, translations, arg);
+			translate_all(s, i, lang, translations, arg);
 			args.push_back(std::move(arg));
 		} else {
 			// This is an escape sequence *inside* the template string to translate itself.
@@ -739,10 +740,10 @@ static void translate_string(std::wstring_view s, Translations *translations,
 	if (translations != nullptr) {
 		if (use_plural)
 			toutput = translations->getPluralTranslation(
-					textdomain, output, number);
+					lang, textdomain, output, number);
 		else
 			toutput = translations->getTranslation(
-					textdomain, output);
+					lang, textdomain, output);
 	} else {
 		toutput = output;
 	}
@@ -780,7 +781,7 @@ static void translate_string(std::wstring_view s, Translations *translations,
 }
 
 static void translate_all(std::wstring_view s, size_t &i,
-		Translations *translations, std::wstring &res)
+		const std::vector<std::wstring> &lang, Translations *translations, std::wstring &res)
 {
 	res.clear();
 	res.reserve(s.length());
@@ -858,7 +859,7 @@ static void translate_all(std::wstring_view s, size_t &i,
 				}
 			}
 			std::wstring translated;
-			translate_string(s, translations, textdomain, i, translated, use_plural, number);
+			translate_string(s, lang, translations, textdomain, i, translated, use_plural, number);
 			res.append(translated);
 		} else {
 			// Another escape sequence, such as colors. Preserve it.
@@ -868,17 +869,17 @@ static void translate_all(std::wstring_view s, size_t &i,
 }
 
 // Translate string server side
-std::wstring translate_string(std::wstring_view s, Translations *translations)
+std::wstring translate_string(std::wstring_view s, const std::vector<std::wstring> &lang, Translations *translations)
 {
 	size_t i = 0;
 	std::wstring res;
-	translate_all(s, i, translations, res);
+	translate_all(s, i, lang, translations, res);
 	return res;
 }
 
 std::wstring translate_string(std::wstring_view s)
 {
-	return translate_string(s, g_client_translations);
+	return translate_string(s, get_effective_locale(), g_client_translations);
 }
 
 static const std::array<std::wstring_view, 30> disallowed_dir_names = {

src/util/string.h

@@ -665,7 +665,7 @@ std::vector<std::basic_string<T> > split(const std::basic_string<T> &s, T delim)
 }
 
 [[nodiscard]]
-std::wstring translate_string(std::wstring_view s, Translations *translations);
+std::wstring translate_string(std::wstring_view s, const std::vector<std::wstring> &lang,  Translations *translations);
 
 [[nodiscard]]
 std::wstring translate_string(std::wstring_view s);
@@ -756,11 +756,12 @@ inline const std::string duration_to_string(int sec)
  *
  * @return A std::string
  */
+template<typename T>
 [[nodiscard]]
-inline std::string str_join(const std::vector<std::string> &list,
+inline std::basic_string<T> str_join(const std::vector<std::basic_string<T>> &list,
-		std::string_view delimiter)
+		std::basic_string_view<T> delimiter)
 {
-	std::ostringstream oss;
+	std::basic_ostringstream<T> oss;
 	bool first = true;
 	for (const auto &part : list) {
 		if (!first)
@@ -771,6 +772,13 @@ inline std::string str_join(const std::vector<std::string> &list,
 	return oss.str();
 }
 
+template<typename T>
+inline std::basic_string<T> str_join(const std::vector<std::basic_string<T>> &list,
+		const T *delimiter)
+{
+	return str_join(list, std::basic_string_view<T>(delimiter));
+}
+
 #if IS_CLIENT_BUILD
 /**
  * Create a UTF8 std::string from an core::stringw.