Better translation exporting for mods

Better version of translation exporting logic. Compared to existiing
version it:
- places generated json's in same directory structure as recommended for
mods (`modname/Content/configtranslation/language.json`). Files are
placed in same directory before (`exported`) to reduce chance of
information loss on overwrite
- (mostly) correctly handled mods that overwrite strings from another
submod of the same mod. For now only simple cases are handled (within
same mod, and without long overwrite chains), which seems to be
sufficient for existing mods

New translation is done by server (vcmiserver / VCMI_Server.exe) and not
by client command - this is due to reloading of library in runtime which
at the moment can't be done on client, especially during ongoing game

client/ClientCommandManager.cpp

@@ -190,7 +190,7 @@ void ClientCommandManager::handleRedrawCommand()
 
 void ClientCommandManager::handleTranslateGameCommand(bool onlyMissing)
 {
-	std::map<std::string, std::map<std::string, std::string>> textsByMod;
+	std::map<std::string, ExportedStrings> textsByMod;
 	LIBRARY->generaltexth->exportAllTexts(textsByMod, onlyMissing);
 
 	const boost::filesystem::path outPath = VCMIDirs::get().userExtractedPath() / ( onlyMissing ? "translationMissing" : "translation");
@@ -200,7 +200,7 @@ void ClientCommandManager::handleTranslateGameCommand(bool onlyMissing)
 	{
 		JsonNode output;
 
-		for(const auto & stringEntry : modEntry.second)
+		for(const auto & stringEntry : modEntry.second.strings)
 		{
 			if(boost::algorithm::starts_with(stringEntry.first, "map."))
 				continue;
@@ -212,7 +212,9 @@ void ClientCommandManager::handleTranslateGameCommand(bool onlyMissing)
 
 		if (!output.isNull())
 		{
-			const boost::filesystem::path filePath = outPath / (modEntry.first + ".json");
+			std::string filename = modEntry.first;
+			boost::range::replace(filename, '.', '_');
+			const boost::filesystem::path filePath = outPath / (filename + ".json");
 			std::ofstream file(filePath.c_str());
 			file << output.toString();
 		}
@@ -274,7 +276,7 @@ void ClientCommandManager::handleTranslateMapsCommand()
 		}
 	}
 
-	std::map<std::string, std::map<std::string, std::string>> textsByMod;
+	std::map<std::string, ExportedStrings> textsByMod;
 	LIBRARY->generaltexth->exportAllTexts(textsByMod, false);
 
 	const boost::filesystem::path outPath = VCMIDirs::get().userExtractedPath() / "translation";
@@ -284,7 +286,7 @@ void ClientCommandManager::handleTranslateMapsCommand()
 	{
 		JsonNode output;
 
-		for(const auto & stringEntry : modEntry.second)
+		for(const auto & stringEntry : modEntry.second.strings)
 		{
 			if(boost::algorithm::starts_with(stringEntry.first, "map."))
 				output[stringEntry.first].String() = stringEntry.second;

docs/translators/Translations.md

@@ -107,6 +107,18 @@ If something is not clear - feel free to ask us on Discord or forum. Translation
 
 If you want to start new translation for a mod or to update existing one you may need to export it first. To do that:
 
+- Optionally, backup your mod preset - game may modify active submods of mod being translated
+- Set game language in Launcher to one that you want to target
+- launch VCMI server in translation export mode:
+  - (Windows) Create shortcut for VCMI_Server.exe and append `--translate-mod=XXX` to "Target" field in shortcut properties, where XXX is identifier of mod that you want to translate
+  - (command-line) Open command line and run `vcmiserver --translate-mod=XXX`, where XXX is identifier of mod that you want to translate
+
+After that, start Launcher, switch to Help tab and open "log files directory". You can find exported json's in `extracted/translation` directory.
+
+### Exporting translation (alternative)
+
+Alternatively, you can use vcmi client to do similar actions:
+
 - Enable mod(s) that you want to export and set game language in Launcher to one that you want to target
 - Launch VCMI and start any map to get in game
 - Press Tab to activate chat and enter '/translate'
@@ -117,6 +129,8 @@ If your mod also contains maps or campaigns that you want to translate, then use
 
 If you want to update existing translation, you can use `/translate missing` command that will export only strings that were not translated
 
+NOTE: when translating with this method, some strings may not export correctly, for example strings that were modified in multiple mods. To avoid this, you'll need to disable mods that overrride other strings and do a second re-run of this command
+
 ### Translating mod information
 
 In order to display information in Launcher in language selected by user add following block into your `mod.json`:

lib/GameLibrary.h

@@ -117,9 +117,10 @@ public:
 	/// Loads all game entities
 	void initializeLibrary();
 
-private:
 	// basic initialization. should be called before init(). Can also extract original H3 archives
 	void loadFilesystem(bool extractArchives);
+
+	// loads filesystems of all mods
 	void loadModFilesystem();
 
 #if SCRIPTING_ENABLED

lib/texts/TextLocalizationContainer.cpp

@@ -145,7 +145,7 @@ bool TextLocalizationContainer::identifierExists(const TextIdentifier & UID) con
 	return stringsLocalizations.count(UID.get());
 }
 
-void TextLocalizationContainer::exportAllTexts(std::map<std::string, std::map<std::string, std::string>> & storage, bool onlyMissing) const
+void TextLocalizationContainer::exportAllTexts(std::map<std::string, ExportedStrings> & storage, bool onlyMissing) const
 {
 	std::lock_guard globalLock(globalTextMutex);
 
@@ -157,18 +157,18 @@ void TextLocalizationContainer::exportAllTexts(std::map<std::string, std::map<st
 		if (onlyMissing && entry.second.overriden)
 			continue;
 
-		std::string textToWrite;
+		std::string textToWrite = entry.second.translatedText;
 		std::string modName = entry.second.baseStringModContext;
+		std::string originalMod = entry.second.identifierModContext;
 
-		if (entry.second.baseStringModContext == entry.second.identifierModContext && modName.find('.') != std::string::npos)
+		if (modName == originalMod && modName.find('.') != std::string::npos)
 			modName = modName.substr(0, modName.find('.'));
-
-		boost::range::replace(modName, '.', '_');
-
-		textToWrite = entry.second.translatedText;
+		else
+			if (!vstd::contains(storage[modName].overridenMods, originalMod))
+				storage[modName].overridenMods.push_back(originalMod);
 
 		if (!textToWrite.empty())
-			storage[modName][entry.first] = textToWrite;
+			storage[modName].strings[entry.first] = textToWrite;
 	}
 }
 

lib/texts/TextLocalizationContainer.h

@@ -15,6 +15,15 @@ VCMI_LIB_NAMESPACE_BEGIN
 
 class JsonNode;
 
+struct ExportedStrings
+{
+	/// Strings (string ID -> translation) that were added by this mod
+	std::map<std::string, std::string> strings;
+
+	/// mods that had one or more of their strings overriden by this mod
+	std::vector<std::string> overridenMods;
+};
+
 class DLL_LINKAGE TextLocalizationContainer
 {
 protected:
@@ -79,7 +88,7 @@ public:
 
 	/// Debug method, returns all currently stored texts
 	/// Format: [mod ID][string ID] -> human-readable text
-	void exportAllTexts(std::map<std::string, std::map<std::string, std::string>> & storage, bool onlyMissing) const;
+	void exportAllTexts(std::map<std::string, ExportedStrings> & storage, bool onlyMissing) const;
 
 	/// Add or override subcontainer which can store identifiers
 	void addSubContainer(const TextLocalizationContainer & container);

serverapp/EntryPoint.cpp

@@ -16,12 +16,109 @@
 #include "../lib/VCMIDirs.h"
 #include "../lib/GameLibrary.h"
 #include "../lib/CConfigHandler.h"
+#include "../lib/filesystem/Filesystem.h"
+#include "../lib/modding/CModHandler.h"
+#include "../lib/modding/ModManager.h"
+#include "modding/ModDescription.h"
+#include "texts/CGeneralTextHandler.h"
 
 #include <boost/program_options.hpp>
 
 static const std::string SERVER_NAME_AFFIX = "server";
 static const std::string SERVER_NAME = GameConstants::VCMI_VERSION + std::string(" (") + SERVER_NAME_AFFIX + ')';
 
+static void generateTranslations(const std::string & modID)
+{
+	LIBRARY = new GameLibrary;
+	LIBRARY->loadFilesystem(false);
+	settings.init("config/settings.json", "vcmi:settings");
+
+	ModManager mods;
+
+	if (!mods.isModActive(modID))
+		mods.tryEnableMods({modID});
+
+	for (const auto & submod : mods.getModSettings(modID))
+	{
+		try
+		{
+			if (!submod.second)
+				mods.tryEnableMods({modID + '.' + submod.first});
+		}
+		catch (const std::exception &)
+		{
+			// failed to enable mod - ignore, will be logged later
+		}
+	}
+
+	for (const auto & submod : mods.getModSettings(modID))
+		if (!submod.second)
+			logGlobal->warn("Failed to enable submod %s", submod.first);
+
+	std::map<std::string, ExportedStrings> textsByMod;
+	std::vector<std::string> modsWithOverrides;
+
+	delete LIBRARY;
+	LIBRARY = new GameLibrary;
+	LIBRARY->initializeFilesystem(false);
+	LIBRARY->initializeLibrary();
+	LIBRARY->generaltexth->exportAllTexts(textsByMod, false);
+
+	for(const auto & modEntry : textsByMod)
+	{
+		if (modEntry.first.find('.') != std::string::npos)
+		{
+			for (const auto & otherModID : modEntry.second.overridenMods)
+			{
+				if (otherModID == modID || otherModID.starts_with(modID + '.'))
+				{
+					modsWithOverrides.push_back(modEntry.first);
+					break;
+				}
+			}
+		}
+	}
+
+	const boost::filesystem::path outPath = VCMIDirs::get().userExtractedPath() / "translationFull";
+	boost::filesystem::create_directories(outPath);
+
+	for (const auto & modWithOverrides : modsWithOverrides)
+		mods.tryDisableMod(modWithOverrides);
+
+	CResourceHandler::destroy();
+	delete LIBRARY;
+	LIBRARY = new GameLibrary;
+	LIBRARY->initializeFilesystem(false);
+	LIBRARY->initializeLibrary();
+	LIBRARY->generaltexth->exportAllTexts(textsByMod, false);
+
+	for(const auto & modEntry : textsByMod)
+	{
+		JsonNode output;
+
+		if (modEntry.first != modID && !modEntry.first.starts_with(modID + '.'))
+			continue;
+
+		for(const auto & stringEntry : modEntry.second.strings)
+			output[stringEntry.first].String() = stringEntry.second;
+
+		if (!output.isNull())
+		{
+			std::string preferredLanguage = LIBRARY->generaltexth->getPreferredLanguage();
+			std::string filename = boost::replace_all_copy(modEntry.first, ".", "/Mods/");
+			const boost::filesystem::path dirPath = outPath / filename / "Content/config/translation/";
+			boost::filesystem::create_directories(dirPath);
+			const boost::filesystem::path filePath = dirPath / (preferredLanguage + ".json");
+			std::ofstream file(filePath.c_str());
+			file << output.toString();
+		}
+	}
+
+	logGlobal->info("Translation export complete");
+	logGlobal->info("Extracted files can be found in " + outPath.string() + " directory\n");
+
+}
+
 static void handleCommandOptions(int argc, const char * argv[], boost::program_options::variables_map & options)
 {
 	boost::program_options::options_description opts("Allowed options");
@@ -30,6 +127,7 @@ static void handleCommandOptions(int argc, const char * argv[], boost::program_o
 	("version,v", "display version information and exit")
 	("run-by-client", "indicate that server launched by client on same machine")
 	("dummy-run", "Shutdown immediately after loading was sucessful")
+	("translate-mod", boost::program_options::value<std::string>(), "Export translations for specified mod")
 	("port", boost::program_options::value<ui16>(), "port at which server will listen to connections from client")
 	("lobby", "start server in lobby mode in which server connects to a global lobby");
 
@@ -59,6 +157,13 @@ static void handleCommandOptions(int argc, const char * argv[], boost::program_o
 		exit(0);
 	}
 
+	if(options.count("translate-mod"))
+	{
+		std::string modID = options["translate-mod"].as<std::string>();
+		generateTranslations(modID);
+		exit(0);
+	}
+
 	if(options.count("version"))
 	{
 		printf("%s\n", GameConstants::VCMI_VERSION.c_str());