Added translation support for mods. VCMI translations are now in mod.

Moved existing (English) translation to vcmi mod
Added translations for:
- German
- Polish
- Ukrainian

Mods/vcmi/mod.json

@@ -7,6 +7,21 @@
 		"description" : "Grundlegende Dateien, die für die korrekte Ausführung von VCMI erforderlich sind",
 		"author" : "VCMI-Team",
 		"modType" : "Grafik",
+		
+		"translations" : [
+			"config/vcmi/german.json"
+		]
+	},
+	
+	"polish" : {
+		"name" : "VCMI essential files",
+		"description" : "Essential files required for VCMI to run correctly",
+		"author" : "VCMI Team",
+		"modType" : "Graphical",
+		
+		"translations" : [
+			"config/vcmi/polish.json"
+		]
 	},
 	
 	"ukrainian" : {
@@ -14,15 +29,23 @@
 		"description" : "Ключові файли необхідні для повноцінної роботи VCMI",
 		"author" : "Команда VCMI",
 		"modType" : "Графіка",
+		
+		"translations" : [
+			"config/vcmi/ukrainian.json"
+		]
 	},
 
-	"version" : "1.1",
+	"version" : "1.1.1",
 	"author" : "VCMI Team",
 	"contact" : "http://forum.vcmi.eu/index.php",
 	"modType" : "Graphical",
 	
 	"factions" : [ "config/vcmi/towerFactions" ],
 	"creatures" : [ "config/vcmi/towerCreature" ],
+		
+	"translations" : [
+		"config/vcmi/english.json"
+	],
 
 	"filesystem":
 	{

config/schemas/mod.json

@@ -8,7 +8,7 @@
 		"localizable" : {
 			"type":"object",
 			"additionalProperties" : false,
-			"required" : [ "name", "description", "author", "modType" ],
+			"required" : [ "name", "description", "modType" ],
 			"properties":{
 				"name": {
 					"type":"string",
@@ -18,7 +18,6 @@
 					"type":"string",
 					"description": "More lengthy description of mod. No hard limit"
 				},
-
 				"modType" : {
 					"type":"string",
 					"description": "Type of mod, e.g. Town, Artifacts, Graphical."
@@ -27,6 +26,11 @@
 					"type":"string",
 					"description": "Author of the mod. Can be nickname, real name or name of team"
 				},
+				"translations":{
+					"type":"array",
+					"description": "List of files with translations for this language",
+					"items": { "type":"string", "format" : "textFile" }
+				},
 				"changelog" : {
 					"type":"object",
 					"description": "List of changes/new features in each version",
@@ -112,27 +116,26 @@
 			"type":"boolean",
 			"description": "If set to true, mod will not be enabled automatically on install"
 		},
-		
 		"english" : {
 			"$ref" : "#/definitions/localizable"
 		},
-
 		"german" : {
 			"$ref" : "#/definitions/localizable"
 		},
-
 		"polish" : {
 			"$ref" : "#/definitions/localizable"
 		},
-
 		"russian" : {
 			"$ref" : "#/definitions/localizable"
 		},
-
 		"ukrainian" : {
 			"$ref" : "#/definitions/localizable"
 		},
-
+		"translations":{
+			"type":"array",
+			"description": "List of files with translations for this language",
+			"items": { "type":"string", "format" : "textFile" }
+		},
 		"artifacts": {
 			"type":"array",
 			"description": "List of configuration files for artifacts",

lib/CGeneralTextHandler.cpp

@@ -103,42 +103,25 @@ bool Unicode::isValidString(const char * data, size_t size)
 	return true;
 }
 
-static std::string getSelectedEncoding()
+/// Detects language and encoding of H3 text files based on matching against pregenerated footprints of H3 file
+void CGeneralTextHandler::detectInstallParameters() const
 {
-	auto explicitSetting = settings["general"]["encoding"].String();
-	if (explicitSetting != "auto")
-		return explicitSetting;
-	return settings["session"]["encoding"].String();
-}
+	struct LanguageFootprint
+	{
+		std::string language;
+		std::string encoding;
+		std::array<double, 16> footprint;
+	};
 
-/// Detects encoding of H3 text files based on matching against pregenerated footprints of H3 file
-/// Can also detect language of H3 install, however right now this is not necessary
-static void detectEncoding()
-{
-	static const size_t knownCount = 6;
-
-	// "footprints" of data collected from known versions of H3
-	static const std::array<std::array<double, 16>, knownCount> knownFootprints =
-	{ {
-		{ { 0.0559, 0.0000, 0.1983, 0.0051, 0.0222, 0.0183, 0.4596, 0.2405, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 } },
-		{ { 0.0493, 0.0000, 0.1926, 0.0047, 0.0230, 0.0121, 0.4133, 0.2780, 0.0002, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0259, 0.0008 } },
-		{ { 0.0534, 0.0000, 0.1705, 0.0047, 0.0418, 0.0208, 0.4775, 0.2191, 0.0001, 0.0000, 0.0000, 0.0000, 0.0000, 0.0005, 0.0036, 0.0080 } },
-		{ { 0.0534, 0.0000, 0.1701, 0.0067, 0.0157, 0.0133, 0.4328, 0.2540, 0.0001, 0.0043, 0.0000, 0.0244, 0.0000, 0.0000, 0.0181, 0.0071 } },
-		{ { 0.0548, 0.0000, 0.1744, 0.0061, 0.0031, 0.0009, 0.0046, 0.0136, 0.0000, 0.0004, 0.0000, 0.0000, 0.0227, 0.0061, 0.4882, 0.2252 } },
-		{ { 0.0559, 0.0000, 0.1807, 0.0059, 0.0036, 0.0013, 0.0046, 0.0134, 0.0000, 0.0004, 0.0000, 0.0487, 0.0209, 0.0060, 0.4615, 0.1972 } }
-	} };
-
-	// languages of known footprints
-	static const std::array<std::string, knownCount> knownLanguages =
-	{ {
-		  "English", "French", "German", "Polish", "Russian", "Ukrainian"
-	} };
-
-	// encoding that should be used for known footprints
-	static const std::array<std::string, knownCount> knownEncodings =
-	{ {
-		  "CP1252", "CP1252", "CP1252", "CP1250", "CP1251", "CP1251"
-	} };
+	static const std::vector<LanguageFootprint> knownFootprints =
+	{
+		{ "English",   "CP1252", { { 0.0559, 0.0000, 0.1983, 0.0051, 0.0222, 0.0183, 0.4596, 0.2405, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 } } },
+		{ "French",    "CP1252", { { 0.0493, 0.0000, 0.1926, 0.0047, 0.0230, 0.0121, 0.4133, 0.2780, 0.0002, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0259, 0.0008 } } },
+		{ "German",    "CP1252", { { 0.0534, 0.0000, 0.1705, 0.0047, 0.0418, 0.0208, 0.4775, 0.2191, 0.0001, 0.0000, 0.0000, 0.0000, 0.0000, 0.0005, 0.0036, 0.0080 } } },
+		{ "Polish",    "CP1250", { { 0.0534, 0.0000, 0.1701, 0.0067, 0.0157, 0.0133, 0.4328, 0.2540, 0.0001, 0.0043, 0.0000, 0.0244, 0.0000, 0.0000, 0.0181, 0.0071 } } },
+		{ "Russian",   "CP1251", { { 0.0548, 0.0000, 0.1744, 0.0061, 0.0031, 0.0009, 0.0046, 0.0136, 0.0000, 0.0004, 0.0000, 0.0000, 0.0227, 0.0061, 0.4882, 0.2252 } } },
+		{ "Ukrainian", "CP1251", { { 0.0559, 0.0000, 0.1807, 0.0059, 0.0036, 0.0013, 0.0046, 0.0134, 0.0000, 0.0004, 0.0000, 0.0487, 0.0209, 0.0060, 0.4615, 0.1972 } } },
+	};
 
 	// load file that will be used for footprint generation
 	// this is one of the most text-heavy files in game and consists solely from translated texts
@@ -146,11 +129,11 @@ static void detectEncoding()
 
 	std::array<size_t, 256> charCount;
 	std::array<double, 16> footprint;
-	std::array<double, knownCount> deviations;
+	std::vector<double> deviations;
 
 	boost::range::fill(charCount, 0);
 	boost::range::fill(footprint, 0.0);
-	boost::range::fill(deviations, 0.0);
+	deviations.resize(knownFootprints.size(), 0.0);
 
 	auto data = resource->readAll();
 
@@ -173,21 +156,24 @@ static void detectEncoding()
 	for (size_t i = 0; i < deviations.size(); ++i)
 	{
 		for (size_t j = 0; j < footprint.size(); ++j)
-			deviations[i] += std::abs((footprint[j] - knownFootprints[i][j]));
+			deviations[i] += std::abs((footprint[j] - knownFootprints[i].footprint[j]));
 	}
 
 	size_t bestIndex = boost::range::min_element(deviations) - deviations.begin();
 
 	for (size_t i = 0; i < deviations.size(); ++i)
-		logGlobal->debug("Comparing to %s: %f", knownLanguages[i], deviations[i]);
+		logGlobal->debug("Comparing to %s: %f", knownFootprints[i].language, deviations[i]);
 
-	Settings s = settings.write["session"]["encoding"];
-	s->String() = knownEncodings[bestIndex];
+	Settings language = settings.write["session"]["language"];
+	language->String() = knownFootprints[bestIndex].language;
+
+	Settings encoding = settings.write["session"]["encoding"];
+	encoding->String() = knownFootprints[bestIndex].encoding;
 }
 
 std::string Unicode::toUnicode(const std::string &text)
 {
-	return toUnicode(text, getSelectedEncoding());
+	return toUnicode(text, CGeneralTextHandler::getInstalledEncoding());
 }
 
 std::string Unicode::toUnicode(const std::string &text, const std::string &encoding)
@@ -197,7 +183,7 @@ std::string Unicode::toUnicode(const std::string &text, const std::string &encod
 
 std::string Unicode::fromUnicode(const std::string & text)
 {
-	return fromUnicode(text, getSelectedEncoding());
+	return fromUnicode(text, CGeneralTextHandler::getInstalledEncoding());
 }
 
 std::string Unicode::fromUnicode(const std::string &text, const std::string &encoding)
@@ -389,16 +375,14 @@ void CGeneralTextHandler::readToVector(std::string const & sourceID, std::string
 	while (parser.endLine());
 }
 
-const std::string & CGeneralTextHandler::serialize(const std::string & identifier) const
-{
-	assert(stringsIdentifiers.count(identifier));
-	return stringsIdentifiers.at(identifier);
-}
-
 const std::string & CGeneralTextHandler::deserialize(const TextIdentifier & identifier) const
 {
+	if(stringsOverrides.count(identifier.get()))
+		return stringsOverrides.at(identifier.get());
+
 	if(stringsLocalizations.count(identifier.get()))
 		return stringsLocalizations.at(identifier.get());
+
 	logGlobal->error("Unable to find localization for string '%s'", identifier.get());
 	return identifier.get();
 }
@@ -406,11 +390,20 @@ const std::string & CGeneralTextHandler::deserialize(const TextIdentifier & iden
 void CGeneralTextHandler::registerString(const TextIdentifier & UID, const std::string & localized)
 {
 	assert(UID.get().find("..") == std::string::npos);
-
-	stringsIdentifiers[localized] = UID.get();
 	stringsLocalizations[UID.get()] = localized;
 }
 
+void CGeneralTextHandler::registerStringOverride(const TextIdentifier & UID, const std::string & localized)
+{
+	stringsOverrides[UID.get()] = localized;
+}
+
+void CGeneralTextHandler::loadTranslationOverrides(const JsonNode & config)
+{
+	for ( auto const & node : config.Struct())
+		registerStringOverride(node.first, node.second.String());
+}
+
 CGeneralTextHandler::CGeneralTextHandler():
 	victoryConditions(*this, "core.vcdesc"   ),
 	lossCondtions    (*this, "core.lcdesc"   ),
@@ -441,8 +434,7 @@ CGeneralTextHandler::CGeneralTextHandler():
 	znpc00           (*this, "vcmi.znpc00"  ), // technically - wog
 	qeModCommands    (*this, "vcmi.quickExchange" )
 {
-	if (getSelectedEncoding().empty())
-		detectEncoding();
+	detectInstallParameters();
 
 	readToVector("core.vcdesc",   "DATA/VCDESC.TXT"   );
 	readToVector("core.lcdesc",   "DATA/LCDESC.TXT"   );
@@ -472,11 +464,6 @@ CGeneralTextHandler::CGeneralTextHandler():
 	if (CResourceHandler::get()->existsResource(ResourceID(QE_MOD_COMMANDS, EResType::TEXT)))
 		readToVector("vcmi.quickExchange", QE_MOD_COMMANDS);
 
-	auto vcmiTexts = JsonNode(ResourceID("config/translate.json", EResType::TEXT));
-
-	for ( auto const & node : vcmiTexts.Struct())
-		registerString(node.first, node.second.String());
-
 	{
 		CLegacyConfigParser parser("DATA/RANDTVRN.TXT");
 		parser.endLine();
@@ -638,18 +625,25 @@ int32_t CGeneralTextHandler::pluralText(const int32_t textIndex, const int32_t c
 
 void CGeneralTextHandler::dumpAllTexts()
 {
+	auto escapeString = [](std::string input)
+	{
+		boost::replace_all(input, "\\", "\\\\");
+		boost::replace_all(input, "\n", "\\n");
+		boost::replace_all(input, "\r", "\\r");
+		boost::replace_all(input, "\t", "\\t");
+		boost::replace_all(input, "\"", "\\\"");
+
+		return input;
+	};
+
 	logGlobal->info("BEGIN TEXT EXPORT");
 	for ( auto const & entry : stringsLocalizations)
-	{
-		auto cleanString = entry.second;
-		boost::replace_all(cleanString, "\\", "\\\\");
-		boost::replace_all(cleanString, "\n", "\\n");
-		boost::replace_all(cleanString, "\r", "\\r");
-		boost::replace_all(cleanString, "\t", "\\t");
-		boost::replace_all(cleanString, "\"", "\\\"");
+		if (stringsOverrides.count(entry.first) == 0)
+			logGlobal->info("\"%s\" : \"%s\",", entry.first, escapeString(entry.second));
+
+	for ( auto const & entry : stringsOverrides)
+		logGlobal->info("\"%s\" : \"%s\",", entry.first, escapeString(entry.second));
 
-		logGlobal->info("\"%s\" : \"%s\",", entry.first, cleanString);
-	}
 	logGlobal->info("END TEXT EXPORT");
 }
 
@@ -662,6 +656,22 @@ size_t CGeneralTextHandler::getCampaignLength(size_t campaignID) const
 	return 0;
 }
 
+std::string CGeneralTextHandler::getInstalledLanguage()
+{
+	auto explicitSetting = settings["general"]["language"].String();
+	if (explicitSetting != "auto")
+		return explicitSetting;
+	return settings["session"]["language"].String();
+}
+
+std::string CGeneralTextHandler::getInstalledEncoding()
+{
+	auto explicitSetting = settings["general"]["encoding"].String();
+	if (explicitSetting != "auto")
+		return explicitSetting;
+	return settings["session"]["encoding"].String();
+}
+
 std::vector<std::string> CGeneralTextHandler::findStringsWithPrefix(std::string const & prefix)
 {
 	std::vector<std::string> result;
@@ -698,5 +708,4 @@ std::pair<std::string, std::string> LegacyHelpContainer::operator[](size_t index
 	};
 }
 
-
 VCMI_LIB_NAMESPACE_END

lib/CGeneralTextHandler.h

@@ -149,17 +149,27 @@ class DLL_LINKAGE CGeneralTextHandler
 	/// map identifier -> localization
 	std::unordered_map<std::string, std::string> stringsLocalizations;
 
-	/// map localization -> identifier
-	std::unordered_map<std::string, std::string> stringsIdentifiers;
+	/// map identifier -> localization, high-priority strings from translation json
+	std::unordered_map<std::string, std::string> stringsOverrides;
 
 	void readToVector(const std::string & sourceID, const std::string & sourceName);
 
 	/// number of scenarios in specific campaign. TODO: move to a better location
 	std::vector<size_t> scenariosCountPerCampaign;
+
+	/// Attempts to detect encoding & language of H3 files
+	void detectInstallParameters() const;
 public:
+	/// Loads translation from provided json
+	/// Any entries loaded by this will have priority over texts registered normally
+	void loadTranslationOverrides(JsonNode const & file);
+
 	/// add selected string to internal storage
 	void registerString(const TextIdentifier & UID, const std::string & localized);
 
+	/// add selected string to internal storage as high-priority strings
+	void registerStringOverride(const TextIdentifier & UID, const std::string & localized);
+
 	// returns true if identifier with such name was registered, even if not translated to current language
 	// not required right now, can be added if necessary
 	// bool identifierExists( const std::string identifier) const;
@@ -172,9 +182,6 @@ public:
 		return deserialize(id);
 	}
 
-	/// converts translated string into locale-independent text that can be sent to another client
-	const std::string & serialize(const std::string & identifier) const;
-
 	/// converts identifier into user-readable string
 	const std::string & deserialize(const TextIdentifier & identifier) const;
 
@@ -226,6 +233,12 @@ public:
 	CGeneralTextHandler();
 	CGeneralTextHandler(const CGeneralTextHandler&) = delete;
 	CGeneralTextHandler operator=(const CGeneralTextHandler&) = delete;
+
+	/// Returns name of language preferred by user
+	static std::string getInstalledLanguage();
+
+	/// Returns name of encoding of Heroes III text files
+	static std::string getInstalledEncoding();
 };
 
 VCMI_LIB_NAMESPACE_END

lib/CModHandler.cpp

@@ -24,6 +24,7 @@
 #include "IHandlerBase.h"
 #include "spells/CSpellHandler.h"
 #include "CSkillHandler.h"
+#include "CGeneralTextHandler.h"
 #include "ScriptHandler.h"
 #include "RoadHandler.h"
 #include "RiverHandler.h"
@@ -455,7 +456,7 @@ CContentHandler::CContentHandler()
 
 void CContentHandler::init()
 {
- 	handlers.insert(std::make_pair("heroClasses", ContentTypeHandler(&VLC->heroh->classes, "heroClass")));
+	handlers.insert(std::make_pair("heroClasses", ContentTypeHandler(&VLC->heroh->classes, "heroClass")));
 	handlers.insert(std::make_pair("artifacts", ContentTypeHandler(VLC->arth, "artifact")));
 	handlers.insert(std::make_pair("creatures", ContentTypeHandler(VLC->creh, "creature")));
 	handlers.insert(std::make_pair("factions", ContentTypeHandler(VLC->townh, "faction")));
@@ -687,7 +688,7 @@ void CModInfo::loadLocalData(const JsonNode & data)
 		validated = data["validated"].Bool();
 		checksum  = strtol(data["checksum"].String().c_str(), nullptr, 16);
 	}
-	
+
 	//check compatibility
 	bool wasEnabled = enabled;
 	enabled = enabled && (vcmiCompatibleMin.isNull() || Version::GameVersion().compatible(vcmiCompatibleMin));
@@ -704,10 +705,10 @@ void CModInfo::loadLocalData(const JsonNode & data)
 
 CModHandler::CModHandler() : content(std::make_shared<CContentHandler>())
 {
-    modules.COMMANDERS = false;
-    modules.STACK_ARTIFACT = false;
-    modules.STACK_EXP = false;
-    modules.MITHRIL = false;
+	modules.COMMANDERS = false;
+	modules.STACK_ARTIFACT = false;
+	modules.STACK_EXP = false;
+	modules.MITHRIL = false;
 	for (int i = 0; i < GameConstants::RESOURCE_QUANTITY; ++i)
 	{
 		identifiers.registerObject(CModHandler::scopeBuiltin(), "resource", GameConstants::RESOURCE_NAMES[i], i);
@@ -841,7 +842,7 @@ std::vector <TModID> CModHandler::validateAndSortDependencies(std::vector <TModI
 	// Topological sort algorithm.
 	// TODO: Investigate possible ways to improve performance.
 	boost::range::sort(modsToResolve); // Sort mods per name
-	std::vector <TModID> sortedValidMods; // Vector keeps order of elements (LIFO) 
+	std::vector <TModID> sortedValidMods; // Vector keeps order of elements (LIFO)
 	sortedValidMods.reserve(modsToResolve.size()); // push_back calls won't cause memory reallocation
 	std::set <TModID> resolvedModIDs; // Use a set for validation for performance reason, but set does not keep order of elements
 
@@ -876,7 +877,7 @@ std::vector <TModID> CModHandler::validateAndSortDependencies(std::vector <TModI
 		if(resolvedOnCurrentTreeLevel.size())
 		{
 			resolvedModIDs.insert(resolvedOnCurrentTreeLevel.begin(), resolvedOnCurrentTreeLevel.end());
-		            continue;
+					continue;
 		}
 		// If there're no valid mods on the current mods tree level, no more mod can be resolved, should be end.
 		break;
@@ -1098,6 +1099,18 @@ void CModHandler::initializeConfig()
 	loadConfigFromFile("defaultMods.json");
 }
 
+void CModHandler::loadTranslation(TModID modName)
+{
+	auto const & mod = allMods[modName];
+	std::string language = VLC->generaltexth->getInstalledLanguage();
+
+	for (auto const & config : mod.config["translations"].Vector())
+		VLC->generaltexth->loadTranslationOverrides(JsonNode(ResourceID(config.String(), EResType::TEXT)));
+
+	for (auto const & config : mod.config[language]["translations"].Vector())
+		VLC->generaltexth->loadTranslationOverrides(JsonNode(ResourceID(config.String(), EResType::TEXT)));
+}
+
 void CModHandler::load()
 {
 	CStopWatch totalTime, timer;
@@ -1123,6 +1136,9 @@ void CModHandler::load()
 	for(const TModID & modName : activeMods)
 		content->load(allMods[modName]);
 
+	for(const TModID & modName : activeMods)
+		loadTranslation(modName);
+
 #if SCRIPTING_ENABLED
 	VLC->scriptHandler->performRegistration(VLC);//todo: this should be done before any other handlers load
 #endif

lib/CModHandler.h

@@ -282,6 +282,7 @@ class DLL_LINKAGE CModHandler
 	std::vector<std::string> getModList(std::string path);
 	void loadMods(std::string path, std::string parent, const JsonNode & modSettings, bool enableMods);
 	void loadOneMod(std::string modName, std::string parent, const JsonNode & modSettings, bool enableMods);
+	void loadTranslation(TModID modName);
 public:
 
 	/// returns true if scope is reserved for internal use and can not be used by mods