Better handling of encoding detection for maps and campaigns

Now VCMI will use either preferred language or install language to load
maps and campaigns that are part of "core" mod, or, in other words -
placed in Maps directory of H3 data (like most of manually downloaded
maps and campaigns are)

If game data is in English, then game can safely use encoding of player-
selected language (such as Chinese) to load maps. After all, both GBK
and all Win-125X encoding are superset of ASCII, so English map will
always load up correctly.

Maps that are part of a mod still use mod language as before - it is up
to mod maker to correctly set up mod language.

client/renderSDL/CBitmapFont.cpp

@@ -99,9 +99,7 @@ static AtlasLayout doAtlasPacking(const std::map<int, Point> & images)
 void CBitmapFont::loadFont(const ResourcePath & resource, std::unordered_map<CodePoint, EntryFNT> & loadedChars)
 {
 	auto data = CResourceHandler::get()->load(resource)->readAll();
-	std::string modName = VLC->modh->findResourceOrigin(resource);
-	std::string modLanguage = VLC->modh->getModLanguage(modName);
-	std::string modEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+	std::string modEncoding = VLC->modh->findResourceEncoding(resource);
 
 	height = data.first[5];
 

client/renderSDL/FontChain.cpp

@@ -60,11 +60,9 @@ bool FontChain::bitmapFontsPrioritized(const std::string & bitmapFontName) const
 	if (!vstd::isAlmostEqual(1.0, settings["video"]["fontScalingFactor"].Float()))
 		return false; // If player requested non-100% scaling - use scalable fonts
 
-	std::string modName = CGI->modh->findResourceOrigin(ResourcePath("data/" + bitmapFontName, EResType::BMP_FONT));
-	std::string fontLanguage = CGI->modh->getModLanguage(modName);
 	std::string gameLanguage = CGI->generaltexth->getPreferredLanguage();
-	std::string fontEncoding = Languages::getLanguageOptions(fontLanguage).encoding;
 	std::string gameEncoding = Languages::getLanguageOptions(gameLanguage).encoding;
+	std::string fontEncoding = CGI->modh->findResourceEncoding(ResourcePath("data/" + bitmapFontName, EResType::BMP_FONT));
 
 	// player uses language with different encoding than his bitmap fonts
 	// for example, Polish language with English fonts or Chinese language which can't use H3 fonts at all

lib/campaign/CampaignHandler.cpp

@@ -25,7 +25,6 @@
 #include "../modding/IdentifierStorage.h"
 #include "../modding/ModScope.h"
 #include "../texts/CGeneralTextHandler.h"
-#include "../texts/Languages.h"
 #include "../texts/TextOperations.h"
 
 VCMI_LIB_NAMESPACE_BEGIN
@@ -64,8 +63,7 @@ std::unique_ptr<Campaign> CampaignHandler::getHeader( const std::string & name)
 {
 	ResourcePath resourceID(name, EResType::CAMPAIGN);
 	std::string modName = VLC->modh->findResourceOrigin(resourceID);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(resourceID);
 	
 	auto ret = std::make_unique<Campaign>();
 	auto fileStream = CResourceHandler::get(modName)->load(resourceID);
@@ -80,8 +78,7 @@ std::shared_ptr<CampaignState> CampaignHandler::getCampaign( const std::string &
 {
 	ResourcePath resourceID(name, EResType::CAMPAIGN);
 	std::string modName = VLC->modh->findResourceOrigin(resourceID);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(resourceID);
 	
 	auto ret = std::make_unique<CampaignState>();
 	

lib/mapping/CMapService.cpp

@@ -19,7 +19,6 @@
 #include "../modding/CModHandler.h"
 #include "../modding/ModScope.h"
 #include "../modding/CModInfo.h"
-#include "../texts/Languages.h"
 #include "../VCMI_Lib.h"
 
 #include "CMap.h"
@@ -34,8 +33,7 @@ VCMI_LIB_NAMESPACE_BEGIN
 std::unique_ptr<CMap> CMapService::loadMap(const ResourcePath & name, IGameCallback * cb) const
 {
 	std::string modName = VLC->modh->findResourceOrigin(name);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(name);
 
 	auto stream = getStreamFromFS(name);
 	return getMapLoader(stream, name.getName(), modName, encoding)->loadMap(cb);
@@ -44,8 +42,7 @@ std::unique_ptr<CMap> CMapService::loadMap(const ResourcePath & name, IGameCallb
 std::unique_ptr<CMapHeader> CMapService::loadMapHeader(const ResourcePath & name) const
 {
 	std::string modName = VLC->modh->findResourceOrigin(name);
-	std::string language = VLC->modh->getModLanguage(modName);
-	std::string encoding = Languages::getLanguageOptions(language).encoding;
+	std::string encoding = VLC->modh->findResourceEncoding(name);
 
 	auto stream = getStreamFromFS(name);
 	return getMapLoader(stream, name.getName(), modName, encoding)->loadMapHeader();

lib/modding/CModHandler.cpp

@@ -390,6 +390,36 @@ TModID CModHandler::findResourceOrigin(const ResourcePath & name) const
 	throw std::runtime_error("Resource with name " + name.getName() + " and type " + EResTypeHelper::getEResTypeAsString(name.getType()) + " wasn't found.");
 }
 
+std::string CModHandler::findResourceLanguage(const ResourcePath & name) const
+{
+	std::string modName = findResourceOrigin(name);
+	std::string modLanguage = getModLanguage(modName);
+	return modLanguage;
+}
+
+std::string CModHandler::findResourceEncoding(const ResourcePath & resource) const
+{
+	std::string modName = findResourceOrigin(resource);
+	std::string modLanguage = findResourceLanguage(resource);
+
+	bool potentiallyUserMadeContent = resource.getType() == EResType::MAP || resource.getType() == EResType::CAMPAIGN;
+	if (potentiallyUserMadeContent && modName == ModScope::scopeBuiltin() && modLanguage == "english")
+	{
+		// this might be a map or campaign that player downloaded manually and placed in Maps/ directory
+		// in this case, this file may be in user-preferred language, and not in same language as the rest of H3 data
+		// however at the moment we have no way to detect that for sure - file can be either in English or in user-preferred language
+		// but since all known H3 encodings (Win125X or GBK) are supersets of ASCII, we can safely load English data using encoding of user-preferred language
+		std::string preferredLanguage = VLC->generaltexth->getPreferredLanguage();
+		std::string fileEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+		return fileEncoding;
+	}
+	else
+	{
+		std::string fileEncoding = Languages::getLanguageOptions(modLanguage).encoding;
+		return fileEncoding;
+	}
+}
+
 std::string CModHandler::getModLanguage(const TModID& modId) const
 {
 	if(modId == "core")

lib/modding/CModHandler.h

@@ -62,6 +62,12 @@ public:
 	/// returns ID of mod that provides selected file resource
 	TModID findResourceOrigin(const ResourcePath & name) const;
 
+	/// Returns assumed language ID of mod that provides selected file resource
+	std::string findResourceLanguage(const ResourcePath & name) const;
+
+	/// Returns assumed encoding of language of mod that provides selected file resource
+	std::string findResourceEncoding(const ResourcePath & name) const;
+
 	std::string getModLanguage(const TModID & modId) const;
 
 	std::set<TModID> getModDependencies(const TModID & modId) const;

lib/texts/CLegacyConfigParser.cpp

@@ -32,9 +32,7 @@ protected:
 CLegacyConfigParser::CLegacyConfigParser(const TextPath & resource)
 {
 	auto input = CResourceHandler::get()->load(resource);
-	std::string modName = VLC->modh->findResourceOrigin(resource);
-	std::string language = VLC->modh->getModLanguage(modName);
-	fileEncoding = Languages::getLanguageOptions(language).encoding;
+	fileEncoding = VLC->modh->findResourceEncoding(resource);
 
 	data.reset(new char[input->getSize()]);
 	input->read(reinterpret_cast<uint8_t*>(data.get()), input->getSize());