Fonts will now correctly detect encoding of fonts from mods

client/renderSDL/CBitmapFont.cpp

@@ -11,58 +11,86 @@
 #include "CBitmapFont.h"
 
 #include "SDL_Extensions.h"
+#include "../CGameInfo.h"
+#include "../render/Colors.h"
 
-#include "../../lib/vcmi_endian.h"
+#include "../../lib/CModHandler.h"
-#include "../../lib/filesystem/Filesystem.h"
+#include "../../lib/Languages.h"
-#include "../../lib/TextOperations.h"
 #include "../../lib/Rect.h"
+#include "../../lib/TextOperations.h"
+#include "../../lib/filesystem/Filesystem.h"
+#include "../../lib/vcmi_endian.h"
 
 #include <SDL_surface.h>
 
-std::array<CBitmapFont::BitmapChar, CBitmapFont::totalChars> CBitmapFont::loadChars() const
+void CBitmapFont::loadModFont(const std::string & modName, const ResourceID & resource)
 {
-	std::array<BitmapChar, totalChars> ret;
+	auto data = CResourceHandler::get(modName)->load(resource)->readAll();
+	std::string modLanguage = CGI->modh->getModLanguage(modName);
+	std::string modEncoding = Languages::getLanguageOptions(modLanguage).encoding;
 
-	size_t offset = 32;
+	uint32_t dataHeight = data.first[5];
 
-	for (auto & elem : ret)
+	maxHeight = std::max(maxHeight, dataHeight);
+
+	constexpr size_t symbolsInFile = 0x100;
+	constexpr size_t baseIndex = 32;
+	constexpr size_t offsetIndex = baseIndex + symbolsInFile*12;
+	constexpr size_t dataIndex = offsetIndex + symbolsInFile*4;
+
+	for (uint32_t charIndex = 0; charIndex < symbolsInFile; ++charIndex)
 	{
-		elem.leftOffset =  read_le_u32(data.first.get() + offset); offset+=4;
+		CodePoint codepoint = TextOperations::getUnicodeCodepoint(static_cast<char>(charIndex), modEncoding);
-		elem.width =       read_le_u32(data.first.get() + offset); offset+=4;
-		elem.rightOffset = read_le_u32(data.first.get() + offset); offset+=4;
-	}
 
-	for (auto & elem : ret)
+		BitmapChar symbol;
-	{
-		int pixelOffset =  read_le_u32(data.first.get() + offset); offset+=4;
-		elem.pixels = data.first.get() + 4128 + pixelOffset;
 
-		assert(pixelOffset + 4128 < data.second);
+		symbol.leftOffset =  read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 0);
+		symbol.width =       read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 4);
+		symbol.rightOffset = read_le_u32(data.first.get() + baseIndex + charIndex * 12 + 8);
+		symbol.height = dataHeight;
+
+		uint32_t pixelDataOffset = read_le_u32(data.first.get() + offsetIndex + charIndex * 4);
+		uint32_t pixelsCount = dataHeight * symbol.width;
+
+		symbol.pixels.resize(pixelsCount);
+
+		uint8_t * pixelData = data.first.get() + dataIndex + pixelDataOffset;
+
+		std::copy_n(pixelData, pixelsCount, symbol.pixels.data() );
+
+		chars[codepoint] = symbol;
 	}
-	return ret;
 }
 
 CBitmapFont::CBitmapFont(const std::string & filename):
-	data(CResourceHandler::get()->load(ResourceID("data/" + filename, EResType::BMP_FONT))->readAll()),
+	maxHeight(0)
-	chars(loadChars()),
+{
-	height(data.first.get()[5])
+	ResourceID resource("data/" + filename, EResType::BMP_FONT);
-{}
+
+	loadModFont("core", resource);
+
+	for (auto const & modName : VLC->modh->getActiveMods())
+	{
+		if (CResourceHandler::get(modName)->existsResource(resource))
+			loadModFont(modName, resource);
+	}
+}
 
 size_t CBitmapFont::getLineHeight() const
 {
-	return height;
+	return maxHeight;
 }
 
 size_t CBitmapFont::getGlyphWidth(const char * data) const
 {
-	std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "ASCII");
+	CodePoint localChar = TextOperations::getUnicodeCodepoint(data, 4);
 
-	if (localChar.size() == 1)
+	auto iter = chars.find(localChar);
-	{
+
-		const BitmapChar & ch = chars[ui8(localChar[0])];
+	if (iter == chars.end())
-		return ch.leftOffset + ch.width + ch.rightOffset;
+		return 0;
-	}
+
-	return 0;
+	return iter->second.leftOffset + iter->second.width + iter->second.rightOffset;
 }
 
 void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & character, const SDL_Color & color, int &posX, int &posY) const
@@ -78,7 +106,7 @@ void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & char
 
 	// start of line, may differ from 0 due to end of surface or clipped surface
 	int lineBegin = std::max<int>(0, clipRect.y - posY);
-	int lineEnd   = std::min<int>(height, clipRect.y + clipRect.h - posY - 1);
+	int lineEnd   = std::min<int>(character.height, clipRect.y + clipRect.h - posY - 1);
 
 	// start end end of each row, may differ from 0
 	int rowBegin = std::max<int>(0, clipRect.x - posX);
@@ -88,7 +116,7 @@ void CBitmapFont::renderCharacter(SDL_Surface * surface, const BitmapChar & char
 	for(int dy = lineBegin; dy <lineEnd; dy++)
 	{
 		uint8_t *dstLine = (uint8_t*)surface->pixels;
-		uint8_t *srcLine = character.pixels;
+		const uint8_t *srcLine = character.pixels.data();
 
 		// shift source\destination pixels to current position
 		dstLine += (posY+dy) * surface->pitch + posX * bpp;
@@ -133,10 +161,12 @@ void CBitmapFont::renderText(SDL_Surface * surface, const std::string & data, co
 
 	for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
 	{
-		std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "ASCII");
+		CodePoint codepoint = TextOperations::getUnicodeCodepoint(data.data() + i, data.size() - i);
 
-		if (localChar.size() == 1)
+		auto iter = chars.find(codepoint);
-			renderCharacter(surface, chars[ui8(localChar[0])], color, posX, posY);
+
+		if (iter != chars.end())
+			renderCharacter(surface, iter->second, color, posX, posY);
 	}
 	SDL_UnlockSurface(surface);
 }

client/renderSDL/CBitmapFont.h

@@ -11,30 +11,30 @@
 
 #include "../render/IFont.h"
 
+class ResourceID;
+
 class CBitmapFont : public IFont
 {
-	static const size_t totalChars = 256;
+	using CodePoint = uint32_t;
 
 	struct BitmapChar
 	{
-		si32 leftOffset;
+		int32_t leftOffset;
-		ui32 width;
+		uint32_t width;
-		si32 rightOffset;
+		uint32_t height;
-		ui8 *pixels; // pixels of this character, part of BitmapFont::data
+		int32_t rightOffset;
+		std::vector<uint8_t> pixels; // pixels of this character, part of BitmapFont::data
 	};
 
-	const std::pair<std::unique_ptr<ui8[]>, ui64> data;
+	std::unordered_map<CodePoint, BitmapChar> chars;
+	uint32_t maxHeight;
 
-	const std::array<BitmapChar, totalChars> chars;
+	void loadModFont(const std::string & modName, const ResourceID & resource);
-	const ui8 height;
-
-	std::array<BitmapChar, totalChars> loadChars() const;
 
 	void renderCharacter(SDL_Surface * surface, const BitmapChar & character, const SDL_Color & color, int &posX, int &posY) const;
-
 	void renderText(SDL_Surface * surface, const std::string & data, const SDL_Color & color, const Point & pos) const override;
 public:
-	CBitmapFont(const std::string & filename);
+	explicit CBitmapFont(const std::string & filename);
 
 	size_t getLineHeight() const override;
 	size_t getGlyphWidth(const char * data) const override;

lib/CModHandler.cpp

@@ -1106,6 +1106,9 @@ TModID CModHandler::findResourceOrigin(const ResourceID & name)
 			return modID;
 	}
 
+	if(CResourceHandler::get("core")->existsResource(name))
+		return "core";
+
 	assert(0);
 	return "";
 }

lib/TextOperations.cpp

@@ -112,6 +112,49 @@ bool TextOperations::isValidUnicodeString(const char * data, size_t size)
 	return true;
 }
 
+uint32_t TextOperations::getUnicodeCodepoint(const char * data, size_t maxSize)
+{
+	assert(isValidUnicodeCharacter(data, maxSize));
+	if (!isValidUnicodeCharacter(data, maxSize))
+		return 0;
+
+	// https://en.wikipedia.org/wiki/UTF-8#Encoding
+	switch (getUnicodeCharacterSize(data[0]))
+	{
+		case 1:
+			return static_cast<uint8_t>(data[0]) & 0b1111111;
+		case 2:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b11111 ) << 6) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 0) ;
+		case 3:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b1111 )  << 12) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 6) +
+				((static_cast<uint8_t>(data[2]) & 0b111111) << 0) ;
+		case 4:
+			return
+				((static_cast<uint8_t>(data[0]) & 0b111 )   << 18) +
+				((static_cast<uint8_t>(data[1]) & 0b111111) << 12) +
+				((static_cast<uint8_t>(data[2]) & 0b111111) << 6) +
+				((static_cast<uint8_t>(data[3]) & 0b111111) << 0) ;
+	}
+
+	assert(0);
+	return 0;
+}
+
+uint32_t TextOperations::getUnicodeCodepoint(char data, const std::string & encoding )
+{
+	std::string stringNative(1, data);
+	std::string stringUnicode = toUnicode(stringNative, encoding);
+
+	if (stringUnicode.empty())
+		return 0;
+
+	return getUnicodeCodepoint(stringUnicode.data(), stringUnicode.size());
+}
+
 std::string TextOperations::toUnicode(const std::string &text, const std::string &encoding)
 {
 	return boost::locale::conv::to_utf<char>(text, encoding);

lib/TextOperations.h

@@ -14,6 +14,12 @@ VCMI_LIB_NAMESPACE_BEGIN
 /// Namespace that provides utilites for unicode support (UTF-8)
 namespace TextOperations
 {
+	/// returns 32-bit UTF codepoint for UTF-8 character symbol
+	uint32_t DLL_LINKAGE getUnicodeCodepoint(const char *data, size_t maxSize);
+
+	/// returns 32-bit UTF codepoint for character symbol in selected single-byte encoding
+	uint32_t DLL_LINKAGE getUnicodeCodepoint(char data, const std::string & encoding );
+
 	/// returns length (in bytes) of UTF-8 character starting from specified character
 	size_t DLL_LINKAGE getUnicodeCharacterSize(char firstByte);