games/vcmi
1
0
Fork 0
mirror of https://github.com/vcmi/vcmi.git synced 2025-01-02 00:10:22 +02:00
Code Issues Releases Activity

Unicode conversion functions now require source encoding

Browse Source
This commit is contained in:
Ivan Savenko 2023-02-24 16:15:45 +02:00
parent 8b3309b47d
commit bd70b6fabd
14 changed files with 319 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/renderSDL/CBitmapFont.cpp
View File

@ -55,7 +55,7 @@ size_t CBitmapFont::getLineHeight() const
size_t CBitmapFont::getGlyphWidth(const char * data) const
{
std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])));
std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "ASCII");
if (localChar.size() == 1)
{
@ -133,7 +133,7 @@ void CBitmapFont::renderText(SDL_Surface * surface, const std::string & data, co
for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
{
std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])));
std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "ASCII");
if (localChar.size() == 1)
renderCharacter(surface, chars[ui8(localChar[0])], color, posX, posY);

4
client/renderSDL/CBitmapHanFont.cpp
View File

@ -85,7 +85,7 @@ void CBitmapHanFont::renderText(SDL_Surface * surface, const std::string & data,
for(size_t i=0; i<data.size(); i += TextOperations::getUnicodeCharacterSize(data[i]))
{
std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])));
std::string localChar = TextOperations::fromUnicode(data.substr(i, TextOperations::getUnicodeCharacterSize(data[i])), "GBK");
if (localChar.size() == 1)
fallback->renderCharacter(surface, fallback->chars[ui8(localChar[0])], color, posX, posY);
@ -115,7 +115,7 @@ size_t CBitmapHanFont::getLineHeight() const
size_t CBitmapHanFont::getGlyphWidth(const char * data) const
{
std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])));
std::string localChar = TextOperations::fromUnicode(std::string(data, TextOperations::getUnicodeCharacterSize(data[0])), "GBK");
if (localChar.size() == 1)
return fallback->getGlyphWidth(data);

2
lib/CGeneralTextHandler.cpp
View File

@ -190,7 +190,7 @@ std::string CLegacyConfigParser::readString()
std::string str = readRawString();
if (TextOperations::isValidASCII(str))
return str;
return TextOperations::toUnicode(str);
return TextOperations::toUnicode(str, fileEncoding);
}
float CLegacyConfigParser::readNumber()

5
lib/CGeneralTextHandler.h
View File

@ -17,12 +17,12 @@ class JsonNode;
/// Parser for any text files from H3
class DLL_LINKAGE CLegacyConfigParser
{
std::string fileEncoding;
std::unique_ptr<char[]> data;
char * curr;
char * end;
void init(const std::unique_ptr<CInputStream> & input);
/// extracts part of quoted string.
std::string extractQuotedPart();
@ -56,6 +56,7 @@ public:
bool endLine();
explicit CLegacyConfigParser(std::string URI);
private:
explicit CLegacyConfigParser(const std::unique_ptr<CInputStream> & input);
};

4
lib/CModHandler.cpp
View File

@ -943,8 +943,8 @@ std::vector<std::string> CModHandler::getModList(std::string path)
bool CModHandler::isScopeReserved(const TModID & scope)
{
static const std::array<TModID, 3> reservedScopes = {
"core", "map", "game"
static const std::array<TModID, 6> reservedScopes = {
"core", "map", "game", "root", "saves", "config"
};
return std::find(reservedScopes.begin(), reservedScopes.end(), scope) != reservedScopes.end();

10
lib/TextOperations.cpp
View File

@ -112,21 +112,11 @@ bool TextOperations::isValidUnicodeString(const char * data, size_t size)
return true;
}
std::string TextOperations::toUnicode(const std::string &text)
{
return toUnicode(text, CGeneralTextHandler::getInstalledEncoding());
}
std::string TextOperations::toUnicode(const std::string &text, const std::string &encoding)
{
return boost::locale::conv::to_utf<char>(text, encoding);
}
std::string TextOperations::fromUnicode(const std::string & text)
{
return fromUnicode(text, CGeneralTextHandler::getInstalledEncoding());
}
std::string TextOperations::fromUnicode(const std::string &text, const std::string &encoding)
{
return boost::locale::conv::from_utf<char>(text, encoding);

2
lib/TextOperations.h
View File

@ -31,12 +31,10 @@ namespace TextOperations
bool DLL_LINKAGE isValidUnicodeString(const char * data, size_t size);
/// converts text to UTF-8 from specified encoding or from one specified in settings
std::string DLL_LINKAGE toUnicode(const std::string & text);
std::string DLL_LINKAGE toUnicode(const std::string & text, const std::string & encoding);
/// converts text from unicode to specified encoding or to one specified in settings
/// NOTE: usage of these functions should be avoided if possible
std::string DLL_LINKAGE fromUnicode(const std::string & text);
std::string DLL_LINKAGE fromUnicode(const std::string & text, const std::string & encoding);
///delete specified amount of UTF-8 characters from right

7
lib/filesystem/CBinaryReader.cpp
View File

@ -86,7 +86,7 @@ INSTANTIATE(si64, readInt64)
#undef INSTANTIATE
std::string CBinaryReader::readString()
std::string CBinaryReader::readBaseString()
{
unsigned int len = readUInt32();
assert(len <= 500000); //not too long
@ -95,10 +95,7 @@ std::string CBinaryReader::readString()
std::string ret;
ret.resize(len);
read(reinterpret_cast<ui8*>(&ret[0]), len);
//FIXME: any need to move this into separate "read localized string" method?
if (TextOperations::isValidASCII(ret))
return ret;
return TextOperations::toUnicode(ret);
return ret;
}
return "";

3
lib/filesystem/CBinaryReader.h
View File

@ -73,7 +73,8 @@ public:
ui64 readUInt64();
si64 readInt64();
std::string readString();
/// Reads string without any encoding conversions
std::string readBaseString();
inline bool readBool()
{

2
lib/mapObjects/ObjectTemplate.cpp
View File

@ -198,7 +198,7 @@ void ObjectTemplate::readMsk()
void ObjectTemplate::readMap(CBinaryReader & reader)
{
animationFile = reader.readString();
animationFile = reader.readBaseString();
setSize(8, 6);
ui8 blockMask[6];

15
lib/mapping/CCampaignHandler.cpp
View File

@ -110,14 +110,19 @@ std::unique_ptr<CCampaign> CCampaignHandler::getCampaign( const std::string & na
return ret;
}
std::string CCampaignHandler::readLocalizedString(CBinaryReader & reader)
{
return reader.readBaseString();
}
CCampaignHeader CCampaignHandler::readHeaderFromMemory( CBinaryReader & reader )
{
CCampaignHeader ret;
ret.version = reader.readUInt32();
ret.mapVersion = reader.readUInt8() - 1;//change range of it from [1, 20] to [0, 19]
ret.name = reader.readString();
ret.description = reader.readString();
ret.name = readLocalizedString(reader);
ret.description = readLocalizedString(reader);
if (ret.version > CampaignVersion::RoE)
ret.difficultyChoosenByPlayer = reader.readInt8();
else
@ -136,14 +141,14 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
{
ret.prologVideo = reader.readUInt8();
ret.prologMusic = reader.readUInt8();
ret.prologText = reader.readString();
ret.prologText = readLocalizedString(reader);
}
return ret;
};
CCampaignScenario ret;
ret.conquered = false;
ret.mapName = reader.readString();
ret.mapName = readLocalizedString(reader);
ret.packedMapSize = reader.readUInt32();
if(mapVersion == 18)//unholy alliance
{
@ -155,7 +160,7 @@ CCampaignScenario CCampaignHandler::readScenarioFromMemory( CBinaryReader & read
}
ret.regionColor = reader.readUInt8();
ret.difficulty = reader.readUInt8();
ret.regionText = reader.readString();
ret.regionText = readLocalizedString(reader);
ret.prolog = prologEpilogReader();
ret.epilog = prologEpilogReader();

2
lib/mapping/CCampaignHandler.h
View File

@ -218,6 +218,8 @@ class DLL_LINKAGE CCampaignHandler
{
std::vector<size_t> scenariosCountPerCampaign;
static std::string readLocalizedString(CBinaryReader & reader);
static CCampaignHeader readHeaderFromMemory(CBinaryReader & reader);
static CCampaignScenario readScenarioFromMemory(CBinaryReader & reader, int version, int mapVersion );
static CScenarioTravel readScenarioTravelFromMemory(CBinaryReader & reader, int version);

555
lib/mapping/MapFormatH3M.cpp
View File

File diff suppressed because it is too large Load Diff

17
lib/mapping/MapFormatH3M.h
View File

@ -17,11 +17,11 @@
#include "../int3.h"
#include "../filesystem/CBinaryReader.h"
VCMI_LIB_NAMESPACE_BEGIN
class CGHeroInstance;
class CBinaryReader;
class CArtifactInstance;
class CGObjectInstance;
class CGSeerHut;
@ -234,14 +234,10 @@ private:
/**
* Helper to read map position
*/
inline int3 readInt3()
{
int3 p;
p.x = reader.readUInt8();
p.y = reader.readUInt8();
p.z = reader.readUInt8();
return p;
}
int3 readInt3();
/// reads string from input stream and converts it to unicode
std::string readLocalizedString();
void afterRead();
@ -257,8 +253,7 @@ private:
* (when loading a map then the mapHeader ptr points to the same object)
*/
std::unique_ptr<CMapHeader> mapHeader;
CBinaryReader reader;
std::unique_ptr<CBinaryReader> reader;
CInputStream * inputStream;
};