games/vcmi
1
0
Fork 0
mirror of https://github.com/vcmi/vcmi.git synced 2025-10-31 00:07:39 +02:00
Code Issues Releases Activity

- vcmi can now calculate crc32 checksum of a file

Browse Source 
- reorganized internal filesystem structure - all files from one mod are
now grouped in same FS node
- modhandler will now calculate crc32 checksum for each mod
- modhandler now knows validation status of each mod

todo - use checksum to determine mods that have not changed since last
start and disable validation for them.
This commit is contained in:
Ivan Savenko
2013-11-08 20:36:26 +00:00
parent ee6cdbeffe
commit dd73573c5c
21 changed files with 575 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Global.h
View File

@@ -100,6 +100,7 @@ static_assert(sizeof(bool) == 1, "Bool needs to be 1 byte in size.");
#include <boost/assign.hpp>
#include <boost/cstdint.hpp>
#include <boost/current_function.hpp>
#include <boost/crc.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/date_time/posix_time/posix_time_io.hpp>
#include <boost/filesystem.hpp>

2
launcher/mainwindow_moc.cpp
View File

@@ -17,7 +17,7 @@ void MainWindow::load()
logConfig.configureDefault();
CResourceHandler::initialize();
CResourceHandler::loadMainFileSystem("config/filesystem.json");
CResourceHandler::load("config/filesystem.json");
for (auto & string : VCMIDirs::get().dataPaths())
QDir::addSearchPath("icons", QString::fromUtf8(string.c_str()) + "/launcher/icons");

1
lib/CMakeLists.txt
View File

@@ -22,6 +22,7 @@ set(lib_SRCS
filesystem/CFileInputStream.cpp
filesystem/CZipLoader.cpp
filesystem/Filesystem.cpp
filesystem/ResourceID.cpp
logging/CBasicLogConfigurator.cpp
logging/CLogger.cpp

104
lib/CModHandler.cpp
View File

@@ -204,9 +204,10 @@ CContentHandler::ContentTypeHandler::ContentTypeHandler(IHandlerBase * handler,
}
}
void CContentHandler::ContentTypeHandler::preloadModData(std::string modName, std::vector<std::string> fileList)
bool CContentHandler::ContentTypeHandler::preloadModData(std::string modName, std::vector<std::string> fileList)
{
JsonNode data = JsonUtils::assembleFromFiles(fileList);
bool result;
JsonNode data = JsonUtils::assembleFromFiles(fileList, result);
data.setMeta(modName);
ModInfo & modInfo = modData[modName];
@@ -234,11 +235,13 @@ void CContentHandler::ContentTypeHandler::preloadModData(std::string modName, st
JsonUtils::merge(remoteConf, entry.second);
}
}
return result;
}
void CContentHandler::ContentTypeHandler::loadMod(std::string modName)
bool CContentHandler::ContentTypeHandler::loadMod(std::string modName)
{
ModInfo & modInfo = modData[modName];
bool result = true;
// apply patches
if (!modInfo.patches.isNull())
@@ -257,7 +260,7 @@ void CContentHandler::ContentTypeHandler::loadMod(std::string modName)
if (originalData.size() > index)
{
JsonUtils::merge(originalData[index], data);
JsonUtils::validate(originalData[index], "vcmi:" + objectName, name);
result &= JsonUtils::validate(originalData[index], "vcmi:" + objectName, name);
handler->loadObject(modName, name, originalData[index], index);
originalData[index].clear(); // do not use same data twice (same ID)
@@ -266,9 +269,10 @@ void CContentHandler::ContentTypeHandler::loadMod(std::string modName)
}
}
// normal new object or one with index bigger that data size
JsonUtils::validate(data, "vcmi:" + objectName, name);
result &= JsonUtils::validate(data, "vcmi:" + objectName, name);
handler->loadObject(modName, name, data);
}
return result;
}
void CContentHandler::ContentTypeHandler::afterLoadFinalization()
@@ -287,20 +291,24 @@ CContentHandler::CContentHandler()
//TODO: spells, bonuses, something else?
}
void CContentHandler::preloadModData(std::string modName, JsonNode modConfig)
bool CContentHandler::preloadModData(std::string modName, JsonNode modConfig)
{
bool result = true;
for(auto & handler : handlers)
{
handler.second.preloadModData(modName, modConfig[handler.first].convertTo<std::vector<std::string> >());
result &= handler.second.preloadModData(modName, modConfig[handler.first].convertTo<std::vector<std::string> >());
}
return result;
}
void CContentHandler::loadMod(std::string modName)
bool CContentHandler::loadMod(std::string modName)
{
bool result = true;
for(auto & handler : handlers)
{
handler.second.loadMod(modName);
result &= handler.second.loadMod(modName);
}
return result;
}
void CContentHandler::afterLoadFinalization()
@@ -520,11 +528,70 @@ void CModHandler::initialize(std::vector<std::string> availableMods)
std::ofstream file(*CResourceHandler::get()->getResourceName(ResourceID("config/modSettings.json")), std::ofstream::trunc);
file << modConfig;
loadModFilesystems();
}
std::vector<std::string> CModHandler::getActiveMods()
static JsonNode genDefaultFS()
{
return activeMods;
// default FS config for mods: directory "Content" that acts as H3 root directory
JsonNode defaultFS;
defaultFS[""].Vector().resize(2);
defaultFS[""].Vector()[0]["type"].String() = "zip";
defaultFS[""].Vector()[0]["path"].String() = "/Content.zip";
defaultFS[""].Vector()[1]["type"].String() = "dir";
defaultFS[""].Vector()[1]["path"].String() = "/Content";
return defaultFS;
}
static ISimpleResourceLoader * genModFilesystem(const std::string & modName, const JsonNode & conf)
{
static const JsonNode defaultFS = genDefaultFS();
if (!conf["filesystem"].isNull())
return CResourceHandler::createFileSystem("mods/" + modName, conf["filesystem"]);
else
return CResourceHandler::createFileSystem("mods/" + modName, defaultFS);
}
static ui32 calculateModChecksum(const std::string modName, ISimpleResourceLoader * filesystem)
{
boost::crc_32_type modChecksum;
// first - add current VCMI version into checksum to force re-validation on VCMI updates
modChecksum.process_bytes(reinterpret_cast<const void*>(GameConstants::VCMI_VERSION.data()), GameConstants::VCMI_VERSION.size());
// second - add mod.json into checksum because filesystem does not contains this file
ResourceID modConfFile("mods/" + modName + "/mod", EResType::TEXT);
ui32 configChecksum = CResourceHandler::getInitial()->load(modConfFile)->calculateCRC32();
modChecksum.process_bytes(reinterpret_cast<const void *>(&configChecksum), sizeof(configChecksum));
// third - add all loaded files from this mod into checksum
auto files = filesystem->getFilteredFiles([](const ResourceID & resID)
{
return resID.getType() == EResType::TEXT;
});
for (const ResourceID & file : files)
{
ui32 fileChecksum = filesystem->load(file)->calculateCRC32();
modChecksum.process_bytes(reinterpret_cast<const void *>(&fileChecksum), sizeof(fileChecksum));
}
return modChecksum.checksum();
}
void CModHandler::loadModFilesystems()
{
for(std::string & modName : activeMods)
{
ResourceID modConfFile("mods/" + modName + "/mod", EResType::TEXT);
auto fsConfigData = CResourceHandler::getInitial()->load(modConfFile)->readAll();
const JsonNode fsConfig((char*)fsConfigData.first.get(), fsConfigData.second);
auto filesystem = genModFilesystem(modName, fsConfig);
CResourceHandler::get()->addLoader(filesystem, false);
allMods[modName].checksum = calculateModChecksum(modName, filesystem);
}
}
CModInfo & CModHandler::getModData(TModID modId)
@@ -552,14 +619,16 @@ void CModHandler::loadGameContent()
for(const TModID & modName : activeMods)
{
logGlobal->infoStream() << "\t\t" << allMods[modName].name;
// print message in format [<8-symbols checksum>] <modname>
logGlobal->infoStream() << "\t\t[" << std::noshowbase << std::hex << std::setw(8) << std::setfill('0')
<< allMods[modName].checksum << "] " << allMods[modName].name;
std::string modFileName = "mods/" + modName + "/mod.json";
const JsonNode config = JsonNode(ResourceID(modFileName));
JsonUtils::validate(config, "vcmi:mod", modName);
allMods[modName].validated = JsonUtils::validate(config, "vcmi:mod", modName);
content.preloadModData(modName, config);
allMods[modName].validated &= content.preloadModData(modName, config);
}
logGlobal->infoStream() << "\tParsing mod data: " << timer.getDiff() << " ms";
@@ -568,8 +637,11 @@ void CModHandler::loadGameContent()
for(const TModID & modName : activeMods)
{
content.loadMod(modName);
logGlobal->infoStream() << "\t\t" << allMods[modName].name;
allMods[modName].validated &= content.loadMod(modName);
if (allMods[modName].validated)
logGlobal->infoStream() << "\t\t[DONE] " << allMods[modName].name;
else
logGlobal->errorStream() << "\t\t[FAIL] " << allMods[modName].name;
}
logGlobal->infoStream() << "\tLoading mod data: " << timer.getDiff() << "ms";

27
lib/CModHandler.h
View File

@@ -91,8 +91,9 @@ class CContentHandler
ContentTypeHandler(IHandlerBase * handler, std::string objectName);
/// local version of methods in ContentHandler
void preloadModData(std::string modName, std::vector<std::string> fileList);
void loadMod(std::string modName);
/// returns true if loading was successfull
bool preloadModData(std::string modName, std::vector<std::string> fileList);
bool loadMod(std::string modName);
void afterLoadFinalization();
};
@@ -101,11 +102,13 @@ public:
/// fully initialize object. Will cause reading of H3 config files
CContentHandler();
/// preloads all data from fileList as data from modName
void preloadModData(std::string modName, JsonNode modConfig);
/// preloads all data from fileList as data from modName.
/// returns true if loading was successfull
bool preloadModData(std::string modName, JsonNode modConfig);
/// actually loads data in mod
void loadMod(std::string modName);
/// returns true if loading was successfull
bool loadMod(std::string modName);
/// all data was loaded, time for final validation / integration
void afterLoadFinalization();
@@ -129,12 +132,18 @@ public:
/// list of mods that can't be used in the same time as this one
std::set <TModID> conflicts;
/// CRC-32 checksum of the mod
ui32 checksum;
/// true if mod has passed validation successfully
bool validated;
// mod configuration (mod.json). (no need to store it right now)
// std::shared_ptr<JsonNode> config; //TODO: unique_ptr can't be serialized
template <typename Handler> void serialize(Handler &h, const int version)
{
h & identifier & description & name & dependencies & conflicts;
h & identifier & description & name & dependencies & conflicts & checksum & validated;
}
};
@@ -143,7 +152,8 @@ class DLL_LINKAGE CModHandler
std::map <TModID, CModInfo> allMods;
std::vector <TModID> activeMods;//active mods, in order in which they were loaded
void loadConfigFromFile (std::string name);
void loadConfigFromFile(std::string name);
void loadModFilesystems();
bool hasCircularDependency(TModID mod, std::set <TModID> currentList = std::set <TModID>()) const;
@@ -163,9 +173,6 @@ public:
/// receives list of available mods and trying to load mod.json from all of them
void initialize(std::vector<std::string> availableMods);
/// returns list of mods that should be active with order in which they shoud be loaded
std::vector<std::string> getActiveMods();
CModInfo & getModData(TModID modId);
/// load content from all available mods

5
lib/JsonDetail.cpp
View File

@@ -149,6 +149,11 @@ JsonNode JsonParser::parse(std::string fileName)
return root;
}
bool JsonParser::isValid()
{
return errors.empty();
}
bool JsonParser::extractSeparator()
{
if (!extractWhitespace())

5
lib/JsonDetail.h
View File

@@ -88,6 +88,9 @@ public:
/// do actual parsing. filename is name of file that will printed to console if any errors were found
JsonNode parse(std::string fileName);
/// returns true if parsing was successfull
bool isValid();
};
//Internal class for Json validation. Mostly compilant with json-schema v4 draft
@@ -120,4 +123,4 @@ namespace Validation
std::string check(std::string schemaName, const JsonNode & data);
std::string check(std::string schemaName, const JsonNode & data, ValidationData & validator);
std::string check(const JsonNode & schema, const JsonNode & data, ValidationData & validator);
}
}

25
lib/JsonNode.cpp
View File

@@ -49,6 +49,16 @@ JsonNode::JsonNode(ResourceID && fileURI):
*this = parser.parse(fileURI.getName());
}
JsonNode::JsonNode(ResourceID && fileURI, bool &isValidSyntax):
type(DATA_NULL)
{
auto file = CResourceHandler::get()->load(fileURI)->readAll();
JsonParser parser(reinterpret_cast<char*>(file.first.get()), file.second);
*this = parser.parse(fileURI.getName());
isValidSyntax = parser.isValid();
}
JsonNode::JsonNode(const JsonNode &copy):
type(DATA_NULL),
meta(copy.meta)
@@ -636,8 +646,8 @@ bool JsonUtils::validate(const JsonNode &node, std::string schemaName, std::stri
std::string log = Validation::check(schemaName, node);
if (!log.empty())
{
logGlobal->errorStream() << "Data in " << dataName << " is invalid!";
logGlobal->errorStream() << log;
logGlobal->warnStream() << "Data in " << dataName << " is invalid!";
logGlobal->warnStream() << log;
}
return log.empty();
}
@@ -743,12 +753,21 @@ void JsonUtils::mergeCopy(JsonNode & dest, JsonNode source)
JsonNode JsonUtils::assembleFromFiles(std::vector<std::string> files)
{
bool isValid;
return assembleFromFiles(files, isValid);
}
JsonNode JsonUtils::assembleFromFiles(std::vector<std::string> files, bool &isValid)
{
isValid = true;
JsonNode result;
for(std::string file : files)
{
JsonNode section(ResourceID(file, EResType::TEXT));
bool isValidFile;
JsonNode section(ResourceID(file, EResType::TEXT), isValidFile);
merge(result, section);
isValid |= isValidFile;
}
return result;
}

2
lib/JsonNode.h
View File

@@ -55,6 +55,7 @@ public:
explicit JsonNode(const char * data, size_t datasize);
//Create tree from JSON file
explicit JsonNode(ResourceID && fileURI);
explicit JsonNode(ResourceID && fileURI, bool & isValidSyntax);
//Copy c-tor
JsonNode(const JsonNode &copy);
@@ -164,6 +165,7 @@ namespace JsonUtils
* @param files - list of filenames with parts of json structure
*/
DLL_LINKAGE JsonNode assembleFromFiles(std::vector<std::string> files);
DLL_LINKAGE JsonNode assembleFromFiles(std::vector<std::string> files, bool & isValid);
/// This version loads all files with same name (overriden by mods)
DLL_LINKAGE JsonNode assembleFromFiles(std::string filename);

3
lib/VCMI_Lib.cpp
View File

@@ -65,14 +65,13 @@ void LibClasses::loadFilesystem()
CResourceHandler::initialize();
logGlobal->infoStream()<<"\t Initialization: "<<loadTime.getDiff();
CResourceHandler::loadMainFileSystem("config/filesystem.json");
CResourceHandler::load("config/filesystem.json");
logGlobal->infoStream()<<"\t Data loading: "<<loadTime.getDiff();
modh = new CModHandler;
logGlobal->infoStream()<<"\tMod handler: "<<loadTime.getDiff();
modh->initialize(CResourceHandler::getAvailableMods());
CResourceHandler::setActiveMods(modh->getActiveMods());
logGlobal->infoStream()<<"\t Mod filesystems: "<<loadTime.getDiff();
logGlobal->infoStream()<<"Basic initialization: "<<totalTime.getDiff();

16
lib/filesystem/AdapterLoaders.cpp
View File

@@ -2,6 +2,7 @@
#include "AdapterLoaders.h"
#include "../JsonNode.h"
#include "Filesystem.h"
CMappedFileLoader::CMappedFileLoader(const std::string & mountPoint, const JsonNode &config)
{
@@ -131,18 +132,3 @@ void CFilesystemList::addLoader(ISimpleResourceLoader * loader, bool writeable)
if (writeable)
writeableLoaders.insert(loader);
}
ISimpleResourceLoader * CResourceHandler::get()
{
if(resourceLoader != nullptr)
{
return resourceLoader;
}
else
{
std::stringstream string;
string << "Error: Resource loader wasn't initialized. "
<< "Make sure that you set one via FilesystemFactory::initialize";
throw std::runtime_error(string.str());
}
}

5
lib/filesystem/AdapterLoaders.h
View File

@@ -11,10 +11,11 @@
*/
#include "ISimpleResourceLoader.h"
#include "Filesystem.h"
#include "ResourceID.h"
class CFileInfo;
class CInputStream;
class JsonNode;
/**
* Class that implements file mapping (aka *nix symbolic links)
@@ -78,4 +79,4 @@ public:
* @param writeable - resource shall be treated as writeable
*/
void addLoader(ISimpleResourceLoader * loader, bool writeable);
};
};

2
lib/filesystem/CArchiveLoader.h
View File

@@ -11,7 +11,7 @@
*/
#include "ISimpleResourceLoader.h"
#include "Filesystem.h"
#include "ResourceID.h"
class CFileInfo;
class CFileInputStream;

2
lib/filesystem/CFileInfo.h
View File

@@ -10,7 +10,7 @@
*
*/
#include "Filesystem.h"
#include "ResourceID.h"
/**
* A class which holds information about a file.

2
lib/filesystem/CFilesystemLoader.h
View File

@@ -11,7 +11,7 @@
*/
#include "ISimpleResourceLoader.h"
#include "Filesystem.h"
#include "ResourceID.h"
class CFileInfo;
class CInputStream;

18
lib/filesystem/CInputStream.h
View File

@@ -69,9 +69,27 @@ public:
{
std::unique_ptr<ui8[]> data(new ui8[getSize()]);
seek(0);
size_t readSize = read(data.get(), getSize());
assert(readSize == getSize());
return std::make_pair(std::move(data), getSize());
}
/**
* @brief calculateCRC32 calculates CRC32 checksum for the whole file
* @return calculated checksum
*/
virtual ui32 calculateCRC32()
{
si64 originalPos = tell();
boost::crc_32_type checksum;
auto data = readAll();
checksum.process_bytes(reinterpret_cast<const void *>(data.first.get()), data.second);
seek(originalPos);
return checksum.checksum();
}
};

4
lib/filesystem/CZipLoader.h
View File

@@ -12,7 +12,7 @@
#include "ISimpleResourceLoader.h"
#include "CInputStream.h"
#include "Filesystem.h"
#include "ResourceID.h"
#include "CCompressedStream.h"
// Necessary here in order to get all types
@@ -67,4 +67,4 @@ namespace ZipArchive
///same as above, but extracts only files mentioned in "what" list
bool DLL_LINKAGE extract(std::string from, std::string where, std::vector<std::string> what);
}
}

283
lib/filesystem/Filesystem.cpp
View File

@@ -17,50 +17,97 @@
CFilesystemList * CResourceHandler::resourceLoader = nullptr;
CFilesystemList * CResourceHandler::initialLoader = nullptr;
ResourceID::ResourceID()
:type(EResType::OTHER)
CFilesystemGenerator::CFilesystemGenerator(std::string prefix):
filesystem(new CFilesystemList()),
prefix(prefix)
{
}
ResourceID::ResourceID(std::string name)
CFilesystemGenerator::TLoadFunctorMap CFilesystemGenerator::genFunctorMap()
{
CFileInfo info(std::move(name));
setName(info.getStem());
setType(info.getType());
TLoadFunctorMap map;
map["map"] = boost::bind(&CFilesystemGenerator::loadJsonMap, this, _1, _2);
map["dir"] = boost::bind(&CFilesystemGenerator::loadDirectory, this, _1, _2);
map["lod"] = boost::bind(&CFilesystemGenerator::loadArchive<EResType::ARCHIVE_LOD>, this, _1, _2);
map["snd"] = boost::bind(&CFilesystemGenerator::loadArchive<EResType::ARCHIVE_SND>, this, _1, _2);
map["vid"] = boost::bind(&CFilesystemGenerator::loadArchive<EResType::ARCHIVE_VID>, this, _1, _2);
map["zip"] = boost::bind(&CFilesystemGenerator::loadZipArchive, this, _1, _2);
return map;
}
ResourceID::ResourceID(std::string name, EResType::Type type)
void CFilesystemGenerator::loadConfig(const JsonNode & config)
{
setName(std::move(name));
setType(type);
for(auto & mountPoint : config.Struct())
{
for(auto & entry : mountPoint.second.Vector())
{
CStopWatch timer;
logGlobal->debugStream() << "\t\tLoading resource at " << prefix + entry["path"].String();
auto map = genFunctorMap();
auto functor = map.find(entry["type"].String());
if (functor != map.end())
{
functor->second(mountPoint.first, entry);
logGlobal->debugStream() << "Resource loaded in " << timer.getDiff() << " ms.";
}
else
{
logGlobal->errorStream() << "Unknown filesystem format: " << functor->first;
}
}
}
}
std::string ResourceID::getName() const
CFilesystemList * CFilesystemGenerator::getFilesystem()
{
return name;
return filesystem;
}
EResType::Type ResourceID::getType() const
void CFilesystemGenerator::loadDirectory(const std::string &mountPoint, const JsonNode & config)
{
return type;
std::string URI = prefix + config["path"].String();
int depth = 16;
if (!config["depth"].isNull())
depth = config["depth"].Float();
ResourceID resID(URI, EResType::DIRECTORY);
for(auto & loader : CResourceHandler::getInitial()->getResourcesWithName(resID))
{
auto filename = loader->getResourceName(resID);
filesystem->addLoader(new CFilesystemLoader(mountPoint, *filename, depth), false);
}
}
void ResourceID::setName(std::string name)
void CFilesystemGenerator::loadZipArchive(const std::string &mountPoint, const JsonNode & config)
{
this->name = std::move(name);
size_t dotPos = this->name.find_last_of("/.");
if(dotPos != std::string::npos && this->name[dotPos] == '.')
this->name.erase(dotPos);
// strangely enough but this line takes 40-50% of filesystem loading time
boost::to_upper(this->name);
std::string URI = prefix + config["path"].String();
auto filename = CResourceHandler::getInitial()->getResourceName(ResourceID(URI, EResType::ARCHIVE_ZIP));
if (filename)
filesystem->addLoader(new CZipLoader(mountPoint, *filename), false);
}
void ResourceID::setType(EResType::Type type)
template<EResType::Type archiveType>
void CFilesystemGenerator::loadArchive(const std::string &mountPoint, const JsonNode & config)
{
this->type = type;
std::string URI = prefix + config["path"].String();
auto filename = CResourceHandler::getInitial()->getResourceName(ResourceID(URI, archiveType));
if (filename)
filesystem->addLoader(new CArchiveLoader(mountPoint, *filename), false);
}
void CFilesystemGenerator::loadJsonMap(const std::string &mountPoint, const JsonNode & config)
{
std::string URI = prefix + config["path"].String();
auto filename = CResourceHandler::getInitial()->getResourceName(ResourceID(URI, EResType::TEXT));
if (filename)
{
auto configData = CResourceHandler::getInitial()->load(ResourceID(URI, EResType::TEXT))->readAll();
const JsonNode config((char*)configData.first.get(), configData.second);
filesystem->addLoader(new CMappedFileLoader(mountPoint, config), false);
}
}
void CResourceHandler::clear()
@@ -69,90 +116,6 @@ void CResourceHandler::clear()
delete initialLoader;
}
EResType::Type EResTypeHelper::getTypeFromExtension(std::string extension)
{
boost::to_upper(extension);
static const std::map<std::string, EResType::Type> stringToRes =
boost::assign::map_list_of
(".TXT", EResType::TEXT)
(".JSON", EResType::TEXT)
(".DEF", EResType::ANIMATION)
(".MSK", EResType::MASK)
(".MSG", EResType::MASK)
(".H3C", EResType::CAMPAIGN)
(".H3M", EResType::MAP)
(".FNT", EResType::BMP_FONT)
(".TTF", EResType::TTF_FONT)
(".BMP", EResType::IMAGE)
(".JPG", EResType::IMAGE)
(".PCX", EResType::IMAGE)
(".PNG", EResType::IMAGE)
(".TGA", EResType::IMAGE)
(".WAV", EResType::SOUND)
(".82M", EResType::SOUND)
(".SMK", EResType::VIDEO)
(".BIK", EResType::VIDEO)
(".MJPG", EResType::VIDEO)
(".MPG", EResType::VIDEO)
(".AVI", EResType::VIDEO)
(".MP3", EResType::MUSIC)
(".OGG", EResType::MUSIC)
(".ZIP", EResType::ARCHIVE_ZIP)
(".LOD", EResType::ARCHIVE_LOD)
(".PAC", EResType::ARCHIVE_LOD)
(".VID", EResType::ARCHIVE_VID)
(".SND", EResType::ARCHIVE_SND)
(".PAL", EResType::PALETTE)
(".VCGM1", EResType::CLIENT_SAVEGAME)
(".VSGM1", EResType::SERVER_SAVEGAME)
(".ERM", EResType::ERM)
(".ERT", EResType::ERT)
(".ERS", EResType::ERS);
auto iter = stringToRes.find(extension);
if (iter == stringToRes.end())
return EResType::OTHER;
return iter->second;
}
std::string EResTypeHelper::getEResTypeAsString(EResType::Type type)
{
#define MAP_ENUM(value) (EResType::value, #value)
static const std::map<EResType::Type, std::string> stringToRes = boost::assign::map_list_of
MAP_ENUM(TEXT)
MAP_ENUM(ANIMATION)
MAP_ENUM(MASK)
MAP_ENUM(CAMPAIGN)
MAP_ENUM(MAP)
MAP_ENUM(BMP_FONT)
MAP_ENUM(TTF_FONT)
MAP_ENUM(IMAGE)
MAP_ENUM(VIDEO)
MAP_ENUM(SOUND)
MAP_ENUM(MUSIC)
MAP_ENUM(ARCHIVE_ZIP)
MAP_ENUM(ARCHIVE_LOD)
MAP_ENUM(ARCHIVE_SND)
MAP_ENUM(ARCHIVE_VID)
MAP_ENUM(PALETTE)
MAP_ENUM(CLIENT_SAVEGAME)
MAP_ENUM(SERVER_SAVEGAME)
MAP_ENUM(DIRECTORY)
MAP_ENUM(ERM)
MAP_ENUM(ERT)
MAP_ENUM(ERS)
MAP_ENUM(OTHER);
#undef MAP_ENUM
auto iter = stringToRes.find(type);
assert(iter != stringToRes.end());
return iter->second;
}
void CResourceHandler::initialize()
{
//recurse only into specific directories
@@ -174,7 +137,6 @@ void CResourceHandler::initialize()
//temporary filesystem that will be used to initialize main one.
//used to solve several case-sensivity issues like Mp3 vs MP3
initialLoader = new CFilesystemList;
resourceLoader = new CFilesystemList;
for (auto & path : VCMIDirs::get().dataPaths())
initialLoader->addLoader(new CFilesystemLoader("", path, 0, true), false);
@@ -188,89 +150,36 @@ void CResourceHandler::initialize()
recurseInDir("MODS", 2); // look for mods. Depth 2 is required for now but won't cause speed issues if no mods present
}
void CResourceHandler::loadDirectory(const std::string &prefix, const std::string &mountPoint, const JsonNode & config)
CFilesystemList * CResourceHandler::get()
{
std::string URI = prefix + config["path"].String();
int depth = 16;
if (!config["depth"].isNull())
depth = config["depth"].Float();
ResourceID resID(URI, EResType::DIRECTORY);
for(auto & loader : initialLoader->getResourcesWithName(resID))
{
auto filename = loader->getResourceName(resID);
resourceLoader->addLoader(new CFilesystemLoader(mountPoint, *filename, depth), false);
}
assert(resourceLoader);
return resourceLoader;
}
void CResourceHandler::loadZipArchive(const std::string &prefix, const std::string &mountPoint, const JsonNode & config)
CFilesystemList * CResourceHandler::getInitial()
{
std::string URI = prefix + config["path"].String();
auto filename = initialLoader->getResourceName(ResourceID(URI, EResType::ARCHIVE_ZIP));
if (filename)
resourceLoader->addLoader(new CZipLoader(mountPoint, *filename), false);
assert(initialLoader);
return initialLoader;
}
void CResourceHandler::loadArchive(const std::string &prefix, const std::string &mountPoint, const JsonNode & config, EResType::Type archiveType)
{
std::string URI = prefix + config["path"].String();
auto filename = initialLoader->getResourceName(ResourceID(URI, archiveType));
if (filename)
resourceLoader->addLoader(new CArchiveLoader(mountPoint, *filename), false);
}
void CResourceHandler::loadJsonMap(const std::string &prefix, const std::string &mountPoint, const JsonNode & config)
{
std::string URI = prefix + config["path"].String();
auto filename = initialLoader->getResourceName(ResourceID(URI, EResType::TEXT));
if (filename)
{
auto configData = initialLoader->load(ResourceID(URI, EResType::TEXT))->readAll();
const JsonNode config((char*)configData.first.get(), configData.second);
resourceLoader->addLoader(new CMappedFileLoader(mountPoint, config), false);
}
}
void CResourceHandler::loadMainFileSystem(const std::string &fsConfigURI)
void CResourceHandler::load(const std::string &fsConfigURI)
{
auto fsConfigData = initialLoader->load(ResourceID(fsConfigURI, EResType::TEXT))->readAll();
const JsonNode fsConfig((char*)fsConfigData.first.get(), fsConfigData.second);
loadModFileSystem("", fsConfig["filesystem"]);
resourceLoader = createFileSystem("", fsConfig["filesystem"]);
// hardcoded system-specific path, may not be inside any of data directories
resourceLoader->addLoader(new CFilesystemLoader("SAVES/", VCMIDirs::get().userSavePath()), true);
resourceLoader->addLoader(new CFilesystemLoader("CONFIG/", VCMIDirs::get().userConfigPath()), true);
}
void CResourceHandler::loadModFileSystem(const std::string & prefix, const JsonNode &fsConfig)
CFilesystemList * CResourceHandler::createFileSystem(const std::string & prefix, const JsonNode &fsConfig)
{
for(auto & mountPoint : fsConfig.Struct())
{
for(auto & entry : mountPoint.second.Vector())
{
CStopWatch timer;
logGlobal->debugStream() << "\t\tLoading resource at " << prefix + entry["path"].String();
//TODO: replace if's with string->functor map?
if (entry["type"].String() == "map")
loadJsonMap(prefix, mountPoint.first, entry);
if (entry["type"].String() == "dir")
loadDirectory(prefix, mountPoint.first, entry);
if (entry["type"].String() == "lod")
loadArchive(prefix, mountPoint.first, entry, EResType::ARCHIVE_LOD);
if (entry["type"].String() == "snd")
loadArchive(prefix, mountPoint.first, entry, EResType::ARCHIVE_SND);
if (entry["type"].String() == "vid")
loadArchive(prefix, mountPoint.first, entry, EResType::ARCHIVE_VID);
if (entry["type"].String() == "zip")
loadZipArchive(prefix, mountPoint.first, entry);
logGlobal->debugStream() << "Resource loaded in " << timer.getDiff() << " ms.";
}
}
CFilesystemGenerator generator(prefix);
generator.loadConfig(fsConfig);
return generator.getFilesystem();
}
std::vector<std::string> CResourceHandler::getAvailableMods()
@@ -306,27 +215,3 @@ std::vector<std::string> CResourceHandler::getAvailableMods()
}
return foundMods;
}
void CResourceHandler::setActiveMods(std::vector<std::string> enabledMods)
{
// default FS config for mods: directory "Content" that acts as H3 root directory
JsonNode defaultFS;
defaultFS[""].Vector().resize(2);
defaultFS[""].Vector()[0]["type"].String() = "zip";
defaultFS[""].Vector()[0]["path"].String() = "/Content.zip";
defaultFS[""].Vector()[1]["type"].String() = "dir";
defaultFS[""].Vector()[1]["path"].String() = "/Content";
for(std::string & modName : enabledMods)
{
ResourceID modConfFile("mods/" + modName + "/mod", EResType::TEXT);
auto fsConfigData = initialLoader->load(modConfFile)->readAll();
const JsonNode fsConfig((char*)fsConfigData.first.get(), fsConfigData.second);
if (!fsConfig["filesystem"].isNull())
loadModFileSystem("mods/" + modName, fsConfig["filesystem"]);
else
loadModFileSystem("mods/" + modName, defaultFS);
}
}

184
lib/filesystem/Filesystem.h
View File

@@ -11,130 +11,43 @@
*/
#include "CInputStream.h"
#include "ISimpleResourceLoader.h"
#include "AdapterLoaders.h"
#include "ResourceID.h"
class CFilesystemList;
class JsonNode;
/**
* Specifies the resource type.
*
* Supported file extensions:
*
* Text: .txt .json
* Animation: .def
* Mask: .msk .msg
* Campaign: .h3c
* Map: .h3m
* Font: .fnt
* Image: .bmp, .jpg, .pcx, .png, .tga
* Sound: .wav .82m
* Video: .smk, .bik .mjpg .mpg
* Music: .mp3, .ogg
* Archive: .lod, .snd, .vid .pac .zip
* Palette: .pal
* Savegame: .v*gm1
*/
namespace EResType
/// Helper class that allows generation of a ISimpleResourceLoader entry out of Json config(s)
class DLL_LINKAGE CFilesystemGenerator
{
enum Type
{
TEXT,
ANIMATION,
MASK,
CAMPAIGN,
MAP,
BMP_FONT,
TTF_FONT,
IMAGE,
VIDEO,
SOUND,
MUSIC,
ARCHIVE_VID,
ARCHIVE_ZIP,
ARCHIVE_SND,
ARCHIVE_LOD,
PALETTE,
CLIENT_SAVEGAME,
SERVER_SAVEGAME,
DIRECTORY,
ERM,
ERT,
ERS,
OTHER
};
}
typedef boost::function<void(const std::string &, const JsonNode &)> TLoadFunctor;
typedef std::map<std::string, TLoadFunctor> TLoadFunctorMap;
/**
* A struct which identifies a resource clearly.
*/
class DLL_LINKAGE ResourceID
{
CFilesystemList * filesystem;
std::string prefix;
template<EResType::Type archiveType>
void loadArchive(const std::string & mountPoint, const JsonNode & config);
void loadDirectory(const std::string & mountPoint, const JsonNode & config);
void loadZipArchive(const std::string & mountPoint, const JsonNode & config);
void loadJsonMap(const std::string & mountPoint, const JsonNode & config);
TLoadFunctorMap genFunctorMap();
public:
/**
* Default c-tor.
*/
ResourceID();
/// prefix = prefix that will be given to file entries in all nodes of this filesystem
CFilesystemGenerator(std::string prefix);
/**
* Ctor. Can be used to create indentifier for resource loading using one parameter
*
* @param fullName The resource name including extension.
*/
explicit ResourceID(std::string fullName);
/// loads configuration from json
/// config - configuration to load, using format of "filesystem" entry in config/filesystem.json
void loadConfig(const JsonNode & config);
/**
* Ctor.
*
* @param name The resource name.
* @param type The resource type. A constant from the enumeration EResType.
*/
ResourceID(std::string name, EResType::Type type);
/**
* Compares this object with a another resource identifier.
*
* @param other The other resource identifier.
* @return Returns true if both are equally, false if not.
*/
inline bool operator==(ResourceID const & other) const
{
return name == other.name && type == other.type;
}
std::string getName() const;
EResType::Type getType() const;
void setName(std::string name);
void setType(EResType::Type type);
private:
/** Specifies the resource name. No extension so .pcx and .png can override each other, always in upper case. **/
std::string name;
/**
* Specifies the resource type. EResType::OTHER if not initialized.
* Required to prevent conflicts if files with different types (e.g. text and image) have the same name.
*/
EResType::Type type;
/// returns generated filesystem
CFilesystemList * getFilesystem();
};
namespace std
{
template <> struct hash<ResourceID>
{
size_t operator()(const ResourceID & resourceIdent) const
{
std::hash<int> intHasher;
std::hash<std::string> stringHasher;
return stringHasher(resourceIdent.getName()) ^ intHasher(static_cast<int>(resourceIdent.getType()));
}
};
}
/**
* This class has static methods for a global resource loader access.
*
* Class is not thread-safe. Make sure nobody is calling getInstance while somebody else is calling initialize.
* Class is not thread-safe.
*/
class DLL_LINKAGE CResourceHandler
{
@@ -146,10 +59,11 @@ public:
*
* @return Returns an instance of resource loader.
*/
static ISimpleResourceLoader * get();
static CFilesystemList * get();
static CFilesystemList * getInitial();
/**
* Creates instance of resource loader.
* Creates instance of initial resource loader.
* Will not fill filesystem with data
*
*/
@@ -163,48 +77,26 @@ public:
static void clear();
/**
* Will load all filesystem data from Json data at this path (config/filesystem.json)
* @param prefix - prefix for all paths in filesystem config
* Will load all filesystem data from Json data at this path (normally - config/filesystem.json)
* @param fsConfigURI - URI from which data will be loaded
*/
static void loadModFileSystem(const std::string &prefix, const JsonNode & fsConfig);
static void loadMainFileSystem(const std::string & fsConfigURI);
static void loadDirectory(const std::string &prefix, const std::string & mountPoint, const JsonNode & config);
static void loadZipArchive(const std::string &prefix, const std::string & mountPoint, const JsonNode & config);
static void loadArchive(const std::string &prefix, const std::string & mountPoint, const JsonNode & config, EResType::Type archiveType);
static void loadJsonMap(const std::string &prefix, const std::string & mountPoint, const JsonNode & config);
static void load(const std::string & fsConfigURI);
/**
* @brief createModFileSystem - creates filesystem out of config file
* @param prefix - prefix for all paths in filesystem config
* @param fsConfig - configuration to load
* @return generated filesystem that contains all config entries
*/
static CFilesystemList * createFileSystem(const std::string &prefix, const JsonNode & fsConfig);
/**
* Checks all subfolders of MODS directory for presence of mods
* If this directory has mod.json file it will be added to resources
*/
static std::vector<std::string> getAvailableMods();
static void setActiveMods(std::vector<std::string> enabledMods); //WARNING: not reentrable. Do not call it twice!!!
private:
/** Instance of resource loader */
static CFilesystemList * resourceLoader;
static CFilesystemList * initialLoader;
};
/**
* A helper class which provides a functionality to convert extension strings to EResTypes.
*/
class DLL_LINKAGE EResTypeHelper
{
public:
/**
* Converts a extension string to a EResType enum object.
*
* @param extension The extension string e.g. .BMP, .PNG
* @return Returns a EResType enum object
*/
static EResType::Type getTypeFromExtension(std::string extension);
/**
* Gets the EResType as a string representation.
*
* @param type the EResType
* @return the type as a string representation
*/
static std::string getEResTypeAsString(EResType::Type type);
};

134
lib/filesystem/ResourceID.cpp Normal file
View File

@@ -0,0 +1,134 @@
#include "StdInc.h"
#include "ResourceID.h"
#include "CFileInfo.h"
ResourceID::ResourceID()
:type(EResType::OTHER)
{
}
ResourceID::ResourceID(std::string name)
{
CFileInfo info(std::move(name));
setName(info.getStem());
setType(info.getType());
}
ResourceID::ResourceID(std::string name, EResType::Type type)
{
setName(std::move(name));
setType(type);
}
std::string ResourceID::getName() const
{
return name;
}
EResType::Type ResourceID::getType() const
{
return type;
}
void ResourceID::setName(std::string name)
{
this->name = std::move(name);
size_t dotPos = this->name.find_last_of("/.");
if(dotPos != std::string::npos && this->name[dotPos] == '.')
this->name.erase(dotPos);
// strangely enough but this line takes 40-50% of filesystem loading time
boost::to_upper(this->name);
}
void ResourceID::setType(EResType::Type type)
{
this->type = type;
}
EResType::Type EResTypeHelper::getTypeFromExtension(std::string extension)
{
boost::to_upper(extension);
static const std::map<std::string, EResType::Type> stringToRes =
boost::assign::map_list_of
(".TXT", EResType::TEXT)
(".JSON", EResType::TEXT)
(".DEF", EResType::ANIMATION)
(".MSK", EResType::MASK)
(".MSG", EResType::MASK)
(".H3C", EResType::CAMPAIGN)
(".H3M", EResType::MAP)
(".FNT", EResType::BMP_FONT)
(".TTF", EResType::TTF_FONT)
(".BMP", EResType::IMAGE)
(".JPG", EResType::IMAGE)
(".PCX", EResType::IMAGE)
(".PNG", EResType::IMAGE)
(".TGA", EResType::IMAGE)
(".WAV", EResType::SOUND)
(".82M", EResType::SOUND)
(".SMK", EResType::VIDEO)
(".BIK", EResType::VIDEO)
(".MJPG", EResType::VIDEO)
(".MPG", EResType::VIDEO)
(".AVI", EResType::VIDEO)
(".MP3", EResType::MUSIC)
(".OGG", EResType::MUSIC)
(".ZIP", EResType::ARCHIVE_ZIP)
(".LOD", EResType::ARCHIVE_LOD)
(".PAC", EResType::ARCHIVE_LOD)
(".VID", EResType::ARCHIVE_VID)
(".SND", EResType::ARCHIVE_SND)
(".PAL", EResType::PALETTE)
(".VCGM1", EResType::CLIENT_SAVEGAME)
(".VSGM1", EResType::SERVER_SAVEGAME)
(".ERM", EResType::ERM)
(".ERT", EResType::ERT)
(".ERS", EResType::ERS);
auto iter = stringToRes.find(extension);
if (iter == stringToRes.end())
return EResType::OTHER;
return iter->second;
}
std::string EResTypeHelper::getEResTypeAsString(EResType::Type type)
{
#define MAP_ENUM(value) (EResType::value, #value)
static const std::map<EResType::Type, std::string> stringToRes = boost::assign::map_list_of
MAP_ENUM(TEXT)
MAP_ENUM(ANIMATION)
MAP_ENUM(MASK)
MAP_ENUM(CAMPAIGN)
MAP_ENUM(MAP)
MAP_ENUM(BMP_FONT)
MAP_ENUM(TTF_FONT)
MAP_ENUM(IMAGE)
MAP_ENUM(VIDEO)
MAP_ENUM(SOUND)
MAP_ENUM(MUSIC)
MAP_ENUM(ARCHIVE_ZIP)
MAP_ENUM(ARCHIVE_LOD)
MAP_ENUM(ARCHIVE_SND)
MAP_ENUM(ARCHIVE_VID)
MAP_ENUM(PALETTE)
MAP_ENUM(CLIENT_SAVEGAME)
MAP_ENUM(SERVER_SAVEGAME)
MAP_ENUM(DIRECTORY)
MAP_ENUM(ERM)
MAP_ENUM(ERT)
MAP_ENUM(ERS)
MAP_ENUM(OTHER);
#undef MAP_ENUM
auto iter = stringToRes.find(type);
assert(iter != stringToRes.end());
return iter->second;
}

150
lib/filesystem/ResourceID.h Normal file
View File

@@ -0,0 +1,150 @@
#pragma once
/*
* ResourceID.h, part of VCMI engine
*
* Authors: listed in file AUTHORS in main folder
*
* License: GNU General Public License v2.0 or later
* Full text of license available in license.txt file, in main folder
*
*/
/**
* Specifies the resource type.
*
* Supported file extensions:
*
* Text: .txt .json
* Animation: .def
* Mask: .msk .msg
* Campaign: .h3c
* Map: .h3m
* Font: .fnt
* Image: .bmp, .jpg, .pcx, .png, .tga
* Sound: .wav .82m
* Video: .smk, .bik .mjpg .mpg
* Music: .mp3, .ogg
* Archive: .lod, .snd, .vid .pac .zip
* Palette: .pal
* Savegame: .v*gm1
*/
namespace EResType
{
enum Type
{
TEXT,
ANIMATION,
MASK,
CAMPAIGN,
MAP,
BMP_FONT,
TTF_FONT,
IMAGE,
VIDEO,
SOUND,
MUSIC,
ARCHIVE_VID,
ARCHIVE_ZIP,
ARCHIVE_SND,
ARCHIVE_LOD,
PALETTE,
CLIENT_SAVEGAME,
SERVER_SAVEGAME,
DIRECTORY,
ERM,
ERT,
ERS,
OTHER
};
}
/**
* A struct which identifies a resource clearly.
*/
class DLL_LINKAGE ResourceID
{
public:
/**
* Default c-tor.
*/
ResourceID();
/**
* Ctor. Can be used to create indentifier for resource loading using one parameter
*
* @param fullName The resource name including extension.
*/
explicit ResourceID(std::string fullName);
/**
* Ctor.
*
* @param name The resource name.
* @param type The resource type. A constant from the enumeration EResType.
*/
ResourceID(std::string name, EResType::Type type);
/**
* Compares this object with a another resource identifier.
*
* @param other The other resource identifier.
* @return Returns true if both are equally, false if not.
*/
inline bool operator==(ResourceID const & other) const
{
return name == other.name && type == other.type;
}
std::string getName() const;
EResType::Type getType() const;
void setName(std::string name);
void setType(EResType::Type type);
private:
/** Specifies the resource name. No extension so .pcx and .png can override each other, always in upper case. **/
std::string name;
/**
* Specifies the resource type. EResType::OTHER if not initialized.
* Required to prevent conflicts if files with different types (e.g. text and image) have the same name.
*/
EResType::Type type;
};
namespace std
{
template <> struct hash<ResourceID>
{
size_t operator()(const ResourceID & resourceIdent) const
{
std::hash<int> intHasher;
std::hash<std::string> stringHasher;
return stringHasher(resourceIdent.getName()) ^ intHasher(static_cast<int>(resourceIdent.getType()));
}
};
}
/**
* A helper class which provides a functionality to convert extension strings to EResTypes.
*/
class DLL_LINKAGE EResTypeHelper
{
public:
/**
* Converts a extension string to a EResType enum object.
*
* @param extension The extension string e.g. .BMP, .PNG
* @return Returns a EResType enum object
*/
static EResType::Type getTypeFromExtension(std::string extension);
/**
* Gets the EResType as a string representation.
*
* @param type the EResType
* @return the type as a string representation
*/
static std::string getEResTypeAsString(EResType::Type type);
};