From 9237e6d97db8081f1973a02dfa9300c67b7c619f Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Sat, 26 Oct 2013 19:33:34 +0000 Subject: [PATCH] Improved json validation - split JsonNode.cpp into JsonNode and JsonDetail files - validation should be notably faster (at least 10% faster loading) - support for "format" field, allows checking existance of files. - minor fixes in schemas - msk/msg files are now optional --- client/Graphics.cpp | 5 - config/creatures/dungeon.json | 1 - config/creatures/special.json | 2 +- config/schemas/artifact.json | 9 +- config/schemas/creature.json | 33 +- config/schemas/faction.json | 33 +- config/schemas/hero.json | 18 +- config/schemas/heroClass.json | 12 +- config/schemas/mod.json | 10 +- config/schemas/townStructure.json | 9 +- lib/CCreatureHandler.cpp | 1 - lib/CDefObjInfoHandler.cpp | 25 +- lib/CMakeLists.txt | 1 + lib/JsonDetail.cpp | 1085 +++++++++++++++++++++++++++++ lib/JsonDetail.h | 123 ++++ lib/JsonNode.cpp | 877 +---------------------- lib/JsonNode.h | 121 ---- 17 files changed, 1313 insertions(+), 1052 deletions(-) create mode 100644 lib/JsonDetail.cpp create mode 100644 lib/JsonDetail.h diff --git a/client/Graphics.cpp b/client/Graphics.cpp index c160de924..71ef627f4 100644 --- a/client/Graphics.cpp +++ b/client/Graphics.cpp @@ -365,11 +365,6 @@ void Graphics::addImageListEntry(size_t index, std::string listName, std::string { if (!imageName.empty()) { - ResourceID resID("SPRITES/" + imageName, EResType::IMAGE); - if (!CResourceHandler::get()->existsResource(resID) && // file not found - imageName.find(':') == std::string::npos) // and entry does not refers to frame in def file - logGlobal->errorStream() << "Required image " << "SPRITES/" << imageName << " is missing!"; - JsonNode entry; entry["frame"].Float() = index; entry["file"].String() = imageName; diff --git a/config/creatures/dungeon.json b/config/creatures/dungeon.json index 301f6f3c6..56d9a2809 100644 --- a/config/creatures/dungeon.json +++ b/config/creatures/dungeon.json @@ -87,7 +87,6 @@ "defend": "HARPDFND.wav", "killed": "HARPKILL.wav", "move": "HARPMOVE.wav", - "shoot": "silence", "wince": "HARPWNCE.wav" } }, diff --git a/config/creatures/special.json b/config/creatures/special.json index 452aa9bff..293e9b99d 100644 --- a/config/creatures/special.json +++ b/config/creatures/special.json @@ -115,7 +115,7 @@ "abilities": { "shooter" : { "type" : "SHOOTER" } }, "graphics" : { - "animation" : "This should never be used" + "animation": "CLCBOW.DEF" // needed to pass validation, never used }, "sound": {} } diff --git a/config/schemas/artifact.json b/config/schemas/artifact.json index f608e291f..b42b3278a 100644 --- a/config/schemas/artifact.json +++ b/config/schemas/artifact.json @@ -50,15 +50,18 @@ "properties":{ "image": { "type":"string", - "description": "Base image for this artifact, used for example in hero screen" + "description": "Base image for this artifact, used for example in hero screen", + "format" : "imageFile" }, "large": { "type":"string", - "description": "Large image, used for drag-and-drop and popup messages" + "description": "Large image, used for drag-and-drop and popup messages", + "format" : "imageFile" }, "map": { "type":"string", - "description": ".def file for adventure map" + "description": ".def file for adventure map", + "format" : "defFile" } } }, diff --git a/config/schemas/creature.json b/config/schemas/creature.json index 4b68bec26..4a6982c6a 100644 --- a/config/schemas/creature.json +++ b/config/schemas/creature.json @@ -153,7 +153,7 @@ "minItems" : 10, "maxItems" : 10, "description": "Strength of the bonus", - "anyof" : [ + "anyOf" : [ { "items": { "type" : "number" } }, { "items": { "type" : "boolean" } } ] @@ -191,20 +191,24 @@ }, "iconLarge": { "type":"string", - "description": "Large icon for this creature, used for example in town screen" + "description": "Large icon for this creature, used for example in town screen", + "format" : "imageFile" }, "iconSmall": { "type":"string", - "description": "Small icon for this creature, used for example in exchange screen" + "description": "Small icon for this creature, used for example in exchange screen", + "format" : "imageFile" }, "map": { "type":"string", - "description": ".def file with animation of this creature on adventure map" + "description": ".def file with animation of this creature on adventure map", + "format" : "defFile" }, "animation": { "type":"string", - "description": ".def file with animation of this creature in battles" + "description": ".def file with animation of this creature in battles", + "format" : "defFile" }, "missile": { "type":"object", @@ -214,7 +218,8 @@ "properties":{ "projectile": { "type":"string", - "description": "Path to projectile animation" + "description": "Path to projectile animation", + "format" : "defFile" }, "frameAngles": { "type":"array", @@ -261,14 +266,14 @@ "additionalProperties" : false, "description": "Various sound files associated with this creature", "properties":{ - "attack": { "type":"string" }, - "defend": { "type":"string" }, - "killed": { "type":"string" }, - "startMoving": { "type":"string" }, - "endMoving": { "type":"string" }, - "move": { "type":"string" }, - "shoot": { "type":"string" }, - "wince": { "type":"string" } + "attack": { "type":"string", "format" : "soundFile" }, + "defend": { "type":"string", "format" : "soundFile" }, + "killed": { "type":"string", "format" : "soundFile" }, + "startMoving": { "type":"string", "format" : "soundFile" }, + "endMoving": { "type":"string", "format" : "soundFile" }, + "move": { "type":"string", "format" : "soundFile" }, + "shoot": { "type":"string", "format" : "soundFile" }, + "wince": { "type":"string", "format" : "soundFile" } } } } diff --git a/config/schemas/faction.json b/config/schemas/faction.json index 335e65a32..8ec9402f2 100644 --- a/config/schemas/faction.json +++ b/config/schemas/faction.json @@ -6,8 +6,8 @@ "additionalProperties" : false, "required" : [ "small", "large" ], "properties" : { - "small" : { "type" : "string" }, - "large" : { "type" : "string" } + "small" : { "type" : "string", "format" : "imageFile" }, + "large" : { "type" : "string", "format" : "imageFile" } } }, "townIconPair" : { @@ -63,11 +63,13 @@ "properties":{ "120px": { "type":"string", - "description": "Version that is 120 pixels in height" + "description": "Version that is 120 pixels in height", + "format" : "imageFile" }, "130px": { "type":"string", - "description": "Version that is 130 pixels in height" + "description": "Version that is 130 pixels in height", + "format" : "imageFile" } } }, @@ -120,15 +122,18 @@ "properties":{ "capitol": { "type":"string", - "description": "Town with capitol" + "description": "Town with capitol", + "format" : "defFile" }, "castle": { "type":"string", - "description": "Town with built fort" + "description": "Town with built fort", + "format" : "defFile" }, "village": { "type":"string", - "description": "Village without built fort" + "description": "Village without built fort", + "format" : "defFile" }, "dwellings" : { "type" : "array", @@ -141,7 +146,7 @@ "required" : [ "name", "graphics" ], "properties" : { "name": { "type":"string" }, - "graphics": { "type":"string" } + "graphics": { "type":"string", "format" : "defFile" } } } } @@ -169,7 +174,8 @@ }, "buildingsIcons": { "type" : "string", - "description": "Path to .def file with building icons" + "description": "Path to .def file with building icons", + "format" : "animationFile" }, "buildings": { "type" : "object", @@ -193,7 +199,8 @@ }, "hallBackground": { "type":"string", - "description": "background image for town hall" + "description": "background image for town hall", + "format" : "imageFile" }, "hallSlots": { "type":"array", @@ -243,7 +250,8 @@ }, "musicTheme": { "type":"string", - "description": "Path to town music theme" + "description": "Path to town music theme", + "format" : "musicFile" }, "siege": { "$ref" : "vcmi:townSiege" @@ -256,7 +264,8 @@ }, "townBackground": { "type":"string", - "description": "Background for town screen" + "description": "Background for town screen", + "format" : "imageFile" }, "primaryResource": { "type":"string", diff --git a/config/schemas/hero.json b/config/schemas/hero.json index cd742511f..923ed9fbe 100644 --- a/config/schemas/hero.json +++ b/config/schemas/hero.json @@ -32,11 +32,13 @@ }, "max": { "type":"number", - "description": "max" + "description": "max", + "minimum" : 1 }, "min": { "type":"number", - "description": "min" + "description": "min", + "minimum" : 1 } } } @@ -65,19 +67,23 @@ "properties":{ "large": { "type":"string", - "description": "Large version of portrait for use in hero screen" + "description": "Large version of portrait for use in hero screen", + "format" : "imageFile" }, "small": { "type":"string", - "description": "Small version of portrait for use on adventure map" + "description": "Small version of portrait for use on adventure map", + "format" : "imageFile" }, "specialtyLarge": { "type":"string", - "description": "Large image of hero specilty, used in hero screen" + "description": "Large image of hero specilty, used in hero screen", + "format" : "imageFile" }, "specialtySmall": { "type":"string", - "description": "Small image of hero specialty for use in exchange screen" + "description": "Small image of hero specialty for use in exchange screen", + "format" : "imageFile" } } }, diff --git a/config/schemas/heroClass.json b/config/schemas/heroClass.json index 12c62ae9d..d6558610e 100644 --- a/config/schemas/heroClass.json +++ b/config/schemas/heroClass.json @@ -24,11 +24,13 @@ "properties":{ "female": { "type":"string", - "description": "Female version" + "description": "Female version", + "format" : "defFile" }, "male": { "type":"string", - "description": "Male version" + "description": "Male version", + "format" : "defFile" } } }, @@ -40,11 +42,13 @@ "properties":{ "female": { "type":"string", - "description": "Female version. Warning: not implemented!" + "description": "Female version. Warning: not implemented!", + "format" : "defFile" }, "male": { "type":"string", - "description": "Male version" + "description": "Male version", + "format" : "defFile" } } } diff --git a/config/schemas/mod.json b/config/schemas/mod.json index 4dd1649a7..978d7379f 100644 --- a/config/schemas/mod.json +++ b/config/schemas/mod.json @@ -50,27 +50,27 @@ "artifacts": { "type":"array", "description": "List of configuration files for artifacts", - "items": { "type":"string" } + "items": { "type":"string", "format" : "textFile" } }, "creatures": { "type":"array", "description": "List of configuration files for creatures", - "items": { "type":"string" } + "items": { "type":"string", "format" : "textFile" } }, "factions": { "type":"array", "description": "List of configuration files for towns/factions", - "items": { "type":"string" } + "items": { "type":"string", "format" : "textFile" } }, "heroClasses": { "type":"array", "description": "List of configuration files for hero classes", - "items": { "type":"string" } + "items": { "type":"string", "format" : "textFile" } }, "heroes": { "type":"array", "description": "List of configuration files for heroes", - "items": { "type":"string" } + "items": { "type":"string", "format" : "textFile" } }, "filesystem": { diff --git a/config/schemas/townStructure.json b/config/schemas/townStructure.json index f381cb127..2e2e9f2f0 100644 --- a/config/schemas/townStructure.json +++ b/config/schemas/townStructure.json @@ -8,15 +8,18 @@ "properties":{ "animation": { "type":"string", - "description" : "Main animation file for this building" + "description" : "Main animation file for this building", + "format" : "animationFile" }, "area": { "type":"string", - "description" : "Area that indicate when building is selected. Must be 8-bit image" + "description" : "Area that indicate when building is selected. Must be 8-bit image", + "format" : "imageFile" }, "border": { "type":"string", - "description" : "Golden border around building, displayed when building is selected" + "description" : "Golden border around building, displayed when building is selected", + "format" : "imageFile" }, "builds": { "type":"number", diff --git a/lib/CCreatureHandler.cpp b/lib/CCreatureHandler.cpp index 439767d6a..3297af8ea 100644 --- a/lib/CCreatureHandler.cpp +++ b/lib/CCreatureHandler.cpp @@ -525,7 +525,6 @@ void CCreatureHandler::loadAnimationInfo(std::vector &h3Data) void CCreatureHandler::loadUnitAnimInfo(JsonNode & graphics, CLegacyConfigParser & parser) { - graphics["map"].String(); //create empty string. Real value will be loaded from H3 txt's graphics["timeBetweenFidgets"].Float() = parser.readNumber(); JsonNode & animationTime = graphics["animationTime"]; diff --git a/lib/CDefObjInfoHandler.cpp b/lib/CDefObjInfoHandler.cpp index ec6f17836..b1d9d3108 100644 --- a/lib/CDefObjInfoHandler.cpp +++ b/lib/CDefObjInfoHandler.cpp @@ -34,15 +34,26 @@ CGDefInfo::CGDefInfo() void CGDefInfo::fetchInfoFromMSK() { + ResourceID resID("SPRITES/" + name, EResType::MASK); - auto msk = CResourceHandler::get()->load(ResourceID(std::string("SPRITES/") + name, EResType::MASK))->readAll(); - - width = msk.first.get()[0]; - height = msk.first.get()[1]; - for(int i=0; i<6; ++i) + if (CResourceHandler::get()->existsResource(resID)) { - coverageMap[i] = msk.first.get()[i+2]; - shadowCoverage[i] = msk.first.get()[i+8]; + auto msk = CResourceHandler::get()->load(resID)->readAll(); + + width = msk.first.get()[0]; + height = msk.first.get()[1]; + for(int i=0; i<6; ++i) + { + coverageMap[i] = msk.first.get()[i+2]; + shadowCoverage[i] = msk.first.get()[i+8]; + } + } + else + { + //maximum possible size of H3 object + //TODO: remove hardcode and move this data into modding system + width = 8; + height = 6; } } diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 9a02628de..1d0f5331a 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -60,6 +60,7 @@ set(lib_SRCS GameConstants.cpp HeroBonus.cpp IGameCallback.cpp + JsonDetail.cpp JsonNode.cpp NetPacksLib.cpp ResourceSet.cpp diff --git a/lib/JsonDetail.cpp b/lib/JsonDetail.cpp new file mode 100644 index 000000000..5b9a7bc7d --- /dev/null +++ b/lib/JsonDetail.cpp @@ -0,0 +1,1085 @@ +/* + * JsonDetail.cpp, part of VCMI engine + * + * Authors: listed in file AUTHORS in main folder + * + * License: GNU General Public License v2.0 or later + * Full text of license available in license.txt file, in main folder + * + */ + +#include "StdInc.h" +#include "JsonDetail.h" + +#include "CGeneralTextHandler.h" +#include "filesystem/Filesystem.h" +#include "ScopeGuard.h" + +static const JsonNode nullNode; + +template +void JsonWriter::writeContainer(Iterator begin, Iterator end) +{ + if (begin == end) + return; + + prefix += '\t'; + + writeEntry(begin++); + + while (begin != end) + { + out<<",\n"; + writeEntry(begin++); + } + + out<<"\n"; + prefix.resize(prefix.size()-1); +} + +void JsonWriter::writeEntry(JsonMap::const_iterator entry) +{ + out << prefix; + writeString(entry->first); + out << " : "; + writeNode(entry->second); +} + +void JsonWriter::writeEntry(JsonVector::const_iterator entry) +{ + out << prefix; + writeNode(*entry); +} + +void JsonWriter::writeString(const std::string &string) +{ + static const std::string escaped = "\"\\\b\f\n\r\t"; + + out <<'\"'; + size_t pos=0, start=0; + for (; poswarnStream()<<"File " << fileName << " is not a valid JSON file!"; + logGlobal->warnStream()<= '0' && input[pos] <= '9') + return extractFloat(node); + return error("Value expected!"); + } + } +} + +bool JsonParser::extractWhitespace(bool verbose) +{ + while (true) + { + while (pos < input.size() && (ui8)input[pos] <= ' ') + { + if (input[pos] == '\n') + { + lineCount++; + lineStart = pos+1; + } + pos++; + } + if (pos >= input.size() || input[pos] != '/') + break; + + pos++; + if (pos == input.size()) + break; + if (input[pos] == '/') + pos++; + else + error("Comments must consist from two slashes!", true); + + while (pos < input.size() && input[pos] != '\n') + pos++; + } + + if (pos >= input.size() && verbose) + return error("Unexpected end of file!"); + return true; +} + +bool JsonParser::extractEscaping(std::string &str) +{ + switch(input[pos]) + { + break; case '\"': str += '\"'; + break; case '\\': str += '\\'; + break; case 'b': str += '\b'; + break; case 'f': str += '\f'; + break; case 'n': str += '\n'; + break; case 'r': str += '\r'; + break; case 't': str += '\t'; + break; default: return error("Unknown escape sequence!", true); + }; + return true; +} + +bool JsonParser::extractString(std::string &str) +{ + if (input[pos] != '\"') + return error("String expected!"); + pos++; + + size_t first = pos; + + while (pos != input.size()) + { + if (input[pos] == '\"') // Correct end of string + { + str.append( &input[first], pos-first); + pos++; + return true; + } + if (input[pos] == '\\') // Escaping + { + str.append( &input[first], pos-first); + pos++; + if (pos == input.size()) + break; + extractEscaping(str); + first = pos + 1; + } + if (input[pos] == '\n') // end-of-line + { + str.append( &input[first], pos-first); + return error("Closing quote not found!", true); + } + if ((unsigned char)(input[pos]) < ' ') // control character + { + str.append( &input[first], pos-first); + first = pos+1; + error("Illegal character in the string!", true); + } + pos++; + } + return error("Unterminated string!"); +} + +bool JsonParser::extractString(JsonNode &node) +{ + std::string str; + if (!extractString(str)) + return false; + + node.setType(JsonNode::DATA_STRING); + node.String() = str; + return true; +} + +bool JsonParser::extractLiteral(const std::string &literal) +{ + if (literal.compare(0, literal.size(), &input[pos], literal.size()) != 0) + { + while (pos < input.size() && ((input[pos]>'a' && input[pos]<'z') + || (input[pos]>'A' && input[pos]<'Z'))) + pos++; + return error("Unknown literal found", true); + } + + pos += literal.size(); + return true; +} + +bool JsonParser::extractNull(JsonNode &node) +{ + if (!extractLiteral("null")) + return false; + + node.clear(); + return true; +} + +bool JsonParser::extractTrue(JsonNode &node) +{ + if (!extractLiteral("true")) + return false; + + node.Bool() = true; + return true; +} + +bool JsonParser::extractFalse(JsonNode &node) +{ + if (!extractLiteral("false")) + return false; + + node.Bool() = false; + return true; +} + +bool JsonParser::extractStruct(JsonNode &node) +{ + node.setType(JsonNode::DATA_STRUCT); + pos++; + + if (!extractWhitespace()) + return false; + + //Empty struct found + if (input[pos] == '}') + { + pos++; + return true; + } + + while (true) + { + if (!extractWhitespace()) + return false; + + std::string key; + if (!extractString(key)) + return false; + + if (node.Struct().find(key) != node.Struct().end()) + error("Dublicated element encountered!", true); + + if (!extractSeparator()) + return false; + + if (!extractElement(node.Struct()[key], '}')) + return false; + + if (input[pos] == '}') + { + pos++; + return true; + } + } +} + +bool JsonParser::extractArray(JsonNode &node) +{ + pos++; + node.setType(JsonNode::DATA_VECTOR); + + if (!extractWhitespace()) + return false; + + //Empty array found + if (input[pos] == ']') + { + pos++; + return true; + } + + while (true) + { + //NOTE: currently 50% of time is this vector resizing. + //May be useful to use list during parsing and then swap() all items to vector + node.Vector().resize(node.Vector().size()+1); + + if (!extractElement(node.Vector().back(), ']')) + return false; + + if (input[pos] == ']') + { + pos++; + return true; + } + } +} + +bool JsonParser::extractElement(JsonNode &node, char terminator) +{ + if (!extractValue(node)) + return false; + + if (!extractWhitespace()) + return false; + + bool comma = (input[pos] == ','); + if (comma ) + { + pos++; + if (!extractWhitespace()) + return false; + } + + if (input[pos] == terminator) + { + //FIXME: MOD COMPATIBILITY: Too many of these right now, re-enable later + //if (comma) + //error("Extra comma found!", true); + return true; + } + + if (!comma) + error("Comma expected!", true); + + return true; +} + +bool JsonParser::extractFloat(JsonNode &node) +{ + assert(input[pos] == '-' || (input[pos] >= '0' && input[pos] <= '9')); + bool negative=false; + double result=0; + + if (input[pos] == '-') + { + pos++; + negative = true; + } + + if (input[pos] < '0' || input[pos] > '9') + return error("Number expected!"); + + //Extract integer part + while (input[pos] >= '0' && input[pos] <= '9') + { + result = result*10+(input[pos]-'0'); + pos++; + } + + if (input[pos] == '.') + { + //extract fractional part + pos++; + double fractMult = 0.1; + if (input[pos] < '0' || input[pos] > '9') + return error("Decimal part expected!"); + + while (input[pos] >= '0' && input[pos] <= '9') + { + result = result + fractMult*(input[pos]-'0'); + fractMult /= 10; + pos++; + } + } + //TODO: exponential part + if (negative) + result = -result; + + node.setType(JsonNode::DATA_FLOAT); + node.Float() = result; + return true; +} + +bool JsonParser::error(const std::string &message, bool warning) +{ + std::ostringstream stream; + std::string type(warning?" warning: ":" error: "); + + stream << "At line " << lineCount << ", position "< stringToType = + boost::assign::map_list_of + ("null", JsonNode::DATA_NULL) ("boolean", JsonNode::DATA_BOOL) + ("number", JsonNode::DATA_FLOAT) ("string", JsonNode::DATA_STRING) + ("array", JsonNode::DATA_VECTOR) ("object", JsonNode::DATA_STRUCT); + +namespace +{ + namespace Common + { + std::string emptyCheck(Validation::ValidationData &, const JsonNode &, const JsonNode &, const JsonNode &) + { + // check is not needed - e.g. incorporated into another check + return ""; + } + + std::string notImplementedCheck(Validation::ValidationData &, const JsonNode &, const JsonNode &, const JsonNode &) + { + return "Not implemented entry in schema"; + } + + std::string schemaListCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data, + std::string errorMsg, std::function isValid) + { + std::string errors = "\n"; + size_t result = 0; + + for(auto & schemaEntry : schema.Vector()) + { + std::string error = check(schemaEntry, data, validator); + if (error.empty()) + { + result++; + } + else + { + errors += error; + errors += "\n"; + } + } + if (isValid(result)) + return ""; + else + return validator.makeErrorMessage(errorMsg) + errors; + } + + std::string allOfCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + return schemaListCheck(validator, baseSchema, schema, data, "Failed to pass all schemas", [&](size_t count) + { + return count == schema.Vector().size(); + }); + } + + std::string anyOfCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + return schemaListCheck(validator, baseSchema, schema, data, "Failed to pass any schema", [&](size_t count) + { + return count > 0; + }); + } + + std::string oneOfCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + return schemaListCheck(validator, baseSchema, schema, data, "Failed to pass exactly one schema", [&](size_t count) + { + return count == 1; + }); + } + + std::string notCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (check(schema, data, validator).empty()) + return validator.makeErrorMessage("Successful validation against negative check"); + return ""; + } + + std::string enumCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + for(auto & enumEntry : schema.Vector()) + { + if (data == enumEntry) + return ""; + } + return validator.makeErrorMessage("Key must have one of predefined values"); + } + + std::string typeCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + JsonNode::JsonType type = stringToType.find(schema.String())->second; + if(type != data.getType() && data.getType() != JsonNode::DATA_NULL) + return validator.makeErrorMessage("Type mismatch!"); + return ""; + } + + std::string refCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string URI = schema.String(); + //node must be validated using schema pointed by this reference and not by data here + //Local reference. Turn it into more easy to handle remote ref + if (boost::algorithm::starts_with(URI, "#")) + URI = validator.usedSchemas.back() + URI; + + return check(JsonUtils::getSchema(URI), data, validator); + } + + std::string formatCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + auto formats = Validation::getKnownFormats(); + std::string errors; + auto checker = formats.find(schema.String()); + if (checker != formats.end()) + { + std::string result = checker->second(data); + if (!result.empty()) + errors += validator.makeErrorMessage(result); + } + else + errors += validator.makeErrorMessage("Unknown format: " + schema.String()); + return errors; + } + } + + namespace String + { + std::string maxLengthCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (data.String().size() > schema.Float()) + return validator.makeErrorMessage("String too long"); + return ""; + } + + std::string minLengthCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (data.String().size() < schema.Float()) + return validator.makeErrorMessage("String too short"); + return ""; + } + } + + namespace Number + { + + std::string maximumCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (baseSchema["exclusiveMaximum"].Bool()) + { + if (data.Float() >= schema.Float()) + return validator.makeErrorMessage("Value is too large"); + } + else + { + if (data.Float() > schema.Float()) + return validator.makeErrorMessage("Value is too large"); + } + return ""; + } + + std::string minimumCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (baseSchema["exclusiveMinimum"].Bool()) + { + if (data.Float() <= schema.Float()) + return validator.makeErrorMessage("Value is too small"); + } + else + { + if (data.Float() < schema.Float()) + return validator.makeErrorMessage("Value is too small"); + } + return ""; + } + + std::string multipleOfCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + double result = data.Float() / schema.Float(); + if (floor(result) != result) + return validator.makeErrorMessage("Value is not divisible"); + return ""; + } + } + + namespace Vector + { + std::string itemEntryCheck(Validation::ValidationData & validator, const JsonVector items, const JsonNode & schema, size_t index) + { + validator.currentPath.push_back(JsonNode()); + validator.currentPath.back().Float() = index; + auto onExit = vstd::makeScopeGuard([&] + { + validator.currentPath.pop_back(); + }); + + if (!schema.isNull()) + return check(schema, items[index], validator); + return ""; + } + + std::string itemsCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + for (size_t i=0; i i) + errors += itemEntryCheck(validator, data.Vector(), schema.Vector()[i], i); + } + else + { + errors += itemEntryCheck(validator, data.Vector(), schema, i); + } + } + return errors; + } + + std::string additionalItemsCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + // "items" is struct or empty (defaults to empty struct) - validation always successfull + const JsonNode & items = baseSchema["items"]; + if (items.getType() != JsonNode::DATA_VECTOR) + return ""; + + for (size_t i=items.Vector().size(); i schema.Float()) + return validator.makeErrorMessage("Too many items in the list!"); + return ""; + } + + std::string uniqueItemsCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (schema.Bool()) + { + for (auto itA = schema.Vector().begin(); itA != schema.Vector().end(); itA++) + { + auto itB = itA; + while (++itB != schema.Vector().end()) + { + if (*itA == *itB) + return validator.makeErrorMessage("List must consist from unique items"); + } + } + } + return ""; + } + } + + namespace Struct + { + std::string maxPropertiesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (data.Struct().size() > schema.Float()) + return validator.makeErrorMessage("Too many items in the list!"); + return ""; + } + + std::string minPropertiesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + if (data.Struct().size() < schema.Float()) + return validator.makeErrorMessage("Too few items in the list"); + return ""; + } + + std::string uniquePropertiesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + for (auto itA = data.Struct().begin(); itA != data.Struct().end(); itA++) + { + auto itB = itA; + while (++itB != data.Struct().end()) + { + if (itA->second == itB->second) + return validator.makeErrorMessage("List must consist from unique items"); + } + } + return ""; + } + + std::string requiredCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + for(auto & required : schema.Vector()) + { + if (data[required.String()].isNull()) + errors += validator.makeErrorMessage("Required entry " + required.String() + " is missing"); + } + return errors; + } + + std::string dependenciesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + for(auto & deps : schema.Struct()) + { + if (!data[deps.first].isNull()) + { + if (deps.second.getType() == JsonNode::DATA_VECTOR) + { + JsonVector depList = deps.second.Vector(); + for(auto & depEntry : depList) + { + if (data[depEntry.String()].isNull()) + errors += validator.makeErrorMessage("Property " + depEntry.String() + " required for " + deps.first + " is missing"); + } + } + else + { + if (!check(deps.second, data, validator).empty()) + errors += validator.makeErrorMessage("Requirements for " + deps.first + " are not fulfilled"); + } + } + } + return errors; + } + + std::string propertyEntryCheck(Validation::ValidationData & validator, const JsonNode &node, const JsonNode & schema, std::string nodeName) + { + validator.currentPath.push_back(JsonNode()); + validator.currentPath.back().String() = nodeName; + auto onExit = vstd::makeScopeGuard([&] + { + validator.currentPath.pop_back(); + }); + + // there is schema specifically for this item + if (!schema.isNull()) + return check(schema, node, validator); + return ""; + } + + std::string propertiesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + + for(auto & entry : data.Struct()) + errors += propertyEntryCheck(validator, entry.second, schema[entry.first], entry.first); + return errors; + } + + std::string additionalPropertiesCheck(Validation::ValidationData & validator, const JsonNode & baseSchema, const JsonNode & schema, const JsonNode & data) + { + std::string errors; + for(auto & entry : data.Struct()) + { + if (baseSchema["properties"].Struct().count(entry.first) == 0) + { + // try generic additionalItems schema + if (schema.getType() == JsonNode::DATA_STRUCT) + return propertyEntryCheck(validator, entry.second, schema, entry.first); + + // or, additionalItems field can be bool which indicates if such items are allowed + if (!schema.isNull() && schema.Bool() == false) // present and set to false - error + return validator.makeErrorMessage("Unknown entry found: " + entry.first); + } + } + return errors; + } + } + + namespace Formats + { + #define TEST_FILE(prefix, file, type) \ + if (CResourceHandler::get()->existsResource(ResourceID(prefix + file, type))) \ + return "" + + std::string testAnimation(std::string path) + { + TEST_FILE("Sprites/", path, EResType::ANIMATION); + TEST_FILE("Sprites/", path, EResType::TEXT); + return "Animation file \"" + path + "\" was not found"; + } + + std::string textFile(const JsonNode & node) + { + TEST_FILE("", node.String(), EResType::TEXT); + return "Text file \"" + node.String() + "\" was not found"; + } + + std::string musicFile(const JsonNode & node) + { + TEST_FILE("", node.String(), EResType::MUSIC); + return "Music file \"" + node.String() + "\" was not found"; + } + + std::string soundFile(const JsonNode & node) + { + TEST_FILE("Sounds/", node.String(), EResType::SOUND); + return "Sound file \"" + node.String() + "\" was not found"; + } + + std::string defFile(const JsonNode & node) + { + TEST_FILE("Sprites/", node.String(), EResType::ANIMATION); + return "Def file \"" + node.String() + "\" was not found"; + } + + std::string animationFile(const JsonNode & node) + { + return testAnimation(node.String()); + } + + std::string imageFile(const JsonNode & node) + { + TEST_FILE("Data/", node.String(), EResType::IMAGE); + TEST_FILE("Sprites/", node.String(), EResType::IMAGE); + if (node.String().find(':') != std::string::npos) + return testAnimation(node.String().substr(0, node.String().find(':'))); + return "Image file not found"; + } + + #undef TEST_FILE + } + + Validation::TValidatorMap createCommonFields() + { + Validation::TValidatorMap ret; + + ret["format"] = Common::formatCheck; + ret["allOf"] = Common::allOfCheck; + ret["anyOf"] = Common::anyOfCheck; + ret["oneOf"] = Common::oneOfCheck; + ret["enum"] = Common::enumCheck; + ret["type"] = Common::typeCheck; + ret["not"] = Common::notCheck; + ret["$ref"] = Common::refCheck; + + // fields that don't need implementation + ret["title"] = Common::emptyCheck; + ret["$schema"] = Common::emptyCheck; + ret["default"] = Common::emptyCheck; + ret["description"] = Common::emptyCheck; + ret["definitions"] = Common::emptyCheck; + return ret; + } + + Validation::TValidatorMap createStringFields() + { + Validation::TValidatorMap ret = createCommonFields(); + ret["maxLength"] = String::maxLengthCheck; + ret["minLength"] = String::minLengthCheck; + + ret["pattern"] = Common::notImplementedCheck; + return ret; + } + + Validation::TValidatorMap createNumberFields() + { + Validation::TValidatorMap ret = createCommonFields(); + ret["maximum"] = Number::maximumCheck; + ret["minimum"] = Number::minimumCheck; + ret["multipleOf"] = Number::multipleOfCheck; + + ret["exclusiveMaximum"] = Common::emptyCheck; + ret["exclusiveMinimum"] = Common::emptyCheck; + return ret; + } + + Validation::TValidatorMap createVectorFields() + { + Validation::TValidatorMap ret = createCommonFields(); + ret["items"] = Vector::itemsCheck; + ret["minItems"] = Vector::minItemsCheck; + ret["maxItems"] = Vector::maxItemsCheck; + ret["uniqueItems"] = Vector::uniqueItemsCheck; + ret["additionalItems"] = Vector::additionalItemsCheck; + return ret; + } + + Validation::TValidatorMap createStructFields() + { + Validation::TValidatorMap ret = createCommonFields(); + ret["additionalProperties"] = Struct::additionalPropertiesCheck; + ret["uniqueProperties"] = Struct::uniquePropertiesCheck; + ret["maxProperties"] = Struct::maxPropertiesCheck; + ret["minProperties"] = Struct::minPropertiesCheck; + ret["dependencies"] = Struct::dependenciesCheck; + ret["properties"] = Struct::propertiesCheck; + ret["required"] = Struct::requiredCheck; + + ret["patternProperties"] = Common::notImplementedCheck; + return ret; + } + + Validation::TFormatMap createFormatMap() + { + Validation::TFormatMap ret; + ret["textFile"] = Formats::textFile; + ret["musicFile"] = Formats::musicFile; + ret["soundFile"] = Formats::soundFile; + ret["defFile"] = Formats::defFile; + ret["animationFile"] = Formats::animationFile; + ret["imageFile"] = Formats::imageFile; + + return ret; + } +} + +namespace Validation +{ + std::string ValidationData::makeErrorMessage(const std::string &message) + { + std::string errors; + errors += "At "; + if (!currentPath.empty()) + { + for(const JsonNode &path : currentPath) + { + errors += "/"; + if (path.getType() == JsonNode::DATA_STRING) + errors += path.String(); + else + errors += boost::lexical_cast(static_cast(path.Float())); + } + } + else + errors += ""; + errors += "\n\t Error: " + message + "\n"; + return errors; + } + + std::string check(std::string schemaName, const JsonNode & data) + { + ValidationData validator; + return check(schemaName, data, validator); + } + + std::string check(std::string schemaName, const JsonNode & data, ValidationData & validator) + { + validator.usedSchemas.push_back(schemaName); + auto onscopeExit = vstd::makeScopeGuard([&]() + { + validator.usedSchemas.pop_back(); + }); + return check(JsonUtils::getSchema(schemaName), data, validator); + } + + std::string check(const JsonNode & schema, const JsonNode & data, ValidationData & validator) + { + const TValidatorMap & knownFields = getKnownFieldsFor(data.getType()); + std::string errors; + for(auto & entry : schema.Struct()) + { + auto checker = knownFields.find(entry.first); + if (checker != knownFields.end()) + errors += checker->second(validator, schema, entry.second, data); + //else + // errors += validator.makeErrorMessage("Unknown entry in schema " + entry.first); + } + return errors; + } + + const TValidatorMap & getKnownFieldsFor(JsonNode::JsonType type) + { + static const TValidatorMap commonFields = createCommonFields(); + static const TValidatorMap numberFields = createNumberFields(); + static const TValidatorMap stringFields = createStringFields(); + static const TValidatorMap vectorFields = createVectorFields(); + static const TValidatorMap structFields = createStructFields(); + + switch (type) + { + case JsonNode::DATA_FLOAT: return numberFields; + case JsonNode::DATA_STRING: return stringFields; + case JsonNode::DATA_VECTOR: return vectorFields; + case JsonNode::DATA_STRUCT: return structFields; + default: return commonFields; + } + } + + const TFormatMap & getKnownFormats() + { + static TFormatMap knownFormats = createFormatMap(); + return knownFormats; + } + +} // Validation namespace \ No newline at end of file diff --git a/lib/JsonDetail.h b/lib/JsonDetail.h new file mode 100644 index 000000000..e9299e5ed --- /dev/null +++ b/lib/JsonDetail.h @@ -0,0 +1,123 @@ +/* + * JsonDetail.h, part of VCMI engine + * + * Authors: listed in file AUTHORS in main folder + * + * License: GNU General Public License v2.0 or later + * Full text of license available in license.txt file, in main folder + * + */ + +#pragma once + +#include "JsonNode.h" + +class JsonWriter +{ + //prefix for each line (tabulation) + std::string prefix; + std::ostream &out; +public: + template + void writeContainer(Iterator begin, Iterator end); + void writeEntry(JsonMap::const_iterator entry); + void writeEntry(JsonVector::const_iterator entry); + void writeString(const std::string &string); + void writeNode(const JsonNode &node); + JsonWriter(std::ostream &output, const JsonNode &node); +}; + +//Tiny string class that uses const char* as data for speed, members are private +//for ease of debugging and some compatibility with std::string +class constString +{ + const char *data; + const size_t datasize; + +public: + constString(const char * inputString, size_t stringSize): + data(inputString), + datasize(stringSize) + { + } + + inline size_t size() const + { + return datasize; + }; + + inline const char& operator[] (size_t position) + { + assert (position < datasize); + return data[position]; + } +}; + +//Internal class for string -> JsonNode conversion +class JsonParser +{ + std::string errors; // Contains description of all encountered errors + constString input; // Input data + ui32 lineCount; // Currently parsed line, starting from 1 + size_t lineStart; // Position of current line start + size_t pos; // Current position of parser + + //Helpers + bool extractEscaping(std::string &str); + bool extractLiteral(const std::string &literal); + bool extractString(std::string &string); + bool extractWhitespace(bool verbose = true); + bool extractSeparator(); + bool extractElement(JsonNode &node, char terminator); + + //Methods for extracting JSON data + bool extractArray(JsonNode &node); + bool extractFalse(JsonNode &node); + bool extractFloat(JsonNode &node); + bool extractNull(JsonNode &node); + bool extractString(JsonNode &node); + bool extractStruct(JsonNode &node); + bool extractTrue(JsonNode &node); + bool extractValue(JsonNode &node); + + //Add error\warning message to list + bool error(const std::string &message, bool warning=false); + +public: + JsonParser(const char * inputString, size_t stringSize); + + /// do actual parsing. filename is name of file that will printed to console if any errors were found + JsonNode parse(std::string fileName); +}; + +//Internal class for Json validation. Mostly compilant with json-schema v4 draft +namespace Validation +{ + /// struct used to pass data around during validation + struct ValidationData + { + /// path from root node to current one. + /// JsonNode is used as variant - either string (name of node) or as float (index in list) + std::vector currentPath; + + /// Stack of used schemas. Last schema is the one used currently. + /// May contain multiple items in case if remote references were found + std::vector usedSchemas; + + /// generates error message + std::string makeErrorMessage(const std::string &message); + }; + + typedef std::function TFormatValidator; + typedef std::unordered_map TFormatMap; + typedef std::function TFieldValidator; + typedef std::unordered_map TValidatorMap; + + /// map of known fields in schema + const TValidatorMap & getKnownFieldsFor(JsonNode::JsonType type); + const TFormatMap & getKnownFormats(); + + std::string check(std::string schemaName, const JsonNode & data); + std::string check(std::string schemaName, const JsonNode & data, ValidationData & validator); + std::string check(const JsonNode & schema, const JsonNode & data, ValidationData & validator); +} \ No newline at end of file diff --git a/lib/JsonNode.cpp b/lib/JsonNode.cpp index 6797eae2d..2927f7803 100644 --- a/lib/JsonNode.cpp +++ b/lib/JsonNode.cpp @@ -18,6 +18,7 @@ #include "VCMI_Lib.h" //for identifier resolution #include "CModHandler.h" #include "CGeneralTextHandler.h" +#include "JsonDetail.h" using namespace JsonDetail; @@ -302,873 +303,6 @@ JsonNode & JsonNode::resolvePointer(const std::string &jsonPointer) return ::resolvePointer(*this, jsonPointer); } -//////////////////////////////////////////////////////////////////////////////// - -template -void JsonWriter::writeContainer(Iterator begin, Iterator end) -{ - if (begin == end) - return; - - prefix += '\t'; - - writeEntry(begin++); - - while (begin != end) - { - out<<",\n"; - writeEntry(begin++); - } - - out<<"\n"; - prefix.resize(prefix.size()-1); -} - -void JsonWriter::writeEntry(JsonMap::const_iterator entry) -{ - out << prefix; - writeString(entry->first); - out << " : "; - writeNode(entry->second); -} - -void JsonWriter::writeEntry(JsonVector::const_iterator entry) -{ - out << prefix; - writeNode(*entry); -} - -void JsonWriter::writeString(const std::string &string) -{ - static const std::string escaped = "\"\\\b\f\n\r\t"; - - out <<'\"'; - size_t pos=0, start=0; - for (; poswarnStream()<<"File " << fileName << " is not a valid JSON file!"; - logGlobal->warnStream()<= '0' && input[pos] <= '9') - return extractFloat(node); - return error("Value expected!"); - } - } -} - -bool JsonParser::extractWhitespace(bool verbose) -{ - while (true) - { - while (pos < input.size() && (ui8)input[pos] <= ' ') - { - if (input[pos] == '\n') - { - lineCount++; - lineStart = pos+1; - } - pos++; - } - if (pos >= input.size() || input[pos] != '/') - break; - - pos++; - if (pos == input.size()) - break; - if (input[pos] == '/') - pos++; - else - error("Comments must consist from two slashes!", true); - - while (pos < input.size() && input[pos] != '\n') - pos++; - } - - if (pos >= input.size() && verbose) - return error("Unexpected end of file!"); - return true; -} - -bool JsonParser::extractEscaping(std::string &str) -{ - switch(input[pos]) - { - break; case '\"': str += '\"'; - break; case '\\': str += '\\'; - break; case 'b': str += '\b'; - break; case 'f': str += '\f'; - break; case 'n': str += '\n'; - break; case 'r': str += '\r'; - break; case 't': str += '\t'; - break; default: return error("Unknown escape sequence!", true); - }; - return true; -} - -bool JsonParser::extractString(std::string &str) -{ - if (input[pos] != '\"') - return error("String expected!"); - pos++; - - size_t first = pos; - - while (pos != input.size()) - { - if (input[pos] == '\"') // Correct end of string - { - str.append( &input[first], pos-first); - pos++; - return true; - } - if (input[pos] == '\\') // Escaping - { - str.append( &input[first], pos-first); - pos++; - if (pos == input.size()) - break; - extractEscaping(str); - first = pos + 1; - } - if (input[pos] == '\n') // end-of-line - { - str.append( &input[first], pos-first); - return error("Closing quote not found!", true); - } - if ((unsigned char)(input[pos]) < ' ') // control character - { - str.append( &input[first], pos-first); - first = pos+1; - error("Illegal character in the string!", true); - } - pos++; - } - return error("Unterminated string!"); -} - -bool JsonParser::extractString(JsonNode &node) -{ - std::string str; - if (!extractString(str)) - return false; - - node.setType(JsonNode::DATA_STRING); - node.String() = str; - return true; -} - -bool JsonParser::extractLiteral(const std::string &literal) -{ - if (literal.compare(0, literal.size(), &input[pos], literal.size()) != 0) - { - while (pos < input.size() && ((input[pos]>'a' && input[pos]<'z') - || (input[pos]>'A' && input[pos]<'Z'))) - pos++; - return error("Unknown literal found", true); - } - - pos += literal.size(); - return true; -} - -bool JsonParser::extractNull(JsonNode &node) -{ - if (!extractLiteral("null")) - return false; - - node.clear(); - return true; -} - -bool JsonParser::extractTrue(JsonNode &node) -{ - if (!extractLiteral("true")) - return false; - - node.Bool() = true; - return true; -} - -bool JsonParser::extractFalse(JsonNode &node) -{ - if (!extractLiteral("false")) - return false; - - node.Bool() = false; - return true; -} - -bool JsonParser::extractStruct(JsonNode &node) -{ - node.setType(JsonNode::DATA_STRUCT); - pos++; - - if (!extractWhitespace()) - return false; - - //Empty struct found - if (input[pos] == '}') - { - pos++; - return true; - } - - while (true) - { - if (!extractWhitespace()) - return false; - - std::string key; - if (!extractString(key)) - return false; - - if (node.Struct().find(key) != node.Struct().end()) - error("Dublicated element encountered!", true); - - if (!extractSeparator()) - return false; - - if (!extractElement(node.Struct()[key], '}')) - return false; - - if (input[pos] == '}') - { - pos++; - return true; - } - } -} - -bool JsonParser::extractArray(JsonNode &node) -{ - pos++; - node.setType(JsonNode::DATA_VECTOR); - - if (!extractWhitespace()) - return false; - - //Empty array found - if (input[pos] == ']') - { - pos++; - return true; - } - - while (true) - { - //NOTE: currently 50% of time is this vector resizing. - //May be useful to use list during parsing and then swap() all items to vector - node.Vector().resize(node.Vector().size()+1); - - if (!extractElement(node.Vector().back(), ']')) - return false; - - if (input[pos] == ']') - { - pos++; - return true; - } - } -} - -bool JsonParser::extractElement(JsonNode &node, char terminator) -{ - if (!extractValue(node)) - return false; - - if (!extractWhitespace()) - return false; - - bool comma = (input[pos] == ','); - if (comma ) - { - pos++; - if (!extractWhitespace()) - return false; - } - - if (input[pos] == terminator) - { - //FIXME: MOD COMPATIBILITY: Too many of these right now, re-enable later - //if (comma) - //error("Extra comma found!", true); - return true; - } - - if (!comma) - error("Comma expected!", true); - - return true; -} - -bool JsonParser::extractFloat(JsonNode &node) -{ - assert(input[pos] == '-' || (input[pos] >= '0' && input[pos] <= '9')); - bool negative=false; - double result=0; - - if (input[pos] == '-') - { - pos++; - negative = true; - } - - if (input[pos] < '0' || input[pos] > '9') - return error("Number expected!"); - - //Extract integer part - while (input[pos] >= '0' && input[pos] <= '9') - { - result = result*10+(input[pos]-'0'); - pos++; - } - - if (input[pos] == '.') - { - //extract fractional part - pos++; - double fractMult = 0.1; - if (input[pos] < '0' || input[pos] > '9') - return error("Decimal part expected!"); - - while (input[pos] >= '0' && input[pos] <= '9') - { - result = result + fractMult*(input[pos]-'0'); - fractMult /= 10; - pos++; - } - } - //TODO: exponential part - if (negative) - result = -result; - - node.setType(JsonNode::DATA_FLOAT); - node.Float() = result; - return true; -} - -bool JsonParser::error(const std::string &message, bool warning) -{ - std::ostringstream stream; - std::string type(warning?" warning: ":" error: "); - - stream << "At line " << lineCount << ", position "< stringToType = - boost::assign::map_list_of - ("null", JsonNode::DATA_NULL) ("boolean", JsonNode::DATA_BOOL) - ("number", JsonNode::DATA_FLOAT) ("string", JsonNode::DATA_STRING) - ("array", JsonNode::DATA_VECTOR) ("object", JsonNode::DATA_STRUCT); - -std::string JsonValidator::validateEnum(const JsonNode &node, const JsonVector &enumeration) -{ - for(auto & enumEntry : enumeration) - { - if (node == enumEntry) - return ""; - } - return fail("Key must have one of predefined values"); -} - -std::string JsonValidator::validatesSchemaList(const JsonNode &node, const JsonNode &schemas, std::string errorMsg, std::function isValid) -{ - if (!schemas.isNull()) - { - std::string errors = "\n"; - size_t result = 0; - - for(auto & schema : schemas.Vector()) - { - std::string error = validateNode(node, schema); - if (error.empty()) - { - result++; - } - else - { - errors += error; - errors += "\n"; - } - } - if (isValid(result)) - { - return ""; - } - return fail(errorMsg) + errors; - } - return ""; -} - -std::string JsonValidator::validateNodeType(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - - // data must be valid against all schemas in the list - errors += validatesSchemaList(node, schema["allOf"], "Failed to pass all schemas", [&](size_t count) - { - return count == schema["allOf"].Vector().size(); - }); - - // data must be valid against any non-zero number of schemas in the list - errors += validatesSchemaList(node, schema["anyOf"], "Failed to pass any schema", [&](size_t count) - { - return count > 0; - }); - - // data must be valid against one and only one schema - errors += validatesSchemaList(node, schema["oneOf"], "Failed to pass one and only one schema", [&](size_t count) - { - return count == 1; - }); - - // data must NOT be valid against schema - if (!schema["not"].isNull()) - { - if (validateNode(node, schema["not"]).empty()) - errors += fail("Successful validation against negative check"); - } - return errors; -} - -// Basic checks common for any nodes -std::string JsonValidator::validateNode(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - - assert(!schema.isNull()); // can this error be triggered? - - if (node.isNull()) - return ""; // node not present. consider to be "valid" - - if (!schema["$ref"].isNull()) - { - std::string URI = schema["$ref"].String(); - //node must be validated using schema pointed by this reference and not by data here - //Local reference. Turn it into more easy to handle remote ref - if (boost::algorithm::starts_with(URI, "#")) - URI = usedSchemas.back() + URI; - - return validateRoot(node, URI); - } - - // basic schema check - auto & typeNode = schema["type"]; - if ( !typeNode.isNull()) - { - JsonNode::JsonType type = stringToType.find(typeNode.String())->second; - if(type != node.getType()) - return errors + fail("Type mismatch!"); // different type. Any other checks are useless - } - - errors += validateNodeType(node, schema); - - // enumeration - data must be equeal to one of items in list - if (!schema["enum"].isNull()) - errors += validateEnum(node, schema["enum"].Vector()); - - // try to run any type-specific checks - if (node.getType() == JsonNode::DATA_VECTOR) errors += validateVector(node, schema); - if (node.getType() == JsonNode::DATA_STRUCT) errors += validateStruct(node, schema); - if (node.getType() == JsonNode::DATA_STRING) errors += validateString(node, schema); - if (node.getType() == JsonNode::DATA_FLOAT) errors += validateNumber(node, schema); - - return errors; -} - -std::string JsonValidator::validateVectorItem(const JsonVector items, const JsonNode & schema, const JsonNode & additional, size_t index) -{ - currentPath.push_back(JsonNode()); - currentPath.back().Float() = index; - auto onExit = vstd::makeScopeGuard([&] - { - currentPath.pop_back(); - }); - - if (!schema.isNull()) - { - // case 1: schema is vector. Validate items agaist corresponding items in vector - if (schema.getType() == JsonNode::DATA_VECTOR) - { - if (schema.Vector().size() > index) - return validateNode(items[index], schema.Vector()[index]); - } - else // case 2: schema has to be struct. Apply it to all items, completely ignore additionalItems - { - return validateNode(items[index], schema); - } - } - - // othervice check against schema in additional items field - if (additional.getType() == JsonNode::DATA_STRUCT) - return validateNode(items[index], additional); - - // or, additionalItems field can be bool which indicates if such items are allowed - if (!additional.isNull() && additional.Bool() == false) // present and set to false - error - return fail("Unknown entry found"); - - // by default - additional items are allowed - return ""; -} - -//Checks "items" entry from schema (type-specific check for Vector) -std::string JsonValidator::validateVector(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - auto & vector = node.Vector(); - - { - auto & items = schema["items"]; - auto & additional = schema["additionalItems"]; - - for (size_t i=0; i schema["maxItems"].Float()) - errors += fail("Too many items in the list!"); - - if (vstd::contains(schema.Struct(), "minItems") && vector.size() < schema["minItems"].Float()) - errors += fail("Too few items in the list"); - - if (schema["uniqueItems"].Bool()) - { - for (auto itA = vector.begin(); itA != vector.end(); itA++) - { - auto itB = itA; - while (++itB != vector.end()) - { - if (*itA == *itB) - errors += fail("List must consist from unique items"); - } - } - } - return errors; -} - -std::string JsonValidator::validateStructItem(const JsonNode &node, const JsonNode & schema, const JsonNode & additional, std::string nodeName) -{ - currentPath.push_back(JsonNode()); - currentPath.back().String() = nodeName; - auto onExit = vstd::makeScopeGuard([&] - { - currentPath.pop_back(); - }); - - // there is schema specifically for this item - if (!schema[nodeName].isNull()) - return validateNode(node, schema[nodeName]); - - // try generic additionalItems schema - if (additional.getType() == JsonNode::DATA_STRUCT) - return validateNode(node, additional); - - // or, additionalItems field can be bool which indicates if such items are allowed - if (!additional.isNull() && additional.Bool() == false) // present and set to false - error - return fail("Unknown entry found: " + nodeName); - - // by default - additional items are allowed - return ""; -} - -//Checks "properties" entry from schema (type-specific check for Struct) -std::string JsonValidator::validateStruct(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - auto & map = node.Struct(); - - { - auto & properties = schema["properties"]; - auto & additional = schema["additionalProperties"]; - - for(auto & entry : map) - errors += validateStructItem(entry.second, properties, additional, entry.first); - } - - for(auto & required : schema["required"].Vector()) - { - if (node[required.String()].isNull()) - errors += fail("Required entry " + required.String() + " is missing"); - } - - //Copy-paste from vector code. yay! - if (vstd::contains(schema.Struct(), "maxProperties") && map.size() > schema["maxProperties"].Float()) - errors += fail("Too many items in the list!"); - - if (vstd::contains(schema.Struct(), "minItems") && map.size() < schema["minItems"].Float()) - errors += fail("Too few items in the list"); - - if (schema["uniqueItems"].Bool()) - { - for (auto itA = map.begin(); itA != map.end(); itA++) - { - auto itB = itA; - while (++itB != map.end()) - { - if (itA->second == itB->second) - errors += fail("List must consist from unique items"); - } - } - } - - // dependencies. Format is object/struct where key is the name of key in data - // and value is either: - // a) array of fields that must be present - // b) struct with schema against which data should be valid - // These checks are triggered only if key is present - for(auto & deps : schema["dependencies"].Struct()) - { - if (vstd::contains(map, deps.first)) - { - if (deps.second.getType() == JsonNode::DATA_VECTOR) - { - JsonVector depList = deps.second.Vector(); - for(auto & depEntry : depList) - { - if (!vstd::contains(map, depEntry.String())) - errors += fail("Property " + depEntry.String() + " required for " + deps.first + " is missing"); - } - } - else - { - if (!validateNode(node, deps.second).empty()) - errors += fail("Requirements for " + deps.first + " are not fulfilled"); - } - } - } - - // TODO: missing fields from draft v4 - // patternProperties - return errors; -} - -std::string JsonValidator::validateString(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - auto & string = node.String(); - - if (vstd::contains(schema.Struct(), "maxLength") && string.size() > schema["maxLength"].Float()) - errors += fail("String too long"); - - if (vstd::contains(schema.Struct(), "minLength") && string.size() < schema["minLength"].Float()) - errors += fail("String too short"); - - // TODO: missing fields from draft v4 - // pattern - return errors; -} - -std::string JsonValidator::validateNumber(const JsonNode &node, const JsonNode &schema) -{ - std::string errors; - auto & value = node.Float(); - if (vstd::contains(schema.Struct(), "maximum")) - { - if (schema["exclusiveMaximum"].Bool()) - { - if (value >= schema["maximum"].Float()) - errors += fail("Value is too large"); - } - else - { - if (value > schema["maximum"].Float()) - errors += fail("Value is too large"); - } - } - - if (vstd::contains(schema.Struct(), "minimum")) - { - if (schema["exclusiveMinimum"].Bool()) - { - if (value <= schema["minimum"].Float()) - errors += fail("Value is too small"); - } - else - { - if (value < schema["minimum"].Float()) - errors += fail("Value is too small"); - } - } - - if (vstd::contains(schema.Struct(), "multipleOf")) - { - double result = value / schema["multipleOf"].Float(); - if (floor(result) != result) - errors += ("Value is not divisible"); - } - return errors; -} - -//basic schema validation (like checking $schema entry). -std::string JsonValidator::validateRoot(const JsonNode &node, std::string schemaName) -{ - const JsonNode & schema = JsonUtils::getSchema(schemaName); - - usedSchemas.push_back(schemaName.substr(0, schemaName.find('#'))); - auto onExit = vstd::makeScopeGuard([&] - { - usedSchemas.pop_back(); - }); - - if (!schema.isNull()) - return validateNode(node, schema); - else - return fail("Schema not found!"); -} - -std::string JsonValidator::fail(const std::string &message) -{ - std::string errors; - errors += "At "; - if (!currentPath.empty()) - { - for(const JsonNode &path : currentPath) - { - errors += "/"; - if (path.getType() == JsonNode::DATA_STRING) - errors += path.String(); - else - errors += boost::lexical_cast(static_cast(path.Float())); - } - } - else - errors += ""; - errors += "\n\t Error: " + message + "\n"; - return errors; -} - -bool JsonValidator::validate(const JsonNode &root, std::string schemaName, std::string name) -{ - std::string errors = validateRoot(root, schemaName); - - if (!errors.empty()) - { - logGlobal->warnStream() << "Data in " << name << " is invalid!"; - logGlobal->warnStream() << errors; - } - - return errors.empty(); -} - ///JsonUtils void JsonUtils::parseTypedBonusShort(const JsonVector& source, Bonus *dest) @@ -1499,8 +633,13 @@ void JsonUtils::maximize(JsonNode & node, std::string schemaName) bool JsonUtils::validate(const JsonNode &node, std::string schemaName, std::string dataName) { - JsonValidator validator; - return validator.validate(node, schemaName, dataName); + std::string log = Validation::check(schemaName, node); + if (!log.empty()) + { + logGlobal->errorStream() << "Data in " << dataName << " is invalid!"; + logGlobal->errorStream() << log; + } + return log.empty(); } const JsonNode & getSchemaByName(std::string name) diff --git a/lib/JsonNode.h b/lib/JsonNode.h index 0d0499412..828483ae9 100644 --- a/lib/JsonNode.h +++ b/lib/JsonNode.h @@ -192,10 +192,6 @@ namespace JsonUtils DLL_LINKAGE const JsonNode & getSchema(std::string URI); } -////////////////////////////////////////////////////////////////////////////////////////////////////// -// End of public section of the file. Anything below should be only used internally in JsonNode.cpp // -////////////////////////////////////////////////////////////////////////////////////////////////////// - namespace JsonDetail { // convertion helpers for JsonNode::convertTo (partial template function instantiation is illegal in c++) @@ -292,123 +288,6 @@ namespace JsonDetail return node.Bool(); } }; - - class JsonWriter - { - //prefix for each line (tabulation) - std::string prefix; - std::ostream &out; - public: - template - void writeContainer(Iterator begin, Iterator end); - void writeEntry(JsonMap::const_iterator entry); - void writeEntry(JsonVector::const_iterator entry); - void writeString(const std::string &string); - void writeNode(const JsonNode &node); - JsonWriter(std::ostream &output, const JsonNode &node); - }; - - //Tiny string class that uses const char* as data for speed, members are private - //for ease of debugging and some compatibility with std::string - class constString - { - const char *data; - const size_t datasize; - - public: - constString(const char * inputString, size_t stringSize): - data(inputString), - datasize(stringSize) - { - } - - inline size_t size() const - { - return datasize; - }; - - inline const char& operator[] (size_t position) - { - assert (position < datasize); - return data[position]; - } - }; - - //Internal class for string -> JsonNode conversion - class JsonParser - { - std::string errors; // Contains description of all encountered errors - constString input; // Input data - ui32 lineCount; // Currently parsed line, starting from 1 - size_t lineStart; // Position of current line start - size_t pos; // Current position of parser - - //Helpers - bool extractEscaping(std::string &str); - bool extractLiteral(const std::string &literal); - bool extractString(std::string &string); - bool extractWhitespace(bool verbose = true); - bool extractSeparator(); - bool extractElement(JsonNode &node, char terminator); - - //Methods for extracting JSON data - bool extractArray(JsonNode &node); - bool extractFalse(JsonNode &node); - bool extractFloat(JsonNode &node); - bool extractNull(JsonNode &node); - bool extractString(JsonNode &node); - bool extractStruct(JsonNode &node); - bool extractTrue(JsonNode &node); - bool extractValue(JsonNode &node); - - //Add error\warning message to list - bool error(const std::string &message, bool warning=false); - - public: - JsonParser(const char * inputString, size_t stringSize); - - /// do actual parsing. filename is name of file that will printed to console if any errors were found - JsonNode parse(std::string fileName); - }; - - //Internal class for Json validation. Mostly compilant with json-schema v4 draft - class JsonValidator - { - // path from root node to current one. - // JsonNode is used as variant - either string (name of node) or as float (index in list) - std::vector currentPath; - // Stack of used schemas. Last schema is the one used currently. - // May contain multiple items in case if remote references were found - std::vector usedSchemas; - - /// helpers for other validation methods - std::string validateVectorItem(const JsonVector items, const JsonNode & schema, const JsonNode & additional, size_t index); - std::string validateStructItem(const JsonNode &node, const JsonNode &schema, const JsonNode & additional, std::string nodeName); - - std::string validateEnum(const JsonNode &node, const JsonVector &enumeration); - std::string validateNodeType(const JsonNode &node, const JsonNode &schema); - std::string validatesSchemaList(const JsonNode &node, const JsonNode &schemas, std::string errorMsg, std::function isValid); - - /// contains all type-independent checks - std::string validateNode(const JsonNode &node, const JsonNode &schema); - - /// type-specific checks - std::string validateVector(const JsonNode &node, const JsonNode &schema); - std::string validateStruct(const JsonNode &node, const JsonNode &schema); - std::string validateString(const JsonNode &node, const JsonNode &schema); - std::string validateNumber(const JsonNode &node, const JsonNode &schema); - - /// validation of root node of both schema and input data - std::string validateRoot(const JsonNode &node, std::string schemaName); - - /// add error message to list and return false - std::string fail(const std::string &message); - public: - - /// returns true if parsed data is fully compilant with schema - bool validate(const JsonNode &root, std::string schemaName, std::string name); - }; - } // namespace JsonDetail template