From 618cc0ab7a48302fc5c7c8ec2c0ecc935d921f32 Mon Sep 17 00:00:00 2001 From: Daniele Panozzo Date: Tue, 17 Nov 2026 15:40:36 -0500 Subject: [PATCH] Index the rules by pointer instead of copying them for every node verify_json looked up the rules of every node of the input by scanning the whole spec, and copied every rule while doing it: collect_pointer and collect_default_rules iterated with `for (auto i : rules)` and returned copies, the matching loop copied each matching rule again, and comparing a rule's pointer with the node's (`i.at("pointer") == pointer`) built a json from the string for every rule. A spec of N rules on an input of M nodes cost M x N rule copies. The rules are now indexed by pointer once per verify_json / inject_defaults call (RuleIndex: pointers into `rules`, in spec order, and the subset with a default), and the recursion looks nodes up in it and works on references. Smaller copies go too: rule options, required keys and extensions are iterated by reference, and contained_in_list compares strings without building a json per element. Nothing observable changes: the same rules match in the same order, so the results, the injected defaults and the log (including the order of the rules listed in an error) are the same; a rule without "pointer" still throws from at(), and a non-string pointer still matches nothing. The public API is unchanged; only private members of JSE changed. Verify + inject_defaults: polysolve's nonlinear spec 13.1 ms -> 0.078 ms, its linear spec 13.0 ms -> 0.074 ms, an application spec of 53 rules on a large input 252 ms -> 1.7 ms; polysolve's Solver::create (DenseNewton, which validates the linear spec 3 times) 43 ms -> 0.48 ms. The unit tests run in 0.04 s instead of 2 s. New tests cover the behaviour the index must keep: the rules of a pointer in spec order, "skip" and other defaults, non-string pointers and rules without one, inconsistent defaults. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ULwCy3KGG3sBouB7ZLaXwF --- src/jse/jse.cpp | 119 ++++++++++++++++++++++++++++----------- src/jse/jse.h | 7 ++- tests/test_validator.cpp | 95 +++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 35 deletions(-) diff --git a/src/jse/jse.cpp b/src/jse/jse.cpp index ac50c77..5100480 100644 --- a/src/jse/jse.cpp +++ b/src/jse/jse.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include //////////////////////////////////////////////////////////////////////////////// namespace jse @@ -12,18 +14,63 @@ namespace jse //////////// PUBLIC + // The rules of each pointer, and those of them that have a default, in the order of `rules`. + // It holds pointers into `rules`, which must outlive it and stay unchanged. Looking a pointer up + // here is what collect_pointer/collect_default_rules did by scanning (and copying) every rule + // for every node of the input. + struct JSE::RuleIndex + { + std::unordered_map> rules; + std::unordered_map> defaults; + + explicit RuleIndex(const json &all_rules) + { + for (const auto &rule : all_rules) + { + // at() throws for a rule without a pointer, as the scans did + const string *pointer = rule.at("pointer").get_ptr(); + if (pointer == nullptr) + continue; // a pointer that is not a string matches no node + rules[*pointer].push_back(&rule); + if (rule.contains("default")) + defaults[*pointer].push_back(&rule); + } + } + + const std::vector &rules_at(const string &pointer) const + { + return find(rules, pointer); + } + + const std::vector &defaults_at(const string &pointer) const + { + return find(defaults, pointer); + } + + private: + static const std::vector &find( + const std::unordered_map> &map, const string &pointer) + { + static const std::vector none; + const auto it = map.find(pointer); + return it == map.end() ? none : it->second; + } + }; + bool JSE::verify_json(const json &input, const json &rules) { log.clear(); + const RuleIndex index(rules); json input_copy = input; - return verify_json("/", input_copy, rules); + return verify_json("/", input_copy, rules, index); }; json JSE::inject_defaults(const json &input, const json &rules) { log.clear(); + const RuleIndex index(rules); json input_copy = input; - bool b = verify_json("/", input_copy, rules); + bool b = verify_json("/", input_copy, rules, index); assert(b); // Make sure the original file is valid // Check that the new file, after adding defaults, is also valid @@ -36,7 +83,7 @@ namespace jse { std::stringstream s; - for (log_item i : log) + for (const log_item &i : log) { s << i.first << ": " << i.second << std::endl; } @@ -149,11 +196,10 @@ namespace jse throw std::runtime_error("Reached maximal 10 levels of include recursion."); } - bool JSE::verify_json(const string &pointer, json &input, const json &rules) + bool JSE::verify_json(const string &pointer, json &input, const json &rules, const RuleIndex &index) { // Find all rules that apply for the input node - // TODO: accelerate this - std::vector matching_rules = collect_pointer(pointer, rules); + const std::vector &matching_rules = index.rules_at(pointer); // There must be at least one, otherwise warning and return true if not strict if (matching_rules.empty()) @@ -163,10 +209,10 @@ namespace jse } // Test all rules, one and only one must pass, otherwise throw exception - std::vector verified_matching_rules; - for (auto r : matching_rules) + std::vector verified_matching_rules; + for (const json *r : matching_rules) { - if (verify_rule(input, r)) + if (verify_rule(input, *r)) { verified_matching_rules.push_back(r); } @@ -179,17 +225,17 @@ namespace jse { string new_pointer = append_pointer(pointer, "*"); // Make sure there are some rules for the boxed version before recursively checking - if (collect_pointer(new_pointer, rules).size() > 0) - if (verify_json(new_pointer, input, rules)) + if (index.rules_at(new_pointer).size() > 0) + if (verify_json(new_pointer, input, rules, index)) return true; } // If there is only one candidate rule (no ambiguity) and it is an object rule // that failed only because of extra/unknown fields, pinpoint those fields // instead of dumping the whole subtree. - if (strict && matching_rules.size() == 1 && matching_rules[0].value("type", "") == "object") + if (strict && matching_rules.size() == 1 && matching_rules[0]->value("type", "") == "object") { - std::vector extra_keys = find_extra_keys(input, matching_rules[0]); + std::vector extra_keys = find_extra_keys(input, *matching_rules[0]); if (!extra_keys.empty()) { for (const std::string &key : extra_keys) @@ -199,7 +245,7 @@ namespace jse s << "No rule matched for \"" << extra_pointer << "\": " << input[key].dump(/*indent=*/4) << std::endl; s << "No valid rules in this list:\n"; for (int i = 0; i < matching_rules.size(); i++) - s << i << ": " << matching_rules[i].dump(/*indent=*/4) << "\n"; + s << i << ": " << matching_rules[i]->dump(/*indent=*/4) << "\n"; log.push_back(log_item("error", s.str())); } return false; @@ -210,7 +256,7 @@ namespace jse s << "No rule matched for \"" << pointer << "\": " << input.dump(/*indent=*/4) << std::endl; s << "No valid rules in this list:\n"; for (int i = 0; i < matching_rules.size(); i++) - s << i << ": " << matching_rules[i].dump(/*indent=*/4) << "\n"; + s << i << ": " << matching_rules[i]->dump(/*indent=*/4) << "\n"; log.push_back(log_item("error", s.str())); return false; } @@ -220,11 +266,11 @@ namespace jse s << "Multiple rules matched for \"" << pointer << "\": " << input.dump(/*indent=*/4) << std::endl; s << "Multiple valid rules in this list, only one should be valid:"; for (int i = 0; i < verified_matching_rules.size(); i++) - s << i << ": " << verified_matching_rules[i].dump(/*indent=*/4) << "\n"; + s << i << ": " << verified_matching_rules[i]->dump(/*indent=*/4) << "\n"; log.push_back(log_item("error", s.str())); return false; // if not strict, we do not check for extra keys to distinguish between rules } - const json &single_matched_rule = verified_matching_rules.front(); + const json &single_matched_rule = *verified_matching_rules.front(); // If it passes and if it is a dictionary, then test all childrens if (input.is_object()) @@ -233,7 +279,7 @@ namespace jse { const string new_pointer = append_pointer(pointer, key); // first of all, let's check if the specs are correct - const json defaults = collect_default_rules(new_pointer, rules); + const std::vector &defaults = index.defaults_at(new_pointer); // if it is mandatory, make sure there are no defaults if (single_matched_rule.contains("required") && contained_in_list(key, single_matched_rule["required"])) @@ -262,7 +308,7 @@ namespace jse } // now let's make sure it can be validated - if (!verify_json(new_pointer, value, rules)) + if (!verify_json(new_pointer, value, rules, index)) return false; } } @@ -278,18 +324,20 @@ namespace jse continue; string new_pointer = append_pointer(pointer, key); - json defaults = collect_default_rules(new_pointer, rules); // Find the default + const std::vector &defaults = index.defaults_at(new_pointer); // Find the default if (defaults.size() != 1) { log.push_back(log_item("error", "Inconsistent specifications: " + new_pointer + " is an optional field with " + std::to_string(defaults.size()) + " default values.")); return false; } - if (defaults[0]["default"] != "skip") + const json &default_value = defaults[0]->at("default"); + const string *default_string = default_value.get_ptr(); + if (default_string == nullptr || *default_string != "skip") { - input[key] = defaults[0]["default"]; + input[key] = default_value; // Let's validate/inject the default subtree - if (!verify_json(new_pointer, input[key], rules)) + if (!verify_json(new_pointer, input[key], rules, index)) return false; } } @@ -300,7 +348,7 @@ namespace jse if (input.is_array()) { for (auto &i : input) - if (!verify_json(append_pointer(pointer, "*"), i, rules)) + if (!verify_json(append_pointer(pointer, "*"), i, rules, index)) return false; } @@ -358,7 +406,7 @@ namespace jse { std::string ext = p.extension().string(); int count = 0; - for (auto e : rule["extensions"]) + for (const auto &e : rule["extensions"]) if (e == ext) count++; if (count != 1) @@ -427,7 +475,7 @@ namespace jse if (rule.contains("options")) { int count = 0; - for (auto e : rule["options"]) + for (const auto &e : rule["options"]) if (e == input) count++; if (count != 1) @@ -463,7 +511,7 @@ namespace jse // Check that all required fields are present if (rule.contains("required")) - for (auto e : rule["required"]) + for (const auto &e : rule["required"]) if (!input.contains(string(e))) return false; @@ -515,7 +563,7 @@ namespace jse // TODO: accelerate this std::vector matching_rules; - for (auto i : rules) + for (const auto &i : rules) { if (i.contains("default")) matching_rules.push_back(i); @@ -530,7 +578,7 @@ namespace jse // TODO: accelerate this std::vector matching_rules; - for (auto i : rules) + for (const auto &i : rules) { if (i.at("pointer") == pointer && i.contains("default")) matching_rules.push_back(i); @@ -539,15 +587,20 @@ namespace jse return matching_rules; } - bool JSE::contained_in_list(string item, const json &list) + bool JSE::contained_in_list(const string &item, const json &list) { - return std::find(list.begin(), list.end(), item) != list.end(); + // As `element == item`, without making a json of item for every element + return std::find_if(list.begin(), list.end(), [&item](const json &element) { + const string *s = element.get_ptr(); + return s != nullptr && *s == item; + }) + != list.end(); } std::vector JSE::collect_pointer(const string &pointer, const json &rules) { std::vector matching_rules; - for (auto i : rules) + for (const auto &i : rules) { if (i.at("pointer") == pointer) matching_rules.push_back(i); @@ -557,7 +610,7 @@ namespace jse json JSE::find_valid_rule(const string &pointer, const json &input, const json &rules) { - for (auto i : collect_pointer(pointer, rules)) + for (const auto &i : collect_pointer(pointer, rules)) if (verify_rule(input, i)) return i; diff --git a/src/jse/jse.h b/src/jse/jse.h index 2a07887..e665de3 100644 --- a/src/jse/jse.h +++ b/src/jse/jse.h @@ -48,8 +48,11 @@ namespace jse std::vector log; private: + // The rules of each pointer, built once per verify_json/inject_defaults call + struct RuleIndex; + // Verify a node pointed by - bool verify_json(const string &pointer, json &input, const json &rules); + bool verify_json(const string &pointer, json &input, const json &rules, const RuleIndex &index); // Dispatcher for rule verification bool verify_rule(const json &input, const json &rule); @@ -82,7 +85,7 @@ namespace jse std::vector find_extra_keys(const json &input, const json &rule); // Utils - bool contained_in_list(string item, const json &list); + bool contained_in_list(const string &item, const json &list); // Checks if a given json pointer is a subset of a pointer string (containing wildcards). // If it is, the second return parameter is an instantiated pointer diff --git a/tests/test_validator.cpp b/tests/test_validator.cpp index 5711261..ef02709 100644 --- a/tests/test_validator.cpp +++ b/tests/test_validator.cpp @@ -776,3 +776,98 @@ TEST_CASE("null_as_nan", "[validator][inject]") input["f2"] = std::nan(""); } + +TEST_CASE("rules_are_matched_in_spec_order", "[validator]") +{ + // Rules of one pointer, interleaved with others: an ambiguous node lists them in the order of + // the spec. + json rules = R"( + [ + {"pointer": "/", "type": "object", "optional": ["a"]}, + {"pointer": "/a", "type": "int", "doc": "first", "default": 1}, + {"pointer": "/b", "type": "int"}, + {"pointer": "/a", "type": "float", "doc": "second"} + ] + )"_json; + + JSE jse; + REQUIRE(!jse.verify_json(R"({"a": 2})"_json, rules)); + const std::string log = jse.log2str(); + INFO(log); + REQUIRE(log.find("Multiple rules matched for \"/a\"") != std::string::npos); + REQUIRE(log.find("\"first\"") != std::string::npos); + REQUIRE(log.find("\"first\"") < log.find("\"second\"")); + + REQUIRE(!jse.verify_json(R"({"a": "text"})"_json, rules)); + REQUIRE(jse.log2str().find("No rule matched for \"/a\"") != std::string::npos); + REQUIRE(jse.log2str().find("\"first\"") < jse.log2str().find("\"second\"")); + + // A float alone matches the second rule only. + REQUIRE(jse.verify_json(R"({"a": 2.5})"_json, rules)); +} + +TEST_CASE("skip_and_other_defaults", "[inject]") +{ + json rules = R"( + [ + {"pointer": "/", "type": "object", "optional": ["skipped", "number", "word", "object", "list"]}, + {"pointer": "/skipped", "type": "int", "default": "skip"}, + {"pointer": "/number", "type": "int", "default": 3}, + {"pointer": "/word", "type": "string", "default": "skipping"}, + {"pointer": "/object", "type": "object", "optional": ["inner"], "default": {}}, + {"pointer": "/object/inner", "type": "string", "default": "skip"}, + {"pointer": "/list", "type": "list", "default": ["skip"]}, + {"pointer": "/list/*", "type": "string"} + ] + )"_json; + + JSE jse; + jse.strict = true; + REQUIRE(jse.verify_json(json::object(), rules)); + const json out = jse.inject_defaults(json::object(), rules); + INFO(out.dump()); + CHECK(!out.contains("skipped")); + CHECK(out["number"] == 3); + CHECK(out["word"] == "skipping"); + CHECK(out["object"].is_object()); + CHECK(!out["object"].contains("inner")); + CHECK(out["list"] == json::array({"skip"})); +} + +TEST_CASE("rule_pointers", "[validator]") +{ + // A pointer that is not a string matches no node. + json rules = R"( + [ + {"pointer": "/", "type": "object", "optional": ["a"]}, + {"pointer": 5, "type": "int"}, + {"pointer": "/a", "type": "int", "default": 0} + ] + )"_json; + JSE jse; + jse.strict = true; + REQUIRE(jse.verify_json(R"({"a": 1})"_json, rules)); + + // A rule without a pointer is an error in the spec. + rules.push_back(R"({"type": "int"})"_json); + REQUIRE_THROWS_AS(jse.verify_json(R"({"a": 1})"_json, rules), nlohmann::json::exception); +} + +TEST_CASE("inconsistent_defaults", "[validator]") +{ + json rules = R"( + [ + {"pointer": "/", "type": "object", "required": ["r"], "optional": ["o"]}, + {"pointer": "/r", "type": "int", "default": 1}, + {"pointer": "/o", "type": "int", "default": 1}, + {"pointer": "/o", "type": "float", "default": 2.0} + ] + )"_json; + JSE jse; + REQUIRE(!jse.verify_json(R"({"r": 1})"_json, rules)); + CHECK(jse.log2str().find("/r is a mandatory field with a default value") != std::string::npos); + + rules[1].erase("default"); + REQUIRE(!jse.verify_json(R"({"r": 1})"_json, rules)); + CHECK(jse.log2str().find("/o is an optional field with 2 default values") != std::string::npos); +}