add loop option

src/command_lint.cc

@@ -29,8 +29,8 @@ static auto disable_lint_rules(sourcemeta::core::SchemaTransformer &bundle,
 }
 
 static auto reindent(const std::string_view &value,
-                     const std::string &indentation, std::ostream &stream)
-    -> void {
+                     const std::string &indentation,
+                     std::ostream &stream) -> void {
   if (!value.empty()) {
     stream << indentation;
   }

src/command_test.cc

@@ -13,8 +13,8 @@
 #include "utils.h"
 
 static auto get_data(const sourcemeta::core::JSON &test_case,
-                     const std::filesystem::path &base, const bool verbose)
-    -> sourcemeta::core::JSON {
+                     const std::filesystem::path &base,
+                     const bool verbose) -> sourcemeta::core::JSON {
   assert(base.is_absolute());
   assert(test_case.is_object());
   assert(test_case.defines("data") || test_case.defines("dataPath"));

src/command_validate.cc

@@ -6,6 +6,7 @@
 #include <sourcemeta/blaze/evaluator.h>
 
 #include <chrono>   // std::chrono
+#include <cmath>    // for sqrt
 #include <cstdlib>  // EXIT_SUCCESS, EXIT_FAILURE
 #include <iostream> // std::cerr
 #include <set>      // std::set
@@ -66,6 +67,24 @@ auto get_schema_template(
 
 } // namespace
 
+// parse unsigned int option, how many loop for benchmarking
+auto benchmark_loop(
+    const std::map<std::string, std::vector<std::string>> &options) -> size_t {
+
+  size_t loop = 1;
+
+  if (options.contains("loop")) {
+    std::stringstream sstream(options.at("loop").front());
+    sstream >> loop;
+  }
+
+  return loop;
+}
+
+// TODO: Add a flag to emit output using the standard JSON Schema output format
+// TODO: Add a flag to collect annotations
+// TODO: Add a flag to take a pre-compiled schema as input
+
 auto sourcemeta::jsonschema::cli::validate(
     const std::span<const std::string> &arguments) -> int {
   const auto options{
@@ -106,6 +125,7 @@ auto sourcemeta::jsonschema::cli::validate(
 
   const auto fast_mode{options.contains("f") || options.contains("fast")};
   const auto benchmark{options.contains("b") || options.contains("benchmark")};
+  const auto bench_loop{benchmark_loop(options)};
   const auto trace{options.contains("t") || options.contains("trace")};
   const auto json_output{options.contains("j") || options.contains("json")};
 
@@ -155,6 +175,7 @@ auto sourcemeta::jsonschema::cli::validate(
             const auto duration_us{
                 std::chrono::duration_cast<std::chrono::microseconds>(
                     timestamp_end - timestamp_start)};
+
             if (subresult) {
               std::cout << "took: " << duration_us.count() << "us\n";
             } else {
@@ -192,9 +213,8 @@ auto sourcemeta::jsonschema::cli::validate(
           } else if (subresult) {
             log_verbose(options)
                 << "ok: " << safe_weakly_canonical(instance_path).string()
-                << " (entry #" << index << ")"
-                << "\n  matches " << safe_weakly_canonical(schema_path).string()
-                << "\n";
+                << " (entry #" << index << ")" << "\n  matches "
+                << safe_weakly_canonical(schema_path).string() << "\n";
             print_annotations(output, options, std::cerr);
           } else {
             std::cerr << "fail: "
@@ -226,15 +246,49 @@ auto sourcemeta::jsonschema::cli::validate(
           sourcemeta::core::empty_weak_pointer, frame};
       bool subresult{true};
       if (benchmark) {
-        const auto timestamp_start{std::chrono::high_resolution_clock::now()};
-        subresult = evaluator.validate(schema_template, instance);
-        const auto timestamp_end{std::chrono::high_resolution_clock::now()};
-        const auto duration_us{
-            std::chrono::duration_cast<std::chrono::microseconds>(
-                timestamp_end - timestamp_start)};
-        if (subresult) {
-          std::cout << "took: " << duration_us.count() << "us\n";
-        } else {
+        double sum = 0.0, sum2 = 0.0, empty = 0.0;
+
+        // overhead evaluation, if the compiler is kind enough not to optimize
+        // this out!
+        for (auto i = bench_loop; i; i--) {
+          const auto start{std::chrono::high_resolution_clock::now()};
+          const auto end{std::chrono::high_resolution_clock::now()};
+          empty +=
+              (double)(std::chrono::duration_cast<std::chrono::nanoseconds>(
+                           end - start))
+                  .count() /
+              1000.0;
+        }
+        empty /= (double)bench_loop;
+
+        // execution time evaluation
+        for (auto i = bench_loop; i; i--) {
+          const auto start{std::chrono::high_resolution_clock::now()};
+
+          // force fast evaluation
+          subresult = evaluator.validate(schema_template, instance);
+
+          const auto end{std::chrono::high_resolution_clock::now()};
+          const auto delay =
+              (double)(std::chrono::duration_cast<std::chrono::nanoseconds>(
+                           end - start))
+                      .count() /
+                  1000.0 -
+              empty;
+
+          sum += delay;
+          sum2 += delay * delay;
+        }
+
+        // compute and show average execution time and standard deviation
+        auto avg = sum / (double)bench_loop;
+        auto stdev = sqrt(sum2 / (double)bench_loop - avg * avg);
+        std::cout << std::fixed;
+        std::cout.precision(3);
+        std::cout << "took: " << avg << " +- " << stdev << " us (" << empty
+                  << ")\n";
+
+        if (!subresult) {
           error << "error: Schema validation failure\n";
           result = false;
         }

src/main.cc

@@ -35,14 +35,15 @@ Global Options:
    validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...> [--http/-h]
             [--benchmark/-b] [--extension/-e <extension>]
             [--ignore/-i <schemas-or-directories>] [--trace/-t] [--fast/-f]
-            [--template/-m <template.json>] [--json/-j]
+            [--template/-m <template.json>] [--json/-j] [--loop <iterations>]
 
        Validate one or more instances against the given schema.
 
        By default, schemas are validated in exhaustive mode, which results in
        better error messages, at the expense of speed. The --fast/-f option
        makes the schema compiler optimise for speed, at the expense of error
-       messages.
+       messages. Looping in benchmark mode allows to collect the execution time
+       average and standard deviation for the fast mode on basic JSON data.
 
        You may additionally pass a pre-compiled schema template (see the
        `compile` command). However, you still need to pass the original schema

src/utils.cc

@@ -185,8 +185,8 @@ auto parse_options(const std::span<const std::string> &arguments,
   return options;
 }
 
-auto print(const sourcemeta::blaze::SimpleOutput &output, std::ostream &stream)
-    -> void {
+auto print(const sourcemeta::blaze::SimpleOutput &output,
+           std::ostream &stream) -> void {
   stream << "error: Schema validation failure\n";
   output.stacktrace(stream, "  ");
 }
@@ -211,8 +211,8 @@ auto print_annotations(
 }
 
 // TODO: Move this as an operator<< overload for TraceOutput in Blaze itself
-auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
-    -> void {
+auto print(const sourcemeta::blaze::TraceOutput &output,
+           std::ostream &stream) -> void {
   for (auto iterator = output.cbegin(); iterator != output.cend(); iterator++) {
     const auto &entry{*iterator};
 
@@ -377,9 +377,8 @@ auto log_verbose(const std::map<std::string, std::vector<std::string>> &options)
   return null_stream;
 }
 
-auto parse_extensions(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::set<std::string> {
+auto parse_extensions(const std::map<std::string, std::vector<std::string>>
+                          &options) -> std::set<std::string> {
   std::set<std::string> result;
 
   if (options.contains("extension")) {
@@ -405,9 +404,8 @@ auto parse_extensions(
   return result;
 }
 
-auto parse_ignore(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::set<std::filesystem::path> {
+auto parse_ignore(const std::map<std::string, std::vector<std::string>>
+                      &options) -> std::set<std::filesystem::path> {
   std::set<std::filesystem::path> result;
 
   if (options.contains("ignore")) {
@@ -436,9 +434,8 @@ auto safe_weakly_canonical(const std::filesystem::path &input)
              : std::filesystem::weakly_canonical(input);
 }
 
-auto default_dialect(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::optional<std::string> {
+auto default_dialect(const std::map<std::string, std::vector<std::string>>
+                         &options) -> std::optional<std::string> {
 
   if (options.contains("default-dialect")) {
     return options.at("default-dialect").front();

src/utils.h

@@ -44,16 +44,16 @@ auto for_each_json(const std::vector<std::string> &arguments,
                    const std::set<std::string> &extensions)
     -> std::vector<std::pair<std::filesystem::path, sourcemeta::core::JSON>>;
 
-auto print(const sourcemeta::blaze::SimpleOutput &output, std::ostream &stream)
-    -> void;
+auto print(const sourcemeta::blaze::SimpleOutput &output,
+           std::ostream &stream) -> void;
 
 auto print_annotations(
     const sourcemeta::blaze::SimpleOutput &output,
     const std::map<std::string, std::vector<std::string>> &options,
     std::ostream &stream) -> void;
 
-auto print(const sourcemeta::blaze::TraceOutput &output, std::ostream &stream)
-    -> void;
+auto print(const sourcemeta::blaze::TraceOutput &output,
+           std::ostream &stream) -> void;
 
 auto resolver(const std::map<std::string, std::vector<std::string>> &options,
               const bool remote,
@@ -63,20 +63,17 @@ auto resolver(const std::map<std::string, std::vector<std::string>> &options,
 auto log_verbose(const std::map<std::string, std::vector<std::string>> &options)
     -> std::ostream &;
 
-auto parse_extensions(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::set<std::string>;
+auto parse_extensions(const std::map<std::string, std::vector<std::string>>
+                          &options) -> std::set<std::string>;
 
-auto parse_ignore(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::set<std::filesystem::path>;
+auto parse_ignore(const std::map<std::string, std::vector<std::string>>
+                      &options) -> std::set<std::filesystem::path>;
 
 auto safe_weakly_canonical(const std::filesystem::path &input)
     -> std::filesystem::path;
 
-auto default_dialect(
-    const std::map<std::string, std::vector<std::string>> &options)
-    -> std::optional<std::string>;
+auto default_dialect(const std::map<std::string, std::vector<std::string>>
+                         &options) -> std::optional<std::string>;
 
 } // namespace sourcemeta::jsonschema::cli