add loop option

Author: Fabien Coelho

src/command_validate.cc

@@ -10,10 +10,26 @@
 #include <iostream> // std::cerr
 #include <set>      // std::set
 #include <string>   // std::string
+#include <cmath>    // for sqrt
 
 #include "command.h"
 #include "utils.h"
 
+// parse unsigned int option, how many loop for benchmarking
+auto benchmark_loop(
+    const std::map<std::string, std::vector<std::string>> &options)
+    -> size_t {
+
+  size_t loop = 1;
+
+  if (options.contains("loop")) {
+    std::stringstream sstream(options.at("loop").front());
+    sstream >> loop;
+  }
+
+  return loop;
+}
+
 // TODO: Add a flag to emit output using the standard JSON Schema output format
 // TODO: Add a flag to collect annotations
 // TODO: Add a flag to take a pre-compiled schema as input
@@ -56,6 +72,7 @@ auto sourcemeta::jsonschema::cli::validate(
 
   const auto fast_mode{options.contains("f") || options.contains("fast")};
   const auto benchmark{options.contains("b") || options.contains("benchmark")};
+  const auto bench_loop{benchmark_loop(options)};
   const auto trace{options.contains("t") || options.contains("trace")};
 
   const sourcemeta::core::JSON bundled{
@@ -103,6 +120,7 @@ auto sourcemeta::jsonschema::cli::validate(
             const auto duration_us{
                 std::chrono::duration_cast<std::chrono::microseconds>(
                     timestamp_end - timestamp_start)};
+
             if (subresult) {
               std::cout << "took: " << duration_us.count() << "us\n";
             } else {
@@ -158,15 +176,43 @@ auto sourcemeta::jsonschema::cli::validate(
           sourcemeta::core::empty_weak_pointer, frame};
       bool subresult{true};
       if (benchmark) {
-        const auto timestamp_start{std::chrono::high_resolution_clock::now()};
-        subresult = evaluator.validate(schema_template, instance);
-        const auto timestamp_end{std::chrono::high_resolution_clock::now()};
-        const auto duration_us{
-            std::chrono::duration_cast<std::chrono::microseconds>(
-                timestamp_end - timestamp_start)};
-        if (subresult) {
-          std::cout << "took: " << duration_us.count() << "us\n";
-        } else {
+        double sum = 0.0, sum2 = 0.0, empty = 0.0;
+        size_t count = 0;
+
+        // overhead evaluation, if the compiler is kind enough not to optimize this out!
+        for (auto i = bench_loop; i; i--)  {
+          const auto start{std::chrono::high_resolution_clock::now()};
+          count++;
+          const auto end{std::chrono::high_resolution_clock::now()};
+          empty += (double)
+            (std::chrono::duration_cast<std::chrono::microseconds>(end - start)).count();
+        }
+        assert(count == bench_loop);
+        empty /= (double) bench_loop;
+
+        // execution time evaluation
+        for (auto i = bench_loop; i; i--)  {
+          const auto start{std::chrono::high_resolution_clock::now()};
+
+          // force fast evaluation
+          subresult = evaluator.validate(schema_template, instance);
+
+          const auto end{std::chrono::high_resolution_clock::now()};
+          const auto delay = (double) (std::chrono::duration_cast<
+                std::chrono::microseconds>(end - start)).count() - empty;
+
+          sum += delay;
+          sum2 += delay * delay;
+        }
+
+        // compute and show average execution time and standard deviation
+        auto avg = sum / (double) bench_loop;
+        auto stdev = sqrt(sum2 / (double) bench_loop - avg * avg);
+        std::cout << std::fixed;
+        std::cout.precision(3);
+        std::cout << "took: " << avg << " +- " << stdev << " us (" << empty << ")\n";
+
+        if (!subresult) {
           error << "error: Schema validation failure\n";
           result = false;
         }

src/main.cc

@@ -35,13 +35,15 @@ Global Options:
    validate <schema.json|.yaml> <instance.json|.jsonl|.yaml...> [--http/-h]
             [--benchmark/-b] [--extension/-e <extension>]
             [--ignore/-i <schemas-or-directories>] [--trace/-t] [--fast/-f]
+            [--loop <iterations>]
 
        Validate one or more instances against the given schema.
 
        By default, schemas are validated in exhaustive mode, which results in
        better error messages, at the expense of speed. The --fast/-f option
        makes the schema compiler optimise for speed, at the expense of error
-       messages.
+       messages. Looping in benchmark mode allows to collect the execution time
+       average and standard deviation for the fast mode on basic JSON data.
 
    metaschema [schemas-or-directories...] [--http/-h]
               [--extension/-e <extension>]