Add more metrics for benchmarking. (#157)

Author: fruffy

core/control_plane/control_plane_objects.cpp

@@ -310,7 +310,7 @@ void TableConfiguration::setTableKeyMatch(const IR::Expression *tableKeyMatch) {
     _tableKeyMatch = tableKeyMatch;
     // When we set the table key match, we also need to recompute the match of all table entries.
     auto z3TableKeyMatch = Z3Cache::set(tableKeyMatch);
-    for (auto &tableMatchEntry : _tableEntries) {
+    for (const auto &tableMatchEntry : _tableEntries) {
         tableMatchEntry.get().setZ3Condition(z3TableKeyMatch);
     }
 }

core/lib/incremental_analysis.h

@@ -111,16 +111,15 @@ class IncrementalAnalysis : public ICastable {
     /// affect the semantics of the program, and specialize the program if necessary.
     std::optional<const IR::P4Program *> processControlPlaneUpdate(
         const IR::P4Program &program, const ControlPlaneUpdate &controlPlaneUpdate) {
+        printInfo("Processing 1 control plane update.");
         ASSIGN_OR_RETURN(SymbolSet symbolSet, convertControlPlaneUpdate(controlPlaneUpdate),
                          std::nullopt);
         ASSIGN_OR_RETURN(bool changeNeeded, checkForSemanticsChange(symbolSet), std::nullopt);
         printInfo("Change in semantics detected: %1%", changeNeeded ? "yes" : "no");
         if (!changeNeeded) {
             return std::optional{nullptr};
         }
-
-        auto newProgram = specializeProgram(program);
-        return newProgram;
+        return specializeProgram(program);
     }
 
     /// Receive a series of control plane updates, convert each update to its intermediate
@@ -143,7 +142,6 @@ class IncrementalAnalysis : public ICastable {
                 return std::optional{nullptr};
             }
         }
-
         return specializeProgram(program);
     }
 

core/specialization/flay_service.cpp

@@ -66,24 +66,33 @@ int FlayServiceBase::specializeProgram() {
 int FlayServiceBase::processControlPlaneUpdate(const ControlPlaneUpdate &controlPlaneUpdate) {
     Util::ScopedTimer timer("Processing control plane update");
     const auto *optimizedProg = &originalProgram();
+    _updateCount++;
+    bool hasRespecialized = false;
     for (const auto &[analysisName, incrementalAnalysis] : _incrementalAnalysisMap) {
         auto optProgram =
             incrementalAnalysis->processControlPlaneUpdate(*optimizedProg, controlPlaneUpdate);
         if (!optProgram.has_value()) {
             return EXIT_FAILURE;
         }
         if (optProgram.value() != nullptr) {
+            _respecializationCount++;
             optimizedProg = optProgram.value();
             _optimizedProgram = optimizedProg;
+            hasRespecialized = true;
         }
     }
+    if (hasRespecialized) {
+        _respecializationCount++;
+    }
     return EXIT_SUCCESS;
 }
 
 int FlayServiceBase::processControlPlaneUpdate(
     const std::vector<const ControlPlaneUpdate *> &controlPlaneUpdates) {
     Util::ScopedTimer timer("Processing control plane updates");
+    _updateCount += controlPlaneUpdates.size();
     const auto *optimizedProg = &originalProgram();
+    bool hasRespecialized = false;
     for (const auto &[analysisName, incrementalAnalysis] : _incrementalAnalysisMap) {
         auto optProgram =
             incrementalAnalysis->processControlPlaneUpdate(*optimizedProg, controlPlaneUpdates);
@@ -93,8 +102,12 @@ int FlayServiceBase::processControlPlaneUpdate(
         if (optProgram.value() != nullptr) {
             optimizedProg = optProgram.value();
             _optimizedProgram = optimizedProg;
+            hasRespecialized = true;
         }
     }
+    if (hasRespecialized) {
+        _respecializationCount++;
+    }
     return EXIT_SUCCESS;
 }
 
@@ -107,19 +120,19 @@ void FlayServiceBase::recordProgramChange() const {
               statementCountBefore, statementCountAfter, stmtPct);
 }
 
-std::vector<AnalysisStatistics *> FlayServiceBase::computeFlayServiceStatistics() const {
+FlayServiceStatisticsMap FlayServiceBase::computeFlayServiceStatistics() const {
     auto statementCountBefore = countStatements(midEndProgram());
     auto statementCountAfter = countStatements(optimizedProgram());
     auto cyclomaticComplexity = computeCyclomaticComplexity(midEndProgram());
     auto numParsersPaths = ParserPathsCounter::computeParserPaths(midEndProgram());
-    std::vector<AnalysisStatistics *> statistics;
-    statistics.reserve(_incrementalAnalysisMap.size() + 1);
+    FlayServiceStatisticsMap statistics;
     for (const auto &[analysisName, incrementalAnalysis] : _incrementalAnalysisMap) {
-        statistics.push_back(incrementalAnalysis->computeAnalysisStatistics());
+        statistics.emplace(analysisName, incrementalAnalysis->computeAnalysisStatistics());
     }
-    statistics.push_back(new FlayServiceStatistics(&optimizedProgram(), statementCountBefore,
-                                                   statementCountAfter, cyclomaticComplexity,
-                                                   numParsersPaths));
+    statistics.emplace(
+        "main", new FlayServiceStatistics(&optimizedProgram(), statementCountBefore,
+                                          statementCountAfter, cyclomaticComplexity,
+                                          numParsersPaths, updateCount(), respecializationCount()));
     return statistics;
 }
 

core/specialization/flay_service.h

@@ -47,37 +47,56 @@ inline double measureSizeDifference(const IR::P4Program &programBefore,
 struct FlayServiceStatistics : public AnalysisStatistics {
     FlayServiceStatistics(const IR::P4Program *optimizedProgram, uint64_t statementCountBefore,
                           uint64_t statementCountAfter, size_t cyclomaticComplexity,
-                          size_t numParsersPaths)
+                          size_t numParsersPaths, size_t numUpdatesProcessed,
+                          size_t numRespecializations)
         : optimizedProgram(optimizedProgram),
           statementCountBefore(statementCountBefore),
           statementCountAfter(statementCountAfter),
           cyclomaticComplexity(cyclomaticComplexity),
-          numParsersPaths(numParsersPaths) {}
+          numParsersPaths(numParsersPaths),
+          numUpdatesProcessed(numUpdatesProcessed),
+          numRespecializations(numRespecializations) {}
 
-    // The optimized program.
+    /// The optimized program.
     const IR::P4Program *optimizedProgram;
-    // The number of statements in the original program.
+    /// The number of statements in the original program.
     uint64_t statementCountBefore;
-    // The number of statements in the optimized program.
+    /// The number of statements in the optimized program.
     uint64_t statementCountAfter;
-    // The cyclomatic complexity of the input program.
+    /// The cyclomatic complexity of the input program.
     size_t cyclomaticComplexity;
-    // The total number of paths for parsers
+    /// The total number of paths for parsers
     size_t numParsersPaths;
+    /// The total number of updates processed.
+    size_t numUpdatesProcessed = 0;
+    /// The total number of times a respecialization was necessary.
+    size_t numRespecializations = 0;
 
     [[nodiscard]] std::string toFormattedString() const override {
         std::stringstream output;
         output << "\nstatement_count_before:" << statementCountBefore << "\n";
         output << "statement_count_after:" << statementCountAfter << "\n";
         output << "cyclomatic_complexity:" << cyclomaticComplexity << "\n";
         output << "num_parsers_paths:" << numParsersPaths << "\n";
+        output << "num_updates_processed:" << numUpdatesProcessed << "\n";
+        output << "num_respecializations:" << numRespecializations << "\n";
         return output.str();
     }
 
     DECLARE_TYPEINFO(FlayServiceStatistics);
 };
 
+/// Maps a particular specialization category to its statistics.
+using FlayServiceStatisticsMap = ordered_map<std::string, AnalysisStatistics *>;
+
 class FlayServiceBase {
+ private:
+    /// Number of updates processed.
+    size_t _updateCount = 0;
+
+    /// Number of times respecialization was necessary.
+    size_t _respecializationCount = 0;
+
  protected:
     /// The incremental analysis.
     IncrementalAnalysisMap _incrementalAnalysisMap;
@@ -94,6 +113,12 @@ class FlayServiceBase {
 
     int specializeProgram();
 
+    /// Return the number of updates processed.
+    [[nodiscard]] size_t updateCount() const { return _updateCount; }
+
+    /// Return the number of times respecialization was necessary.
+    [[nodiscard]] size_t respecializationCount() const { return _respecializationCount; }
+
  public:
     explicit FlayServiceBase(const FlayCompilerResult &compilerResult,
                              IncrementalAnalysisMap incrementalAnalysisMap);
@@ -121,7 +146,7 @@ class FlayServiceBase {
         const std::vector<const ControlPlaneUpdate *> &controlPlaneUpdates);
 
     /// Compute and return some statistics on the changes in the program.
-    [[nodiscard]] std::vector<AnalysisStatistics *> computeFlayServiceStatistics() const;
+    [[nodiscard]] FlayServiceStatisticsMap computeFlayServiceStatistics() const;
 };
 
 }  // namespace P4::P4Tools::Flay

core/specialization/service_wrapper.cpp

@@ -82,7 +82,7 @@ std::vector<std::string> FlayServiceWrapper::findFiles(std::string_view pattern)
     return files;
 }
 
-std::vector<AnalysisStatistics *> FlayServiceWrapper::computeFlayServiceStatistics() const {
+FlayServiceStatisticsMap FlayServiceWrapper::computeFlayServiceStatistics() const {
     return _flayService.computeFlayServiceStatistics();
 }
 

core/specialization/service_wrapper.h

@@ -36,7 +36,7 @@ class FlayServiceWrapper {
     void outputOptimizedProgram(const std::filesystem::path &optimizedOutputFile);
 
     /// Compute and return some statistics on the changes in the program.
-    [[nodiscard]] std::vector<AnalysisStatistics *> computeFlayServiceStatistics() const;
+    [[nodiscard]] FlayServiceStatisticsMap computeFlayServiceStatistics() const;
 };
 
 }  // namespace P4::P4Tools::Flay

flay.cpp

@@ -51,7 +51,7 @@ int runServer(const FlayOptions &flayOptions, const FlayCompilerResult &flayComp
 }
 #endif
 
-std::optional<std::vector<AnalysisStatistics *>> runServiceWrapper(
+std::optional<FlayServiceStatisticsMap> runServiceWrapper(
     const FlayOptions &flayOptions, const FlayCompilerResult &compilerResult,
     IncrementalAnalysisMap incrementalAnalysisMap) {
     auto controlPlaneApi = flayOptions.controlPlaneApi();
@@ -159,7 +159,7 @@ int Flay::mainImpl(const CompilerResult &compilerResult) {
     return ::P4::errorCount() == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
 }
 
-std::optional<std::vector<AnalysisStatistics *>> optimizeProgramImpl(
+std::optional<FlayServiceStatisticsMap> optimizeProgramImpl(
     std::optional<std::reference_wrapper<const std::string>> program,
     const FlayOptions &flayOptions) {
     // Register supported Flay targets.
@@ -213,8 +213,8 @@ std::optional<std::vector<AnalysisStatistics *>> optimizeProgramImpl(
     return runServiceWrapper(flayOptions, flayCompilerResult, std::move(incrementalAnalysisMap));
 }
 
-std::optional<std::vector<AnalysisStatistics *>> Flay::optimizeProgram(
-    const std::string &program, const FlayOptions &flayOptions) {
+std::optional<FlayServiceStatisticsMap> Flay::optimizeProgram(const std::string &program,
+                                                              const FlayOptions &flayOptions) {
     try {
         return optimizeProgramImpl(program, flayOptions);
     } catch (const std::exception &e) {
@@ -226,8 +226,7 @@ std::optional<std::vector<AnalysisStatistics *>> Flay::optimizeProgram(
     return std::nullopt;
 }
 
-std::optional<std::vector<AnalysisStatistics *>> Flay::optimizeProgram(
-    const FlayOptions &flayOptions) {
+std::optional<FlayServiceStatisticsMap> Flay::optimizeProgram(const FlayOptions &flayOptions) {
     try {
         return optimizeProgramImpl(std::nullopt, flayOptions);
     } catch (const std::exception &e) {

flay.h

@@ -18,13 +18,12 @@ class Flay : public AbstractP4cTool<FlayOptions> {
     virtual ~Flay() = default;
 
     /// Analyse the given program and return an optimized version.
-    static std::optional<std::vector<AnalysisStatistics *>> optimizeProgram(
-        const std::string &program, const FlayOptions &flayOptions);
+    static std::optional<FlayServiceStatisticsMap> optimizeProgram(const std::string &program,
+                                                                   const FlayOptions &flayOptions);
 
     /// Open the program file specified in the compiler options, preprocess it, and return an
     /// optimized version.
-    static std::optional<std::vector<AnalysisStatistics *>> optimizeProgram(
-        const FlayOptions &flayOptions);
+    static std::optional<FlayServiceStatisticsMap> optimizeProgram(const FlayOptions &flayOptions);
 };
 
 }  // namespace P4::P4Tools::Flay

targets/bmv2/test/testdata/action-synth.ref

@@ -3,4 +3,6 @@ statement_count_before:1
 statement_count_after:1
 cyclomatic_complexity:0
 num_parsers_paths:1
+num_updates_processed:0
+num_respecializations:0
 

targets/bmv2/test/testdata/action-two-params.ref

@@ -4,4 +4,6 @@ statement_count_before:2
 statement_count_after:1
 cyclomatic_complexity:2
 num_parsers_paths:1
+num_updates_processed:0
+num_respecializations:0