Merge pull request #5983 from IntersectMBO/bench-commit

Bench refactoring and fixes

Author: mgmeier

bench/cardano-profile/app/cardano-profile.hs

@@ -16,7 +16,14 @@ import qualified Cardano.Benchmarking.Profile.Types as Types
 
 --------------------------------------------------------------------------------
 
-data Cli = Names | All | ByName String | LibMK | ToJson String | FromJson String
+data Cli =
+    Names
+  | NamesCloudNoEra
+  | All
+  | ByName String
+  | LibMK
+  | ToJson String
+  | FromJson String
 
 --------------------------------------------------------------------------------
 
@@ -26,6 +33,8 @@ main = do
   case cli of
     -- Print all profile names.
     Names -> BSL8.putStrLn $ Aeson.encode Profiles.names
+    -- Print all cloud profile (-nomadperf) names.
+    NamesCloudNoEra -> BSL8.putStrLn $ Aeson.encode Profiles.namesCloudNoEra
     -- Print a map with all profiles, with an optional overlay.
     All -> do
       obj <- lookupOverlay
@@ -76,6 +85,12 @@ cliParser = OA.hsubparser $
           (pure Names)
           (OA.fullDesc <> OA.header "names" <> OA.progDesc "All profiles names")
         )
+  <>
+      OA.command "names-cloud-noera"
+        (OA.info
+          (pure NamesCloudNoEra)
+          (OA.fullDesc <> OA.header "names" <> OA.progDesc "All cloud profiles names (no era suffix)")
+        )
   <>
       OA.command "all"
         (OA.info

bench/cardano-profile/src/Cardano/Benchmarking/Profile.hs

@@ -4,7 +4,7 @@
 --------------------------------------------------------------------------------
 
 module Cardano.Benchmarking.Profile (
-  names, namesNoEra
+  names, namesNoEra, namesCloudNoEra
 , byName
 , profiles
 , libMk
@@ -51,6 +51,10 @@ names :: [String]
 -- Overlay not supported here, using an empty overlay.
 names = Map.keys (profiles mempty)
 
+namesCloudNoEra :: [String]
+-- Overlay not supported here, using an empty overlay.
+namesCloudNoEra = map Types.name profilesNoEraCloud
+
 -- Names:
 -- wb profile all-profiles | jq .[] | jq -r .name | sort | uniq | grep "\-bage"
 namesNoEra :: [String]

flake.nix

@@ -211,7 +211,7 @@
                               workbenchStartArgs = [ "--create-testnet-data" "--trace" ];
                             };
 
-            inherit (pkgs) all-profiles-json;
+            inherit (pkgs) all-profiles-json profile-data-nomadperf;
 
             system-tests = pkgs.writeShellApplication {
               name = "system-tests";

nix.mk

@@ -4,14 +4,6 @@ bump-node-measured:
 	nix flake lock --update-input node-measured
 bump-cardano-deployment: ## Sync the flake.lock to the CI check
 	nix run nixpkgs#nixUnstable -- build .#hydraJobs.cardano-deployment
-membench-1:    ## Membench:  one iteration, current commit
-	nix build .#membench-node-this-1.batch-report      --out-link result-batch-1-report
-membench-1-at: ## Membench:  one iteration, set commit by:  make membench-1-at REV=[master]
-	nix build .#membench-node-measured-1.batch-report  --out-link result-batch-1-report --override-input node-measured github:intersectmbo/cardano-node/${REV}
-membench-5:    ## Membench:  5 iterations, current commit
-	nix build .#membench-node-this-5.batch-report      --out-link result-batch-5-report
-membench-5-at: ## Membench:  5 iterations, set commit by:  make membench-5-at REV=[master]
-	nix build .#membench-node-this-5.batch-report      --out-link result-batch-5-report --override-input node-measured github:intersectmbo/cardano-node/${REV}
 
 CI_TEST_NIXATTR = workbench-ci-test
 workbench-ci-test smoke: ## Workbench:  test a-la Hydra, the ci-test profile, full Nix engaged

nix/custom-config.nix

@@ -5,7 +5,7 @@ self: {
     cacheDir    = "${self.localCluster.stateDir}/.cache";
     stateDir    = "run/current";
     batchName   = "undefined";
-    profileName = "default-bage";
+    profileName = "default-coay";
     backendName = "supervisor";
     basePort    = 30000;
     workbenchDevMode = true;

nix/pkgs.nix

@@ -2,64 +2,36 @@
 final: prev:
 
 let
-  inherit (prev) customConfig;
-  inherit (final) pkgs cardanoNodePackages cardanoNodeProject;
+  inherit (final) pkgs;
   inherit (prev.pkgs) lib;
+  inherit (prev) customConfig;
 
-  # A generic, fully parameteric version of the workbench development environment.
-  workbench = pkgs.callPackage ./workbench {};
+  # A generic, fully parametric version of the workbench development environment.
+  workbench = import ./workbench
+    {inherit pkgs lib; inherit (final) cardanoNodePackages cardanoNodeProject;};
 
-  # A conveniently-parametrisable workbench preset.
-  # See https://input-output-hk.github.io/haskell.nix/user-guide/development/
-  # The general idea is:
-  # 1. backendName -> stateDir -> basePort -> useCabalRun -> backend
-  # 2. batchName -> profileName -> profiling -> backend -> workbench -> runner
-  # * `workbench` is in case a pinned version of the workbench is needed.
+  # Workbench runner instantiated by parameters from customConfig:
   workbench-runner =
-  let
-    backendRegistry =
-      {
-        nomadcloud      = params:
-          import ./workbench/backend/nomad/cloud.nix  params;
-        nomadexec       = params:
-          import ./workbench/backend/nomad/exec.nix   params;
-        supervisor      = params:
-          import ./workbench/backend/supervisor.nix   params;
-      }
-    ;
-  in
-    { stateDir           ? customConfig.localCluster.stateDir
-    , batchName          ? customConfig.localCluster.batchName
-    , profileName        ? customConfig.localCluster.profileName
+    { profileName        ? customConfig.localCluster.profileName
+    , profiling          ? customConfig.profiling
     , backendName        ? customConfig.localCluster.backendName
+    , stateDir           ? customConfig.localCluster.stateDir
     , basePort           ? customConfig.localCluster.basePort
     , useCabalRun        ? customConfig.localCluster.useCabalRun
     , workbenchDevMode   ? customConfig.localCluster.workbenchDevMode
+    , batchName          ? customConfig.localCluster.batchName
     , workbenchStartArgs ? customConfig.localCluster.workbenchStartArgs
-    , profiling          ? customConfig.profiling
     , cardano-node-rev   ? null
-    , workbench          ? pkgs.workbench
     }:
-    let
-        # The `useCabalRun` flag is set in the backend to allow the backend to
-        # override its value. The runner uses the value of `useCabalRun` from
-        # the backend to prevent a runner using a different value.
-        backend = (backendRegistry."${backendName}")
-                   { inherit pkgs lib stateDir basePort useCabalRun; };
-    in import ./workbench/backend/runner.nix
-      {
-        inherit pkgs lib cardanoNodePackages;
-        inherit batchName profileName backend;
-        inherit cardano-node-rev;
-        inherit workbench workbenchDevMode workbenchStartArgs profiling;
+    workbench.runner
+      { inherit profileName profiling backendName stateDir basePort useCabalRun;
+        inherit batchName workbenchDevMode workbenchStartArgs cardano-node-rev;
       };
 
-  # Workbench instantiated by parameters from customConfig:
-  custom-config-workbench-runner = workbench-runner {};
-
 in with final;
 {
   inherit (cardanoNodeProject.args) compiler-nix-name;
+
   inherit workbench workbench-runner;
 
   cabal = haskell-nix.cabal-install.${compiler-nix-name};
@@ -156,6 +128,51 @@ in with final;
 
   all-profiles-json = workbench.profile-names-json;
 
+  # The profile data and backend data of the cloud / "*-nomadperf" profiles.
+  # Useful to mix workbench and cardano-node commits, mostly because of scripts.
+  profile-data-nomadperf = builtins.listToAttrs (
+    builtins.map
+    (cloudName:
+      # Only Conway era cloud profiles are flake outputs.
+      let profileName = "${cloudName}-coay";
+      in {
+        name = profileName;
+        value =
+          let
+              # Default values only ("run/current", 30000, profiling "none").
+              profile = workbench.profile {
+                inherit profileName;
+                inherit (customConfig) profiling;
+              };
+              backend = workbench.backend
+                { backendName = "nomadcloud";
+                  stateDir    = customConfig.localCluster.stateDir;
+                  basePort    = customConfig.localCluster.basePort;
+                  useCabalRun = customConfig.localCluster.useCabalRun;
+                }
+              ;
+              profileData = profile.materialise-profile
+                { inherit backend; }
+              ;
+              backendData = backend.materialise-profile {inherit profileData;};
+          in pkgs.runCommand "workbench-data-${profileName}" {}
+            ''
+            mkdir $out
+            ln -s ${profileData} $out/profileData
+            ln -s ${backendData} $out/backendData
+            ''
+        ;
+        }
+    )
+    # Fetch all "*-nomadperf" profiles.
+    (__fromJSON (__readFile
+      (pkgs.runCommand "cardano-profile-names-cloud-noera" {} ''
+        ${cardanoNodePackages.cardano-profile}/bin/cardano-profile names-cloud-noera > $out
+      ''
+      )
+    ))
+  );
+
   # Disable failing python uvloop tests
   python39 = prev.python39.override {
     packageOverrides = pythonFinal: pythonPrev: {
@@ -164,5 +181,4 @@ in with final;
       });
     };
   };
-} //
-custom-config-workbench-runner.overlay final prev
+}

nix/workbench/analyse/analyse.nix

@@ -41,12 +41,6 @@ let
     in (import ../nomad.nix params).materialise-profile
   ;
 
-  overlay =
-    proTopo: self: super:
-    {
-    }
-  ;
-
   service-modules = {
     node = { config, ... }:
       let selfCfg = config.services.cardano-node;
@@ -63,7 +57,7 @@ in
 
   inherit extraShellPkgs;
   inherit materialise-profile;
-  inherit overlay service-modules;
+  inherit service-modules;
   inherit stateDir basePort;
 
   inherit useCabalRun;

nix/workbench/backend/nomad/cloud.nix

@@ -39,12 +39,6 @@ let
     in (import ../nomad.nix params).materialise-profile
   ;
 
-  overlay =
-    proTopo: self: super:
-    {
-    }
-  ;
-
   service-modules = {
     node = { config, ... }:
       let selfCfg = config.services.cardano-node;
@@ -61,7 +55,7 @@ in
 
   inherit extraShellPkgs;
   inherit materialise-profile;
-  inherit overlay service-modules;
+  inherit service-modules;
   inherit stateDir basePort;
 
   inherit useCabalRun;