hydra-queue-runner: 54b3c935 -> 3e4c33bf

Author: Conni2461

non-critical-infra/hosts/staging-hydra/hydra.nix

@@ -37,7 +37,10 @@ in
   nix.settings = {
     # gc outputs as well, since they are served from the cache
     gc-keep-outputs = lib.mkForce false;
-    allowed-users = [ "hydra" "hydra-www" ];
+    allowed-users = [
+      "hydra"
+      "hydra-www"
+    ];
   };
 
   # Don't rate-limit the journal.
@@ -117,9 +120,13 @@ in
 
     hydra-queue-runner-v2 = {
       enable = true;
-      settings.remoteStoreAddr = [
-        "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
-      ];
+      settings = {
+        queueTriggerTimerInS = 300;
+        concurrentUploadLimit = 2;
+        remoteStoreAddr = [
+          "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
+        ];
+      };
     };
 
     nginx = {

non-critical-infra/modules/hydra-queue-builder-v2.nix

@@ -2,10 +2,12 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-builder-v2;
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -98,6 +100,12 @@ in
         default = true;
       };
 
+      authorizationFile = lib.mkOption {
+        description = "Path to token authorization file if token auth should be used.";
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+      };
+
       mtls = lib.mkOption {
         description = "mtls options";
         default = null;
@@ -127,7 +135,12 @@ in
 
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).builder;
       };
     };
   };
@@ -197,6 +210,10 @@ in
           ++ lib.optionals (cfg.useSubstitutes != null) [
             "--use-substitutes"
           ]
+          ++ lib.optionals (cfg.authorizationFile != null) [
+            "--authorization-file"
+            cfg.authorizationFile
+          ]
           ++ lib.optionals (cfg.mtls != null) [
             "--server-root-ca-cert-path"
             cfg.mtls.serverRootCaCertPath

non-critical-infra/modules/hydra-queue-runner-v2.nix

@@ -2,12 +2,14 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-runner-v2;
 
   format = pkgs.formats.toml { };
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -101,6 +103,21 @@ in
               type = lib.types.int;
               default = 60;
             };
+            maxConcurrentDownloads = lib.mkOption {
+              description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            concurrentUploadLimit = lib.mkOption {
+              description = "Concurrent limit for uploading to s3.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            tokenListPath = lib.mkOption {
+              description = "Path to a list of allowed authentication tokens.";
+              type = lib.types.nullOr lib.types.path;
+              default = null;
+            };
           };
         };
         default = { };
@@ -170,7 +187,12 @@ in
       };
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).runner;
       };
     };
   };

non-critical-infra/packages/hydra-queue-runner/default.nix

@@ -1,5 +1,5 @@
 {
-  rustPackages,
+  rustPackages_1_88,
   fetchFromGitHub,
   pkg-config,
   openssl,
@@ -11,17 +11,19 @@
   nlohmann_json,
   libsodium,
   boost,
+  withOtel ? false,
+  withTokioConsole ? false,
 }:
 let
-  version = "unstable-2025-08-07";
+  version = "unstable-2025-10-17";
   src = fetchFromGitHub {
     owner = "helsinki-systems";
     repo = "hydra-queue-runner";
-    rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca";
-    hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0=";
+    rev = "3e4c33bf76139f752a0a4384e4cf237fdacba5b1";
+    hash = "sha256-dIkOSBkFs/fBeqIaheSs2KU90Euak/XRc0dm+Ju7cHM=";
   };
   useFetchCargoVendor = true;
-  cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8=";
+  cargoHash = "sha256-mElkthAG8AwdnR1ea+FMY4wUc+ZrzAZK68GyLSjFh6Q=";
   nativeBuildInputs = [
     pkg-config
     protobuf
@@ -32,7 +34,7 @@ let
     zlib
     protobuf
 
-    nixVersions.nix_2_29
+    nixVersions.nix_2_31
     nlohmann_json
     libsodium
     boost
@@ -46,7 +48,7 @@ let
   };
 in
 {
-  runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  runner = rustPackages_1_88.rustPlatform.buildRustPackage {
     pname = "hydra-queue-runner";
     inherit version src;
     __structuredAttrs = true;
@@ -59,22 +61,21 @@ in
       buildInputs
       ;
 
-    cargoBuildFlags = [
-      "-p"
-      "queue-runner"
-    ];
-    cargoTestFlags = finalAttrs.cargoBuildFlags;
+    buildAndTestSubdir = "queue-runner";
+    buildFeatures = lib.optional withOtel "otel" ++ lib.optional withTokioConsole "tokio-console";
 
     postInstall = ''
-      wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/queue-runner \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {
       mainProgram = "queue-runner";
     };
-  });
+  };
 
-  builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  builder = rustPackages_1_88.rustPlatform.buildRustPackage {
     pname = "hydra-queue-builder";
     inherit src version;
     __structuredAttrs = true;
@@ -87,18 +88,16 @@ in
       buildInputs
       ;
 
-    cargoBuildFlags = [
-      "-p"
-      "builder"
-    ];
-    cargoTestFlags = finalAttrs.cargoBuildFlags;
+    buildAndTestSubdir = "builder";
 
     postInstall = ''
-      wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/builder \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {
       mainProgram = "builder";
     };
-  });
+  };
 }