hydra-queue-runner: 54b3c935 -> ad269245

Author: Conni2461

non-critical-infra/hosts/staging-hydra/hydra.nix

@@ -117,9 +117,13 @@ in
 
     hydra-queue-runner-v2 = {
       enable = true;
-      settings.remoteStoreAddr = [
-        "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
-      ];
+      settings = {
+        queueTriggerTimerInS = 300;
+        concurrentUploadLimit = 2;
+        remoteStoreAddr = [
+          "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
+        ];
+      };
     };
 
     nginx = {

non-critical-infra/modules/hydra-queue-builder-v2.nix

@@ -2,10 +2,12 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-builder-v2;
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -98,6 +100,12 @@ in
         default = true;
       };
 
+      authorizationFile = lib.mkOption {
+        description = "Path to token authorization file if token auth should be used.";
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+      };
+
       mtls = lib.mkOption {
         description = "mtls options";
         default = null;
@@ -127,7 +135,12 @@ in
 
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).builder;
       };
     };
   };
@@ -197,6 +210,10 @@ in
           ++ lib.optionals (cfg.useSubstitutes != null) [
             "--use-substitutes"
           ]
+          ++ lib.optionals (cfg.authorizationFile != null) [
+            "--authorization-file"
+            cfg.authorizationFile
+          ]
           ++ lib.optionals (cfg.mtls != null) [
             "--server-root-ca-cert-path"
             cfg.mtls.serverRootCaCertPath

non-critical-infra/modules/hydra-queue-runner-v2.nix

@@ -2,12 +2,14 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-runner-v2;
 
   format = pkgs.formats.toml { };
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -101,6 +103,21 @@ in
               type = lib.types.int;
               default = 60;
             };
+            maxConcurrentDownloads = lib.mkOption {
+              description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            concurrentUploadLimit = lib.mkOption {
+              description = "Concurrent limit for uploading to s3.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            tokenListPath = lib.mkOption {
+              description = "Path to a list of allowed authentication tokens.";
+              type = lib.types.nullOr lib.types.path;
+              default = null;
+            };
           };
         };
         default = { };
@@ -170,7 +187,12 @@ in
       };
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).runner;
       };
     };
   };

non-critical-infra/packages/hydra-queue-runner/default.nix

@@ -1,5 +1,5 @@
 {
-  rustPackages,
+  rustPackages_1_88,
   fetchFromGitHub,
   pkg-config,
   openssl,
@@ -13,15 +13,15 @@
   boost,
 }:
 let
-  version = "unstable-2025-08-07";
+  version = "unstable-2025-10-09";
   src = fetchFromGitHub {
     owner = "helsinki-systems";
     repo = "hydra-queue-runner";
-    rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca";
-    hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0=";
+    rev = "ad26924556d7ff6b4945a8f15c04271e8210e703";
+    hash = "sha256-hKZ//s4N+GPiKqRI8AzMO32HZOcJWYu6n5+uwhjx6vA=";
   };
   useFetchCargoVendor = true;
-  cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8=";
+  cargoHash = "sha256-eDkU+K2XvFU9OPhlPPcjqssabWwU8EqWzj7da0MuMAI=";
   nativeBuildInputs = [
     pkg-config
     protobuf
@@ -32,7 +32,7 @@ let
     zlib
     protobuf
 
-    nixVersions.nix_2_29
+    nixVersions.nix_2_31
     nlohmann_json
     libsodium
     boost
@@ -46,7 +46,7 @@ let
   };
 in
 {
-  runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  runner = rustPackages_1_88.rustPlatform.buildRustPackage (finalAttrs: {
     pname = "hydra-queue-runner";
     inherit version src;
     __structuredAttrs = true;
@@ -66,15 +66,17 @@ in
     cargoTestFlags = finalAttrs.cargoBuildFlags;
 
     postInstall = ''
-      wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/queue-runner \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {
       mainProgram = "queue-runner";
     };
   });
 
-  builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  builder = rustPackages_1_88.rustPlatform.buildRustPackage (finalAttrs: {
     pname = "hydra-queue-builder";
     inherit src version;
     __structuredAttrs = true;
@@ -94,7 +96,9 @@ in
     cargoTestFlags = finalAttrs.cargoBuildFlags;
 
     postInstall = ''
-      wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/builder \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {