Skip to content

Commit 5d02095

Browse files
Mic92Mic92
authored
Merge pull request #790 from helsinki-systems/feat/init-queue-runner-v2
feat: init hydra-queue-runner-v2
2 parents 31affcb + 285a237 commit 5d02095

File tree

13 files changed

+999
-48
lines changed

13 files changed

+999
-48
lines changed

‎dns/nixos.org.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ D("nixos.org",
9696
// hydra staging area
9797
A("staging-hydra", "157.180.25.203"),
9898
AAAA("staging-hydra", "2a01:4f9:c012:d5d3::1"),
99+
CNAME("queue-runner.staging-hydra", "staging-hydra"),
99100

100101
// services infra
101102
A("caliban", "65.109.26.213"),

‎non-critical-infra/hosts/staging-hydra/ca.crt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIBnTCCAU+gAwIBAgIUQpxYsPwAyTY70yYO9fcCmCaZreIwBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBDMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExHjAcBgNVBAMM
6+
FWh5ZHJhLXF1ZXVlLXJ1bm5lci1jYTAqMAUGAytlcAMhAM+Mc/XSTXwJeWPxrpqo
7+
SPT5Xwi8/j85VO6TsfBlXFt4o1MwUTAdBgNVHQ4EFgQU0wQG6BxTKtYwlywuyD0a
8+
Vr/1r4gwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywuyD0aVr/1r4gwDwYDVR0TAQH/
9+
BAUwAwEB/zAFBgMrZXADQQA3BRP2+TkkDQPnPy6MQyDCxqfEeV6OQjtspSvCO0UL
10+
GWmfvzrlUQytwTFTPfVzaErbyVPbeYU5y8rmRoGPNSoI
11+
-----END CERTIFICATE-----

‎non-critical-infra/hosts/staging-hydra/client.crt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIBlDCCAUagAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYwwBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBLMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExJjAkBgNVBAMM
6+
HWh5ZHJhLXF1ZXVlLWJ1aWxkZXItbG9jYWxob3N0MCowBQYDK2VwAyEAJqCvjdwY
7+
0l4z0cbwecONx3DfhpPo+SBymv7H2OQt4aejQjBAMB0GA1UdDgQWBBQw54bw+CrK
8+
O53J4qzB4Cknzf7CzTAfBgNVHSMEGDAWgBTTBAboHFMq1jCXLC7IPRpWv/WviDAF
9+
BgMrZXADQQB+XUCZQo0aX6IyEQDrds1d3SWF1rFBv5MjtZS25TYpnQQPgt9iiD6Q
10+
dtCwSVqyAgCG4XlkYBpU4AKVKqyMsSAC
11+
-----END CERTIFICATE-----

‎non-critical-infra/hosts/staging-hydra/default.nix‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
inputs.srvos.nixosModules.server
66
inputs.srvos.nixosModules.hardware-hetzner-cloud-arm
77
../../modules/common.nix
8+
../../modules/hydra-queue-runner-v2.nix
9+
../../modules/hydra-queue-builder-v2.nix
810
./hydra-proxy.nix
911
./hydra.nix
1012
inputs.hydra.nixosModules.hydra

‎non-critical-infra/hosts/staging-hydra/genca.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
set -x
4+
5+
hosts="localhost ofborg-eval02 ofborg-eval03 ofborg-eval04 ofborg-build01 ofborg-build02 ofborg-build03 ofborg-build04 ofborg-build05"
6+
7+
C="DE"
8+
O="NixOS Infra"
9+
10+
newDir="$(date '+%Y-%m-%dT%H:%M')"
11+
mkdir "${newDir}"
12+
cd "${newDir}" || exit
13+
14+
openssl genpkey -algorithm Ed25519 -out ca.key
15+
openssl req -x509 -new -nodes -key ca.key -sha256 -days 18250 -out ca.crt \
16+
-subj "/C=${C}/O=${O}/CN=hydra-queue-runner-ca"
17+
18+
cat <<EOF >server.cnf
19+
[req]
20+
prompt = no
21+
x509_extensions = v3_req
22+
req_extensions = v3_req
23+
default_md = sha256
24+
distinguished_name = req_distinguished_name
25+
26+
[req_distinguished_name]
27+
C = ${C}
28+
O = ${O}
29+
CN = queue-runner.staging-hydra.nixos.org
30+
31+
[v3_req]
32+
basicConstraints = CA:FALSE
33+
keyUsage = nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
34+
extendedKeyUsage = critical, serverAuth
35+
subjectAltName = @alt_names
36+
37+
[alt_names]
38+
DNS.1 = queue-runner.staging-hydra.nixos.org
39+
EOF
40+
41+
openssl genpkey -algorithm Ed25519 -out server.key
42+
openssl req -new -key server.key -out server.csr -config server.cnf
43+
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 18250 -sha256 -extfile server.cnf -extensions v3_req
44+
45+
for host in ${hosts}; do
46+
openssl genpkey -algorithm Ed25519 -out "client-${host}.key"
47+
openssl req -new -key "client-${host}.key" -out "client-${host}.csr" \
48+
-subj "/C=${C}/O=${O}/CN=hydra-queue-builder-${host}"
49+
openssl x509 -req -in "client-${host}.csr" -CA ca.crt -CAkey ca.key -CAcreateserial -out "client-${host}.crt" -days 18250 -sha256
50+
done
51+
52+
rm -rf -- *.csr *.srl
53+
rm server.cnf
54+
55+
cd - || exit

‎non-critical-infra/hosts/staging-hydra/hydra.nix‎

Lines changed: 122 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,11 @@ in
2424
dates = "03,09,15,21:15";
2525
};
2626

27-
# gc outputs as well, since they are served from the cache
28-
nix.settings.gc-keep-outputs = lib.mkForce false;
27+
nix.settings = {
28+
# gc outputs as well, since they are served from the cache
29+
gc-keep-outputs = lib.mkForce false;
30+
allowed-users = [ "hydra-www" ];
31+
};
2932

3033
# Don't rate-limit the journal.
3134
services.journald.rateLimitBurst = 0;
@@ -44,67 +47,136 @@ in
4447
};
4548
};
4649

47-
services.hydra-dev = {
48-
enable = true;
49-
package = pkgs.hydra;
50-
buildMachinesFiles = [
51-
(pkgs.writeText "local" ''
52-
localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
53-
'')
54-
];
55-
logo = ../../../build/hydra-logo.png;
56-
hydraURL = "https://hydra.nixos.org";
57-
notificationSender = "[email protected]";
58-
smtpHost = "localhost";
59-
useSubstitutes = true;
60-
extraConfig = ''
61-
max_servers 30
50+
services = {
51+
hydra-dev = {
52+
enable = true;
53+
package = pkgs.hydra;
54+
buildMachinesFiles = [
55+
(pkgs.writeText "local" ''
56+
localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
57+
'')
58+
];
59+
logo = ../../../build/hydra-logo.png;
60+
hydraURL = "https://hydra.nixos.org";
61+
notificationSender = "[email protected]";
62+
smtpHost = "localhost";
63+
useSubstitutes = true;
64+
extraConfig = ''
65+
max_servers 30
6266
63-
store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
64-
server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
65-
binary_cache_public_uri = https://cache-staging.nixos.org
67+
store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
68+
server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
69+
binary_cache_public_uri = https://cache-staging.nixos.org
6670
67-
<Plugin::Session>
68-
cache_size = 32m
69-
</Plugin::Session>
71+
<Plugin::Session>
72+
cache_size = 32m
73+
</Plugin::Session>
7074
71-
# patchelf:master:3
72-
xxx-jobset-repeats = nixos:reproducibility:1
75+
# patchelf:master:3
76+
xxx-jobset-repeats = nixos:reproducibility:1
7377
74-
upload_logs_to_binary_cache = true
75-
compress_build_logs = false # conflicts with upload_logs_to_binary_cache
78+
upload_logs_to_binary_cache = true
79+
compress_build_logs = false # conflicts with upload_logs_to_binary_cache
7680
77-
log_prefix = https://cache.nixos.org/
81+
log_prefix = https://cache.nixos.org/
7882
79-
evaluator_workers = 1
80-
evaluator_max_memory_size = 4096
83+
evaluator_workers = 1
84+
evaluator_max_memory_size = 4096
8185
82-
max_concurrent_evals = 1
86+
queue_runner_endpoint = http://localhost:8080
8387
84-
# increase the number of active compress slots (CPU is 48*2 on mimas)
85-
max_local_worker_threads = 144
88+
max_concurrent_evals = 1
8689
87-
max_unsupported_time = 86400
90+
# increase the number of active compress slots (CPU is 48*2 on mimas)
91+
max_local_worker_threads = 144
8892
89-
allow_import_from_derivation = false
93+
max_unsupported_time = 86400
9094
91-
max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
92-
max_db_connections = 350
95+
allow_import_from_derivation = false
9396
94-
queue_runner_metrics_address = [::]:9198
97+
max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
98+
max_db_connections = 350
9599
96-
<hydra_notify>
97-
<prometheus>
98-
listen_address = 0.0.0.0
99-
port = 9199
100-
</prometheus>
101-
</hydra_notify>
102-
'';
100+
queue_runner_metrics_address = [::]:9198
101+
102+
<hydra_notify>
103+
<prometheus>
104+
listen_address = 0.0.0.0
105+
port = 9199
106+
</prometheus>
107+
</hydra_notify>
108+
'';
109+
};
110+
111+
hydra-queue-runner-v2 = {
112+
enable = true;
113+
settings.remoteStoreAddr = [
114+
"s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
115+
];
116+
};
117+
118+
hydra-queue-builder-v2 = {
119+
enable = true;
120+
queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org";
121+
mtls = {
122+
serverRootCaCertPath = "${./ca.crt}";
123+
clientCertPath = "${./client.crt}";
124+
clientKeyPath = config.sops.secrets."queue-runner-client.key".path;
125+
domainName = "queue-runner.staging-hydra.nixos.org";
126+
};
127+
};
128+
129+
nginx = {
130+
enable = true;
131+
virtualHosts."queue-runner.staging-hydra.nixos.org" = {
132+
extraConfig = ''
133+
ssl_client_certificate ${./ca.crt};
134+
ssl_verify_depth 2;
135+
ssl_verify_client on;
136+
'';
137+
138+
sslCertificate = ./server.crt;
139+
sslCertificateKey = config.sops.secrets."queue-runner-server.key".path;
140+
onlySSL = true;
141+
142+
locations."/".extraConfig = ''
143+
# This is necessary so that grpc connections do not get closed early
144+
# see https://stackoverflow.com/a/67805465
145+
client_body_timeout 31536000s;
146+
147+
grpc_pass grpc://[::1]:50051;
148+
149+
grpc_read_timeout 31536000s; # 1 year in seconds
150+
grpc_send_timeout 31536000s; # 1 year in seconds
151+
grpc_socket_keepalive on;
152+
153+
grpc_set_header Host $host;
154+
grpc_set_header X-Real-IP $remote_addr;
155+
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
156+
grpc_set_header X-Forwarded-Proto $scheme;
157+
158+
grpc_set_header X-Client-DN $ssl_client_s_dn;
159+
grpc_set_header X-Client-Cert $ssl_client_escaped_cert;
160+
'';
161+
};
162+
};
103163
};
104164

105-
sops.secrets.hydra-users = {
106-
sopsFile = ../../secrets/hydra-users.staging-hydra;
107-
format = "binary";
165+
sops.secrets = {
166+
"queue-runner-server.key" = {
167+
sopsFile = ../../secrets/queue-runner-server.key.staging-hydra;
168+
format = "binary";
169+
owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
170+
};
171+
"queue-runner-client.key" = {
172+
sopsFile = ../../secrets/queue-runner-client.key.staging-hydra;
173+
format = "binary";
174+
owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
175+
};
176+
hydra-users = {
177+
sopsFile = ../../secrets/hydra-users.staging-hydra;
178+
format = "binary";
179+
};
108180
};
109181

110182
systemd = {
@@ -117,6 +189,8 @@ in
117189
services = {
118190
hydra-notify.enable = false;
119191
hydra-queue-runner = {
192+
enable = false;
193+
120194
# restarting the scheduler is very expensive
121195
restartIfChanged = false;
122196
serviceConfig = {

‎non-critical-infra/hosts/staging-hydra/server.crt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIB/jCCAbCgAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYswBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBSMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExLTArBgNVBAMM
6+
JHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAqMAUGAytlcAMh
7+
ANVnDi5rY0Ar4hPbqRJqS+Nw7b5GTg0QxL2DM7l1xTqHo4GkMIGhMAkGA1UdEwQC
8+
MAAwCwYDVR0PBAQDAgPoMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMBMC8GA1UdEQQo
9+
MCaCJHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAdBgNVHQ4E
10+
FgQU4ArR8rzVAt6dFkSXiMUlYYAzbwUwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywu
11+
yD0aVr/1r4gwBQYDK2VwA0EAScS72oaQ8PcYpH26FuRGnKaWe4e7fQ5RmKBUyC+5
12+
CiYIWu4D7fNGYJ15szCfh4nJIuyB0eXBv1ddAGAQMVdhDw==
13+
-----END CERTIFICATE-----

0 commit comments

Comments
 (0)