From 8675aee25b8f87f3984cab0bbf83b6a188da51d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Feb 2025 21:07:16 -0500 Subject: [PATCH 1/5] WIP TEMP nix update, don't mere this! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not until https://github.com/NixOS/nix/pull/10748 lands Flake lock file updates: • Updated input 'nix': 'github:NixOS/nix/970942f45836172fda410a638853382952189eb9?narHash=sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0%3D' (2025-02-12) → 'github:NixOS/nix/5eade4825221d3284fc6555cb20de2c7aa171d72?narHash=sha256-n5kdS1C24tlJxDV6Wm1iBlyvGk%2Bp0gMXRcWVCAipYLs%3D' (2025-02-14) • Updated input 'nix-eval-jobs': 'github:Ericson2314/nix-eval-jobs/5e27c2724a4b07862e7ff1a198aa2ed68dea3e2c?narHash=sha256-7xgSdKnQW11eWd59MnpUNS%2BgwgtOJH2ShzLwByev3rg%3D' (2025-02-14) → 'github:Ericson2314/nix-eval-jobs/de345eb4518d952c2d86261b270f2c31edecd3de?narHash=sha256-dNMJY6%2BG3PwE8lIAhwetPJdA2DxCEKRXPY/EtHmdDh4%3D' (2025-02-14) --- flake.lock | 20 +++++++++++--------- flake.nix | 5 +++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/flake.lock b/flake.lock index ec9a00e63..8217d1685 100644 --- a/flake.lock +++ b/flake.lock @@ -5,6 +5,7 @@ "flake-compat": [], "flake-parts": [], "git-hooks-nix": [], + "nixfmt": [], "nixpkgs": [ "nixpkgs" ], @@ -12,16 +13,16 @@ "nixpkgs-regression": [] }, "locked": { - "lastModified": 1739393420, - "narHash": "sha256-jGFuyYKJjJZsBRoi7ZcaVKt1OYxusz/ld1HA7VD2w/0=", + "lastModified": 1739570697, + "narHash": "sha256-n5kdS1C24tlJxDV6Wm1iBlyvGk+p0gMXRcWVCAipYLs=", "owner": "NixOS", "repo": "nix", - "rev": "970942f45836172fda410a638853382952189eb9", + "rev": "5eade4825221d3284fc6555cb20de2c7aa171d72", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.26-maintenance", + "ref": "legacy-ssh-extensions-for-hydra", "repo": "nix", "type": "github" } @@ -29,15 +30,16 @@ "nix-eval-jobs": { "flake": false, "locked": { - "lastModified": 1739500569, - "narHash": "sha256-3wIReAqdTALv39gkWXLMZQvHyBOc3yPkWT2ZsItxedY=", - "owner": "nix-community", + "lastModified": 1739499741, + "narHash": "sha256-dNMJY6+G3PwE8lIAhwetPJdA2DxCEKRXPY/EtHmdDh4=", + "owner": "Ericson2314", "repo": "nix-eval-jobs", - "rev": "4b392b284877d203ae262e16af269f702df036bc", + "rev": "de345eb4518d952c2d86261b270f2c31edecd3de", "type": "github" }, "original": { - "owner": "nix-community", + "owner": "Ericson2314", + "ref": "nix-2.27", "repo": "nix-eval-jobs", "type": "github" } diff --git a/flake.nix b/flake.nix index dc3aaf5cd..932bf1b93 100644 --- a/flake.nix +++ b/flake.nix @@ -4,7 +4,7 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small"; inputs.nix = { - url = "github:NixOS/nix/2.26-maintenance"; + url = "github:NixOS/nix/legacy-ssh-extensions-for-hydra"; inputs.nixpkgs.follows = "nixpkgs"; # hide nix dev tooling from our lock file @@ -13,10 +13,11 @@ inputs.nixpkgs-regression.follows = ""; inputs.nixpkgs-23-11.follows = ""; inputs.flake-compat.follows = ""; + inputs.nixfmt.follows = ""; }; inputs.nix-eval-jobs = { - url = "github:nix-community/nix-eval-jobs"; + url = "github:Ericson2314/nix-eval-jobs/nix-2.27"; # We want to control the deps precisely flake = false; }; From 4c173daec775e08a0a768021495a72208ec9d970 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 16:22:19 -0400 Subject: [PATCH 2/5] Use `LegacySSHStore` In https://github.com/NixOS/nix/pull/10748 it is extended with everything we need. --- src/hydra-queue-runner/build-remote.cc | 178 +++++++++---------------- src/hydra-queue-runner/state.hh | 8 +- 2 files changed, 67 insertions(+), 119 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 77bde2c45..e8cd6d2f9 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -9,13 +9,10 @@ #include "path.hh" #include "legacy-ssh-store.hh" #include "serve-protocol.hh" -#include "serve-protocol-impl.hh" #include "state.hh" #include "current-process.hh" #include "processes.hh" #include "util.hh" -#include "serve-protocol.hh" -#include "serve-protocol-impl.hh" #include "ssh.hh" #include "finally.hh" #include "url.hh" @@ -39,38 +36,6 @@ bool ::Machine::isLocalhost() const namespace nix::build_remote { -static std::unique_ptr openConnection( - ::Machine::ptr machine, SSHMaster & master) -{ - Strings command = {"nix-store", "--serve", "--write"}; - if (machine->isLocalhost()) { - command.push_back("--builders"); - command.push_back(""); - } else { - auto remoteStore = machine->storeUri.params.find("remote-store"); - if (remoteStore != machine->storeUri.params.end()) { - command.push_back("--store"); - command.push_back(shellEscape(remoteStore->second)); - } - } - - auto ret = master.startCommand(std::move(command), { - "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" - }); - - // XXX: determine the actual max value we can use from /proc. - - // FIXME: Should this be upstreamed into `startCommand` in Nix? - - int pipesize = 1024 * 1024; - - fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize); - fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize); - - return ret; -} - - static void copyClosureTo( ::Machine::Connection & conn, Store & destStore, @@ -87,8 +52,8 @@ static void copyClosureTo( // FIXME: substitute output pollutes our build log /* Get back the set of paths that are already valid on the remote host. */ - auto present = conn.queryValidPaths( - destStore, true, closure, useSubstitutes); + auto present = conn.store->queryValidPaths( + closure, true, useSubstitutes); if (present.size() == closure.size()) return; @@ -103,12 +68,7 @@ static void copyClosureTo( std::unique_lock sendLock(conn.machine->state->sendLock, std::chrono::seconds(600)); - conn.to << ServeProto::Command::ImportPaths; - destStore.exportPaths(missing, conn.to); - conn.to.flush(); - - if (readInt(conn.from) != 1) - throw Error("remote machine failed to import closure"); + conn.store->addMultipleToStoreLegacy(destStore, missing); } @@ -228,7 +188,7 @@ static BuildResult performBuild( counter & nrStepsBuilding ) { - conn.putBuildDerivationRequest(localStore, drvPath, drv, options); + auto kont = conn.store->buildDerivationAsync(drvPath, drv, options); BuildResult result; @@ -237,7 +197,10 @@ static BuildResult performBuild( startTime = time(0); { MaintainCount mc(nrStepsBuilding); - result = ServeProto::Serialise::read(localStore, conn); + result = kont(); + // Without proper call-once functions, we need to manually + // delete after calling. + kont = {}; } stopTime = time(0); @@ -253,7 +216,7 @@ static BuildResult performBuild( // If the protocol was too old to give us `builtOutputs`, initialize // it manually by introspecting the derivation. - if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6) + if (GET_PROTOCOL_MINOR(conn.store->getProtocol()) < 6) { // If the remote is too old to handle CA derivations, we can’t get this // far anyways @@ -286,26 +249,25 @@ static void copyPathFromRemote( const ValidPathInfo & info ) { - /* Receive the NAR from the remote and add it to the - destination store. Meanwhile, extract all the info from the - NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { - /* Note: we should only send the command to dump the store - path to the remote if the NAR is actually going to get read - by the destination store, which won't happen if this path - is already valid on the destination store. Since this - lambda function only gets executed if someone tries to read - from source2, we will send the command from here rather - than outside the lambda. */ - conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path); - conn.to.flush(); - - TeeSource tee(conn.from, sink); - extractNarData(tee, localStore.printStorePath(info.path), narMembers); - }); - - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); + /* Receive the NAR from the remote and add it to the + destination store. Meanwhile, extract all the info from the + NAR that getBuildOutput() needs. */ + auto source2 = sinkToSource([&](Sink & sink) + { + /* Note: we should only send the command to dump the store + path to the remote if the NAR is actually going to get read + by the destination store, which won't happen if this path + is already valid on the destination store. Since this + lambda function only gets executed if someone tries to read + from source2, we will send the command from here rather + than outside the lambda. */ + conn.store->narFromPath(info.path, [&](Source & source) { + TeeSource tee{source, sink}; + extractNarData(tee, conn.store->printStorePath(info.path), narMembers); + }); + }); + + destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); } static void copyPathsFromRemote( @@ -404,30 +366,37 @@ void State::buildRemote(ref destStore, updateStep(ssConnecting); - auto storeRef = machine->completeStoreReference(); - - auto * pSpecified = std::get_if(&storeRef.variant); - if (!pSpecified || pSpecified->scheme != "ssh") { - throw Error("Currently, only (legacy-)ssh stores are supported!"); - } - - LegacySSHStoreConfig storeConfig { - pSpecified->scheme, - pSpecified->authority, - storeRef.params - }; - - auto master = storeConfig.createSSHMaster( - false, // no SSH master yet - logFD.get()); - // FIXME: rewrite to use Store. - auto child = build_remote::openConnection(machine, master); + ::Machine::Connection conn { + .machine = machine, + .store = [&]{ + auto * pSpecified = std::get_if(&machine->storeUri.variant); + if (!pSpecified || pSpecified->scheme != "ssh") { + throw Error("Currently, only (legacy-)ssh stores are supported!"); + } + + auto remoteStore = machine->openStore().dynamic_pointer_cast(); + assert(remoteStore); + + if (machine->isLocalhost()) { + auto rp_new = remoteStore->remoteProgram.get(); + rp_new.push_back("--builders"); + rp_new.push_back(""); + const_cast &>(remoteStore->remoteProgram).assign(rp_new); + } + remoteStore->extraSshArgs = { + "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" + }; + const_cast &>(remoteStore->logFD).assign(logFD.get()); + + return nix::ref{remoteStore}; + }(), + }; { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child->sshPid; + activeStepState->pid = conn.store->getConnectionPid(); } Finally clearPid([&]() { @@ -442,35 +411,12 @@ void State::buildRemote(ref destStore, process. Meh. */ }); - ::Machine::Connection conn { - { - .to = child->in.get(), - .from = child->out.get(), - /* Handshake. */ - .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize - }, - /*.machine =*/ machine, - }; - Finally updateStats([&]() { - bytesReceived += conn.from.read; - bytesSent += conn.to.written; + auto stats = conn.store->getConnectionStats(); + bytesReceived += stats.bytesReceived; + bytesSent += stats.bytesSent; }); - constexpr ServeProto::Version our_version = 0x206; - - try { - conn.remoteVersion = decltype(conn)::handshake( - conn.to, - conn.from, - our_version, - machine->storeUri.render()); - } catch (EndOfFile & e) { - child->sshPid.wait(); - std::string s = chomp(readFile(result.logFile)); - throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s); - } - { auto info(machine->state->connectInfo.lock()); info->consecutiveFailures = 0; @@ -539,7 +485,7 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - auto infos = conn.queryPathInfos(*localStore, outputs); + auto infos = conn.store->queryPathInfosUncached(outputs); size_t totalNarSize = 0; for (auto & [_, info] : infos) totalNarSize += info.narSize; @@ -574,9 +520,11 @@ void State::buildRemote(ref destStore, } } - /* Shut down the connection. */ - child->in = -1; - child->sshPid.wait(); + /* Shut down the connection done by RAII. + + Only difference is kill() instead of wait() (i.e. send signal + then wait()) + */ } catch (Error & e) { /* Disable this machine until a certain period of time has diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 30e01c746..e2d31434f 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -20,9 +20,7 @@ #include "store-api.hh" #include "sync.hh" #include "nar-extractor.hh" -#include "serve-protocol.hh" -#include "serve-protocol-impl.hh" -#include "serve-protocol-connection.hh" +#include "legacy-ssh-store.hh" #include "machines.hh" @@ -292,9 +290,11 @@ struct Machine : nix::Machine bool isLocalhost() const; // A connection to a machine - struct Connection : nix::ServeProto::BasicClientConnection { + struct Connection { // Backpointer to the machine ptr machine; + // Opened store + nix::ref store; }; }; From 9e162dcf527e18b2c41f93970a40c51fae410277 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Feb 2025 18:00:13 -0500 Subject: [PATCH 3/5] Avoid custom logic to copy inputs to the remote builder --- src/hydra-queue-runner/build-remote.cc | 50 ++++---------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index e8cd6d2f9..1699b4cac 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -36,42 +36,6 @@ bool ::Machine::isLocalhost() const namespace nix::build_remote { -static void copyClosureTo( - ::Machine::Connection & conn, - Store & destStore, - const StorePathSet & paths, - SubstituteFlag useSubstitutes = NoSubstitute) -{ - StorePathSet closure; - destStore.computeFSClosure(paths, closure); - - /* Send the "query valid paths" command with the "lock" option - enabled. This prevents a race where the remote host - garbage-collect paths that are already there. Optionally, ask - the remote host to substitute missing paths. */ - // FIXME: substitute output pollutes our build log - /* Get back the set of paths that are already valid on the remote - host. */ - auto present = conn.store->queryValidPaths( - closure, true, useSubstitutes); - - if (present.size() == closure.size()) return; - - auto sorted = destStore.topoSortPaths(closure); - - StorePathSet missing; - for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (!present.count(*i)) missing.insert(*i); - - printMsg(lvlDebug, "sending %d missing paths", missing.size()); - - std::unique_lock sendLock(conn.machine->state->sendLock, - std::chrono::seconds(600)); - - conn.store->addMultipleToStoreLegacy(destStore, missing); -} - - // FIXME: use Store::topoSortPaths(). static StorePaths reverseTopoSortPaths(const std::map & paths) { @@ -163,13 +127,13 @@ static BasicDerivation sendInputs( auto now1 = std::chrono::steady_clock::now(); /* Copy the input closure. */ - if (conn.machine->isLocalhost()) { - StorePathSet closure; - destStore.computeFSClosure(basicDrv.inputSrcs, closure); - copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); - } else { - copyClosureTo(conn, destStore, basicDrv.inputSrcs, Substitute); - } + copyClosure( + destStore, + conn.machine->isLocalhost() ? localStore : *conn.store, + basicDrv.inputSrcs, + NoRepair, + NoCheckSigs, + Substitute); auto now2 = std::chrono::steady_clock::now(); From 45d075e5db46610ff73e6c8cf88337331d11fb33 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Feb 2025 18:02:32 -0500 Subject: [PATCH 4/5] WIP: Avoid custom logic copying outputs from the remote builder We need a replacement for the nar member logic, however. And maybe also a test that fails until this is fixed (this one should not be passing). --- src/hydra-queue-runner/build-remote.cc | 94 ++------------------------ 1 file changed, 4 insertions(+), 90 deletions(-) diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 1699b4cac..275ce612c 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -36,35 +36,6 @@ bool ::Machine::isLocalhost() const namespace nix::build_remote { -// FIXME: use Store::topoSortPaths(). -static StorePaths reverseTopoSortPaths(const std::map & paths) -{ - StorePaths sorted; - StorePathSet visited; - - std::function dfsVisit; - - dfsVisit = [&](const StorePath & path) { - if (!visited.insert(path).second) return; - - auto info = paths.find(path); - auto references = info == paths.end() ? StorePathSet() : info->second.references; - - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) - dfsVisit(i); - - sorted.push_back(path); - }; - - for (auto & i : paths) - dfsVisit(i.first); - - return sorted; -} - static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) { std::string base(drvPath.to_string()); @@ -205,54 +176,6 @@ static BuildResult performBuild( return result; } -static void copyPathFromRemote( - ::Machine::Connection & conn, - NarMemberDatas & narMembers, - Store & localStore, - Store & destStore, - const ValidPathInfo & info -) -{ - /* Receive the NAR from the remote and add it to the - destination store. Meanwhile, extract all the info from the - NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { - /* Note: we should only send the command to dump the store - path to the remote if the NAR is actually going to get read - by the destination store, which won't happen if this path - is already valid on the destination store. Since this - lambda function only gets executed if someone tries to read - from source2, we will send the command from here rather - than outside the lambda. */ - conn.store->narFromPath(info.path, [&](Source & source) { - TeeSource tee{source, sink}; - extractNarData(tee, conn.store->printStorePath(info.path), narMembers); - }); - }); - - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); -} - -static void copyPathsFromRemote( - ::Machine::Connection & conn, - NarMemberDatas & narMembers, - Store & localStore, - Store & destStore, - const std::map & infos -) -{ - auto pathsSorted = reverseTopoSortPaths(infos); - - for (auto & path : pathsSorted) { - auto & info = infos.find(path)->second; - copyPathFromRemote( - conn, narMembers, localStore, destStore, - ValidPathInfo { path, info }); - } - -} - } /* using namespace nix::build_remote; */ @@ -449,21 +372,12 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - auto infos = conn.store->queryPathInfosUncached(outputs); - - size_t totalNarSize = 0; - for (auto & [_, info] : infos) totalNarSize += info.narSize; - - if (totalNarSize > maxOutputSize) { - result.stepStatus = bsNarSizeLimitExceeded; - return; - } - /* Copy each path. */ - printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", - localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize); + printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’", + localStore->printStorePath(step->drvPath), machine->storeUri.render()); + + copyClosure(*conn.store, *destStore, outputs); - build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); auto now2 = std::chrono::steady_clock::now(); result.overhead += std::chrono::duration_cast(now2 - now1).count(); From f3dc4e9228a23dd6ab49e081be5cd43a68092ab8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Feb 2025 17:57:12 -0500 Subject: [PATCH 5/5] WIP ssh-ng:// --- flake.lock | 8 ++--- flake.nix | 2 +- src/hydra-queue-runner/build-remote.cc | 33 ++++++++++---------- src/hydra-queue-runner/builder.cc | 18 ++++------- src/hydra-queue-runner/hydra-queue-runner.cc | 2 +- src/hydra-queue-runner/state.hh | 7 ++--- 6 files changed, 31 insertions(+), 39 deletions(-) diff --git a/flake.lock b/flake.lock index 8217d1685..7d1bcbce2 100644 --- a/flake.lock +++ b/flake.lock @@ -13,16 +13,16 @@ "nixpkgs-regression": [] }, "locked": { - "lastModified": 1739570697, - "narHash": "sha256-n5kdS1C24tlJxDV6Wm1iBlyvGk+p0gMXRcWVCAipYLs=", + "lastModified": 1739571938, + "narHash": "sha256-NlaLAed/xei6RWpU2HIIbDjILRC4l1NIfGeyrn7ALQs=", "owner": "NixOS", "repo": "nix", - "rev": "5eade4825221d3284fc6555cb20de2c7aa171d72", + "rev": "ffc649d2eabdd3e678b5bcc211dd59fd06debf3e", "type": "github" }, "original": { "owner": "NixOS", - "ref": "legacy-ssh-extensions-for-hydra", + "ref": "ssh-ng-extensions-for-hydra", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 932bf1b93..244d3353e 100644 --- a/flake.nix +++ b/flake.nix @@ -4,7 +4,7 @@ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small"; inputs.nix = { - url = "github:NixOS/nix/legacy-ssh-extensions-for-hydra"; + url = "github:NixOS/nix/ssh-ng-extensions-for-hydra"; inputs.nixpkgs.follows = "nixpkgs"; # hide nix dev tooling from our lock file diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 275ce612c..b99bb5507 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -7,7 +7,7 @@ #include "build-result.hh" #include "path.hh" -#include "legacy-ssh-store.hh" +#include "ssh-store.hh" #include "serve-protocol.hh" #include "state.hh" #include "current-process.hh" @@ -119,11 +119,10 @@ static BuildResult performBuild( Store & localStore, StorePath drvPath, const BasicDerivation & drv, - const ServeProto::BuildOptions & options, counter & nrStepsBuilding ) { - auto kont = conn.store->buildDerivationAsync(drvPath, drv, options); + auto kont = conn.store->buildDerivationAsync(drvPath, drv, bmNormal); BuildResult result; @@ -238,7 +237,6 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) void State::buildRemote(ref destStore, ::Machine::ptr machine, Step::ptr step, - const ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers) @@ -258,23 +256,26 @@ void State::buildRemote(ref destStore, .machine = machine, .store = [&]{ auto * pSpecified = std::get_if(&machine->storeUri.variant); - if (!pSpecified || pSpecified->scheme != "ssh") { - throw Error("Currently, only (legacy-)ssh stores are supported!"); + if (!pSpecified || pSpecified->scheme != "ssh-ng") { + throw Error("Currently, only ssh-ng:// stores are supported!"); } - auto remoteStore = machine->openStore().dynamic_pointer_cast(); + auto remoteStore = machine->openStore().dynamic_pointer_cast(); + auto remoteStoreConfig = std::dynamic_pointer_cast(remoteStore); assert(remoteStore); if (machine->isLocalhost()) { - auto rp_new = remoteStore->remoteProgram.get(); + auto rp_new = remoteStoreConfig->remoteProgram.get(); rp_new.push_back("--builders"); rp_new.push_back(""); - const_cast &>(remoteStore->remoteProgram).assign(rp_new); + const_cast &>(remoteStoreConfig->remoteProgram).assign(rp_new); } - remoteStore->extraSshArgs = { + remoteStoreConfig->extraSshArgs = { "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" }; - const_cast &>(remoteStore->logFD).assign(logFD.get()); + + // TODO logging + //const_cast &>(remoteStore->logFD).assign(logFD.get()); return nix::ref{remoteStore}; }(), @@ -283,12 +284,10 @@ void State::buildRemote(ref destStore, { auto activeStepState(activeStep->state_.lock()); if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = conn.store->getConnectionPid(); } Finally clearPid([&]() { auto activeStepState(activeStep->state_.lock()); - activeStepState->pid = -1; /* FIXME: there is a slight race here with step cancellation in State::processQueueChange(), which @@ -299,9 +298,10 @@ void State::buildRemote(ref destStore, }); Finally updateStats([&]() { - auto stats = conn.store->getConnectionStats(); - bytesReceived += stats.bytesReceived; - bytesSent += stats.bytesSent; + // TODO + //auto stats = conn.store->getConnectionStats(); + //bytesReceived += stats.bytesReceived; + //bytesSent += stats.bytesSent; }); { @@ -341,7 +341,6 @@ void State::buildRemote(ref destStore, *localStore, step->drvPath, resolvedDrv, - buildOptions, nrStepsBuilding ); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 4bc00f0cf..22b91420c 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -98,13 +98,6 @@ State::StepResult State::doBuildStep(nix::ref destStore, it). */ BuildID buildId; std::optional buildDrvPath; - // Other fields set below - nix::ServeProto::BuildOptions buildOptions { - .maxLogSize = maxLogSize, - .nrRepeats = step->isDeterministic ? 1u : 0u, - .enforceDeterminism = step->isDeterministic, - .keepFailed = false, - }; auto conn(dbPool.get()); @@ -139,18 +132,19 @@ State::StepResult State::doBuildStep(nix::ref destStore, { auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName)); if (i != jobsetRepeats.end()) - buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second); + warn("jobset repeats is deprecated; nix stopped supporting this correctly a long time ago."); } } if (!build) build = *dependents.begin(); buildId = build->id; buildDrvPath = build->drvPath; - buildOptions.maxSilentTime = build->maxSilentTime; - buildOptions.buildTimeout = build->buildTimeout; + settings.maxLogSize = maxLogSize; + settings.maxSilentTime = build->maxSilentTime; + settings.buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), 1, machine->storeUri.render(), buildId, (dependents.size() - 1)); } if (!buildOneDone) @@ -211,7 +205,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, try { /* FIXME: referring builds may have conflicting timeouts. */ - buildRemote(destStore, machine, step, buildOptions, result, activeStep, updateStep, narMembers); + buildRemote(destStore, machine, step, result, activeStep, updateStep, narMembers); } catch (Error & e) { if (activeStep->state_.lock()->cancelled) { printInfo("marking step %d of build %d as cancelled", stepNr, buildId); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 99411f9fb..82ff3b3ba 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -182,7 +182,7 @@ void State::monitorMachinesFile() getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); if (machinesFiles.empty()) { - parseMachines("localhost " + + parseMachines("ssh-ng://localhost " + (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) + " - " + std::to_string(settings.maxBuildJobs) + " 1 " + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index e2d31434f..8c5958a71 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -20,7 +20,7 @@ #include "store-api.hh" #include "sync.hh" #include "nar-extractor.hh" -#include "legacy-ssh-store.hh" +#include "ssh-store.hh" #include "machines.hh" @@ -294,12 +294,12 @@ struct Machine : nix::Machine // Backpointer to the machine ptr machine; // Opened store - nix::ref store; + nix::ref store; }; }; -class HydraConfig; +struct HydraConfig; class State @@ -542,7 +542,6 @@ private: void buildRemote(nix::ref destStore, Machine::ptr machine, Step::ptr step, - const nix::ServeProto::BuildOptions & buildOptions, RemoteResult & result, std::shared_ptr activeStep, std::function updateStep, NarMemberDatas & narMembers);