Complete target-release test simulating an entire update (#9059)

Finishes the `target-release` `reconfigurator-test`, showing the
simulate update walking through the process of starting new Nexus zones,
waiting for handoff, then expunging the old Nexus zones.

Has two tweaks:

* Fixes a planning report off-by-one bug where we'd claim a zone was
both out of date and expunged (or updated) within the same plan.
* Adds a `set active-nexus-gen N` command to `reconfigurator-cli` to
control Nexus handoff instead of assuming it completes instantly.

Closes #8478

---------

Co-authored-by: Sean Klein <[email protected]>

Author: jgallagher

dev-tools/reconfigurator-cli/src/lib.rs

@@ -150,6 +150,7 @@ impl ReconfiguratorSim {
         builder.set_internal_dns_version(parent_blueprint.internal_dns_version);
         builder.set_external_dns_version(parent_blueprint.external_dns_version);
 
+        let active_nexus_gen = state.config().active_nexus_zone_generation();
         let mut active_nexus_zones = BTreeSet::new();
         let mut not_yet_nexus_zones = BTreeSet::new();
 
@@ -177,20 +178,23 @@ impl ReconfiguratorSim {
 
             match &zone.zone_type {
                 nexus_types::deployment::BlueprintZoneType::Nexus(nexus) => {
-                    if nexus.nexus_generation
-                        == parent_blueprint.nexus_generation
-                    {
+                    if nexus.nexus_generation == active_nexus_gen {
                         active_nexus_zones.insert(zone.id);
-                    } else if nexus.nexus_generation
-                        > parent_blueprint.nexus_generation
-                    {
+                    } else if nexus.nexus_generation > active_nexus_gen {
                         not_yet_nexus_zones.insert(zone.id);
                     }
                 }
                 _ => (),
             }
         }
 
+        if active_nexus_zones.is_empty() {
+            bail!(
+                "no Nexus zones found at current active generation \
+                 ({active_nexus_gen})"
+            );
+        }
+
         builder.set_active_nexus_zones(active_nexus_zones);
         builder.set_not_yet_nexus_zones(not_yet_nexus_zones);
 
@@ -1185,6 +1189,9 @@ enum SetArgs {
     Seed { seed: String },
     /// target number of Nexus instances (for planning)
     NumNexus { num_nexus: u16 },
+    /// specify the generation of Nexus zones that are considered active when
+    /// running the blueprint planner
+    ActiveNexusGen { gen: Generation },
     /// system's external DNS zone name (suffix)
     ExternalDnsZoneName { zone_name: String },
     /// system target release
@@ -2736,6 +2743,12 @@ fn cmd_set(
                 .target_nexus_zone_count(usize::from(num_nexus));
             rv
         }
+        SetArgs::ActiveNexusGen { gen } => {
+            let rv =
+                format!("will use active Nexus zones from generation {gen}");
+            state.config_mut().set_active_nexus_zone_generation(gen);
+            rv
+        }
         SetArgs::ExternalDnsZoneName { zone_name } => {
             let rv = format!(
                 "external DNS zone name: {:?} -> {:?}",

dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt

@@ -446,26 +446,50 @@ blueprint-diff latest
 sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
 inventory-generate
 
-# Add Nexus zones on three sleds
+# Start the Nexus handoff process: A planning step here should place three new
+# Nexus zones, each running from the new TUF repo artifact.
 blueprint-plan latest latest
 blueprint-diff latest
 
-# Propagate configs to the sleds which should be running Nexus
+# Update inventory to reflect the three Nexus zones starting.
 sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
 sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
 sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
 inventory-generate
 
-# Update the Nexus generation from 1 -> 2, initiating
-# quiesce of the old Nexuses
+# Planning now should bump the top-level `nexus_generation` to 2, indicating
+# that we want handoff to begin.
 blueprint-plan latest latest
 blueprint-diff latest
 
-# Expunge three Nexus zones, one at a time
+# Planning again should make no changes: we haven't yet performed a handoff,
+# so the old Nexus zones are still in charge.
 blueprint-plan latest latest
+blueprint-diff latest
+
+# Bump the set of active Nexus zones; this simulates Nexus handoff.
+set active-nexus-gen 2
+
+# Planning should now expunge one of the old Nexus zones.
 blueprint-plan latest latest
+blueprint-diff latest
+
+# Two more planning iterations should expunge the remaining two old install
+# dataset Nexus zones.
+blueprint-plan latest latest
+blueprint-diff latest
 blueprint-plan latest latest
+blueprint-diff latest
 
-# Attempt to plan one more blueprint.
-# There should be no changes attempted here.
+# Update sled configs and inventory to reflect the expunged Nexus zones.
+sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest
+sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest
+sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest
+inventory-generate
+
+# Planning now should note that the old zones are gone, but do nothing else:
+# the update is complete. All in-service zones have image sources from the new
+# TUF repo, and all zones using old images are expunged.
 blueprint-plan latest latest
+blueprint-diff latest
+blueprint-show latest

dev-tools/reconfigurator-cli/tests/output/cmds-stdout

@@ -863,6 +863,8 @@ wiped system
 
 > load state.json
 loaded data from "state.json"
+warnings:
+  could not determine active Nexus generation from serialized state: no target blueprint set (using default of 1)
 result:
   system:
     using collection 6e066695-94bc-4250-bd63-fd799c166cc1 as source of sled inventory data
@@ -875,6 +877,7 @@ result:
   config:
     configured external DNS zone name: oxide.example
     configured silo names: example-silo
+    active Nexus generation: 1
 
 
 > sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a