test failed in CI: test_action_failure_can_unwind_no_pantry #7731
Description
(This test is also mentioned as a flake in #7551, but the error there is different, so I figured I'd open a separate issue since there may be two problems here.)
This test failed on a CI run on #7729:
https://github.com/oxidecomputer/omicron/pull/7729/checks?check_run_id=38198148478
Log showing the specific test failure:
Excerpt from the log showing the failure:
6319 2025-03-04T21:46:15.852Z log file: /var/tmp/omicron_tmp/omicron_nexus-58606776209479ce-test_action_failure_can_unwind_no_pantry.12215.0.log
6320 2025-03-04T21:46:15.852Z note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-58606776209479ce-test_action_failure_can_unwind_no_pantry.12215.0.log"
6321 2025-03-04T21:46:15.852Z DB URL: postgresql://root@[::1]:51763/omicron?sslmode=disable
6322 2025-03-04T21:46:15.852Z DB address: [::1]:51763
6323 2025-03-04T21:46:15.852Z log file: /var/tmp/omicron_tmp/omicron_nexus-58606776209479ce-test_action_failure_can_unwind_no_pantry.12215.2.log
6324 2025-03-04T21:46:15.853Z note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-58606776209479ce-test_action_failure_can_unwind_no_pantry.12215.2.log"
6325 2025-03-04T21:46:15.853Z
6326 2025-03-04T21:46:15.853Z thread 'app::sagas::snapshot_create::test::test_action_failure_can_unwind_no_pantry' panicked at sled-agent/src/sim/collection.rs:274:65:
6327 2025-03-04T21:46:15.853Z called `Result::unwrap()` on an `Err` value: InternalError { internal_message: "Communication Error: error sending request for url (http://127.0.0.1:43413/vmms/15fce7b6-545d-4002-9723-38ce26ffae96)" }
6328 2025-03-04T21:46:15.853Z stack backtrace:
6329 2025-03-04T21:46:15.853Z 0: rust_begin_unwind
6330 2025-03-04T21:46:15.853Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5
6331 2025-03-04T21:46:15.853Z 1: core::panicking::panic_fmt
6332 2025-03-04T21:46:15.853Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14
6333 2025-03-04T21:46:15.853Z 2: core::result::unwrap_failed
6334 2025-03-04T21:46:15.853Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1704:5
6335 2025-03-04T21:46:15.853Z 3: unwrap<(), omicron_common::api::external::error::Error>
6336 2025-03-04T21:46:15.853Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1109:23
6337 2025-03-04T21:46:15.853Z 4: {async_fn#0}<omicron_sled_agent::sim::instance::SimInstance>
6338 2025-03-04T21:46:15.853Z at /work/oxidecomputer/omicron/sled-agent/src/sim/collection.rs:274:13
6339 2025-03-04T21:46:15.853Z 5: {async_fn#0}
6340 2025-03-04T21:46:15.853Z at /work/oxidecomputer/omicron/sled-agent/src/sim/sled_agent.rs:528:58
6341 2025-03-04T21:46:15.853Z 6: {async_fn#0}
6342 2025-03-04T21:46:15.853Z at /work/oxidecomputer/omicron/sled-agent/src/sim/http_entrypoints.rs:773:38
6343 2025-03-04T21:46:15.902Z 7: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, dropshot::handler::HttpResponseUpdatedNoContent, dropshot::error::HttpError, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>>
6344 2025-03-04T21:46:15.902Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/handler.rs:664:78
6345 2025-03-04T21:46:15.903Z 8: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>>
6346 2025-03-04T21:46:15.903Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6347 2025-03-04T21:46:15.903Z 9: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, (dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>), dropshot::handler::HttpResponseUpdatedNoContent>
6348 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/handler.rs:785:16
6349 2025-03-04T21:46:15.903Z 10: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>>
6350 2025-03-04T21:46:15.903Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6351 2025-03-04T21:46:15.903Z 11: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>
6352 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/server.rs:937:69
6353 2025-03-04T21:46:15.903Z 12: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
6354 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/core.rs:331:17
6355 2025-03-04T21:46:15.903Z 13: with_mut<tokio::runtime::task::core::Stage<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>
6356 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/loom/std/unsafe_cell.rs:16:9
6357 2025-03-04T21:46:15.903Z 14: poll<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
6358 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/core.rs:320:13
6359 2025-03-04T21:46:15.903Z 15: tokio::runtime::task::harness::poll_future::{{closure}}
6360 2025-03-04T21:46:15.903Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:532:19
6361 2025-03-04T21:46:15.903Z 16: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
6362 2025-03-04T21:46:15.903Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9
6363 2025-03-04T21:46:15.903Z 17: do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
6364 2025-03-04T21:46:15.903Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40
6365 2025-03-04T21:46:15.903Z 18: __rust_try
6366 2025-03-04T21:46:15.904Z 19: try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>>
6367 2025-03-04T21:46:15.904Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19
6368 2025-03-04T21:46:15.904Z 20: catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
6369 2025-03-04T21:46:15.904Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14
6370 2025-03-04T21:46:15.904Z 21: tokio::runtime::task::harness::poll_future
6371 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:520:18
6372 2025-03-04T21:46:15.904Z 22: tokio::runtime::task::harness::Harness<T,S>::poll_inner
6373 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:209:27
6374 2025-03-04T21:46:15.904Z 23: tokio::runtime::task::harness::Harness<T,S>::poll
6375 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:154:15
6376 2025-03-04T21:46:15.904Z 24: tokio::runtime::task::raw::poll
6377 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/raw.rs:271:5
6378 2025-03-04T21:46:15.904Z 25: tokio::runtime::task::raw::RawTask::poll
6379 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/raw.rs:201:18
6380 2025-03-04T21:46:15.904Z 26: tokio::runtime::task::LocalNotified<S>::run
6381 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/mod.rs:449:9
6382 2025-03-04T21:46:15.904Z 27: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}
6383 2025-03-04T21:46:15.904Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:770:25
6384 2025-03-04T21:46:15.904Z 28: with_budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
6385 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:107:5
6386 2025-03-04T21:46:15.905Z 29: budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
6387 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:73:5
6388 2025-03-04T21:46:15.905Z 30: {closure#0}<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
6389 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:364:43
6390 2025-03-04T21:46:15.905Z 31: tokio::runtime::scheduler::current_thread::Context::enter
6391 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:428:19
6392 2025-03-04T21:46:15.905Z 32: tokio::runtime::scheduler::current_thread::Context::run_task
6393 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:364:23
6394 2025-03-04T21:46:15.905Z 33: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
6395 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:769:35
6396 2025-03-04T21:46:15.905Z 34: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}
6397 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:68
6398 2025-03-04T21:46:15.905Z 35: tokio::runtime::context::scoped::Scoped<T>::set
6399 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/scoped.rs:40:9
6400 2025-03-04T21:46:15.905Z 36: tokio::runtime::context::set_scheduler::{{closure}}
6401 2025-03-04T21:46:15.905Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:26
6402 2025-03-04T21:46:15.905Z 37: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)>
6403 2025-03-04T21:46:15.905Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12
6404 2025-03-04T21:46:15.905Z 38: std::thread::local::LocalKey<T>::with
6405 2025-03-04T21:46:15.905Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9
6406 2025-03-04T21:46:15.905Z 39: tokio::runtime::context::set_scheduler
6407 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:9
6408 2025-03-04T21:46:15.906Z 40: tokio::runtime::scheduler::current_thread::CoreGuard::enter
6409 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:27
6410 2025-03-04T21:46:15.906Z 41: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
6411 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:716:19
6412 2025-03-04T21:46:15.906Z 42: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6413 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:196:28
6414 2025-03-04T21:46:15.906Z 43: tokio::runtime::context::runtime::enter_runtime
6415 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16
6416 2025-03-04T21:46:15.906Z 44: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6417 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:184:9
6418 2025-03-04T21:46:15.906Z 45: tokio::runtime::runtime::Runtime::block_on_inner
6419 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:368:47
6420 2025-03-04T21:46:15.906Z 46: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6421 2025-03-04T21:46:15.906Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:342:13
6422 2025-03-04T21:46:15.906Z 47: test_action_failure_can_unwind_no_pantry
6423 2025-03-04T21:46:15.906Z at ./src/app/sagas/snapshot_create.rs:2225:5
6424 2025-03-04T21:46:15.906Z 48: omicron_nexus::app::sagas::snapshot_create::test::test_action_failure_can_unwind_no_pantry::{{closure}}
6425 2025-03-04T21:46:15.906Z at ./src/app/sagas/snapshot_create.rs:2228:6
6426 2025-03-04T21:46:15.906Z 49: core::ops::function::FnOnce::call_once
6427 2025-03-04T21:46:15.906Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5
6428 2025-03-04T21:46:15.906Z 50: core::ops::function::FnOnce::call_once
6429 2025-03-04T21:46:15.907Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5
6430 2025-03-04T21:46:15.907Z note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
6431 2025-03-04T21:46:15.907Z
6432 2025-03-04T21:46:15.907Z thread 'app::sagas::snapshot_create::test::test_action_failure_can_unwind_no_pantry' panicked at clients/sled-agent-client/src/lib.rs:379:14:
6433 2025-03-04T21:46:15.907Z instance_finish_transition() failed unexpectedly: reqwest::Error { kind: Request, url: "http://[::1]:35423/vmms/15fce7b6-545d-4002-9723-38ce26ffae96/poke", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(IncompleteMessage)) }
6434 2025-03-04T21:46:15.907Z stack backtrace:
6435 2025-03-04T21:46:15.907Z 0: rust_begin_unwind
6436 2025-03-04T21:46:15.907Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5
6437 2025-03-04T21:46:15.907Z 1: core::panicking::panic_fmt
6438 2025-03-04T21:46:15.907Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14
6439 2025-03-04T21:46:15.907Z 2: core::result::unwrap_failed
6440 2025-03-04T21:46:15.907Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1704:5
6441 2025-03-04T21:46:15.907Z 3: core::result::Result<T,E>::expect
6442 2025-03-04T21:46:15.907Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1061:23
6443 2025-03-04T21:46:15.933Z 4: {async_block#0}
6444 2025-03-04T21:46:15.933Z at /work/oxidecomputer/omicron/clients/sled-agent-client/src/lib.rs:375:9
6445 2025-03-04T21:46:15.933Z 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>
6446 2025-03-04T21:46:15.933Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6447 2025-03-04T21:46:15.933Z 6: {async_fn#0}
6448 2025-03-04T21:46:15.934Z at ./src/app/sagas/snapshot_create.rs:2205:47
6449 2025-03-04T21:46:15.934Z 7: {async_block#0}
6450 2025-03-04T21:46:15.934Z at ./src/app/sagas/snapshot_create.rs:2298:30
6451 2025-03-04T21:46:15.934Z 8: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=omicron_nexus::app::sagas::snapshot_create::Params> + core::marker::Send), alloc::alloc::Global>>
6452 2025-03-04T21:46:15.934Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6453 2025-03-04T21:46:15.934Z 9: {async_fn#0}<omicron_nexus::app::sagas::snapshot_create::SagaSnapshotCreate, omicron_nexus::app::sagas::snapshot_create::test::test_action_failure_can_unwind_wrapper::{async_fn#0}::{closure_env#0}, omicron_nexus::app::sagas::snapshot_create::test::test_action_failure_can_unwind_wrapper::{async_fn#0}::{closure_env#1}>
6454 2025-03-04T21:46:15.934Z at ./src/app/sagas/test_helpers.rs:597:36
6455 2025-03-04T21:46:15.934Z 10: {async_fn#0}
6456 2025-03-04T21:46:15.934Z at ./src/app/sagas/snapshot_create.rs:2388:10
6457 2025-03-04T21:46:15.934Z 11: {async_fn#0}
6458 2025-03-04T21:46:15.934Z at ./src/app/sagas/snapshot_create.rs:2229:66
6459 2025-03-04T21:46:15.934Z 12: {async_block#0}
6460 2025-03-04T21:46:15.934Z at ./src/app/sagas/snapshot_create.rs:2225:5
6461 2025-03-04T21:46:15.934Z 13: poll<&mut dyn core::future::future::Future<Output=()>>
6462 2025-03-04T21:46:15.934Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6463 2025-03-04T21:46:15.934Z 14: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6464 2025-03-04T21:46:15.934Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9
6465 2025-03-04T21:46:15.934Z 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
6466 2025-03-04T21:46:15.934Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:57
6467 2025-03-04T21:46:15.934Z 16: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
6468 2025-03-04T21:46:15.934Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:107:5
6469 2025-03-04T21:46:15.934Z 17: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
6470 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:73:5
6471 2025-03-04T21:46:15.935Z 18: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
6472 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:25
6473 2025-03-04T21:46:15.935Z 19: tokio::runtime::scheduler::current_thread::Context::enter
6474 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:428:19
6475 2025-03-04T21:46:15.935Z 20: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
6476 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:728:36
6477 2025-03-04T21:46:15.935Z 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}
6478 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:68
6479 2025-03-04T21:46:15.935Z 22: tokio::runtime::context::scoped::Scoped<T>::set
6480 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/scoped.rs:40:9
6481 2025-03-04T21:46:15.935Z 23: tokio::runtime::context::set_scheduler::{{closure}}
6482 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:26
6483 2025-03-04T21:46:15.935Z 24: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)>
6484 2025-03-04T21:46:15.935Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12
6485 2025-03-04T21:46:15.935Z 25: std::thread::local::LocalKey<T>::with
6486 2025-03-04T21:46:15.935Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9
6487 2025-03-04T21:46:15.935Z 26: tokio::runtime::context::set_scheduler
6488 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:9
6489 2025-03-04T21:46:15.935Z 27: tokio::runtime::scheduler::current_thread::CoreGuard::enter
6490 2025-03-04T21:46:15.935Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:27
6491 2025-03-04T21:46:15.936Z 28: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
6492 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:716:19
6493 2025-03-04T21:46:15.936Z 29: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6494 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:196:28
6495 2025-03-04T21:46:15.936Z 30: tokio::runtime::context::runtime::enter_runtime
6496 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16
6497 2025-03-04T21:46:15.936Z 31: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6498 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:184:9
6499 2025-03-04T21:46:15.936Z 32: tokio::runtime::runtime::Runtime::block_on_inner
6500 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:368:47
6501 2025-03-04T21:46:15.936Z 33: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
6502 2025-03-04T21:46:15.936Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:342:13
6503 2025-03-04T21:46:15.936Z 34: test_action_failure_can_unwind_no_pantry
6504 2025-03-04T21:46:15.936Z at ./src/app/sagas/snapshot_create.rs:2225:5
6505 2025-03-04T21:46:15.936Z 35: omicron_nexus::app::sagas::snapshot_create::test::test_action_failure_can_unwind_no_pantry::{{closure}}
6506 2025-03-04T21:46:15.936Z at ./src/app/sagas/snapshot_create.rs:2228:6
6507 2025-03-04T21:46:15.936Z 36: core::ops::function::FnOnce::call_once
6508 2025-03-04T21:46:15.936Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5
6509 2025-03-04T21:46:15.936Z 37: core::ops::function::FnOnce::call_once
6510 2025-03-04T21:46:15.936Z at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5