Preserve artifact cache unless stale (#1918)

Co-authored-by: Marcin S <[email protected]>

Author: eagr

Cargo.lock

@@ -25,7 +25,7 @@
 
 use polkadot_node_core_pvf::{
 	InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PrepareError,
-	PrepareJobKind, PrepareStats, PvfPrepData, ValidationError, ValidationHost,
+	PrepareJobKind, PvfPrepData, ValidationError, ValidationHost,
 };
 use polkadot_node_primitives::{
 	BlockData, InvalidCandidate, PoV, ValidationResult, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT,
@@ -794,7 +794,7 @@ trait ValidationBackend {
 		validation_result
 	}
 
-	async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<PrepareStats, PrepareError>;
+	async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<(), PrepareError>;
 }
 
 #[async_trait]
@@ -824,7 +824,7 @@ impl ValidationBackend for ValidationHost {
 		})?
 	}
 
-	async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<PrepareStats, PrepareError> {
+	async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<(), PrepareError> {
 		let (tx, rx) = oneshot::channel();
 		if let Err(err) = self.precheck_pvf(pvf, tx).await {
 			// Return an IO error if there was an error communicating with the host.

polkadot/node/core/candidate-validation/src/lib.rs

@@ -377,7 +377,7 @@ impl ValidationBackend for MockValidateCandidateBackend {
 		result
 	}
 
-	async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<PrepareStats, PrepareError> {
+	async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<(), PrepareError> {
 		unreachable!()
 	}
 }
@@ -1014,11 +1014,11 @@ fn pov_decompression_failure_is_invalid() {
 }
 
 struct MockPreCheckBackend {
-	result: Result<PrepareStats, PrepareError>,
+	result: Result<(), PrepareError>,
 }
 
 impl MockPreCheckBackend {
-	fn with_hardcoded_result(result: Result<PrepareStats, PrepareError>) -> Self {
+	fn with_hardcoded_result(result: Result<(), PrepareError>) -> Self {
 		Self { result }
 	}
 }
@@ -1034,7 +1034,7 @@ impl ValidationBackend for MockPreCheckBackend {
 		unreachable!()
 	}
 
-	async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<PrepareStats, PrepareError> {
+	async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<(), PrepareError> {
 		self.result.clone()
 	}
 }
@@ -1051,7 +1051,7 @@ fn precheck_works() {
 
 	let (check_fut, check_result) = precheck_pvf(
 		ctx.sender(),
-		MockPreCheckBackend::with_hardcoded_result(Ok(PrepareStats::default())),
+		MockPreCheckBackend::with_hardcoded_result(Ok(())),
 		relay_parent,
 		validation_code_hash,
 	)
@@ -1113,7 +1113,7 @@ fn precheck_invalid_pvf_blob_compression() {
 
 	let (check_fut, check_result) = precheck_pvf(
 		ctx.sender(),
-		MockPreCheckBackend::with_hardcoded_result(Ok(PrepareStats::default())),
+		MockPreCheckBackend::with_hardcoded_result(Ok(())),
 		relay_parent,
 		validation_code_hash,
 	)

polkadot/node/core/candidate-validation/src/tests.rs

@@ -8,6 +8,7 @@ license.workspace = true
 
 [dependencies]
 always-assert = "0.1"
+blake3 = "1.5"
 cfg-if = "1.0"
 futures = "0.3.21"
 futures-timer = "3.0.2"

polkadot/node/core/pvf/Cargo.toml

@@ -56,7 +56,7 @@ impl TestHost {
 		&self,
 		code: &[u8],
 		executor_params: ExecutorParams,
-	) -> Result<PrepareStats, PrepareError> {
+	) -> Result<(), PrepareError> {
 		let (result_tx, result_rx) = futures::channel::oneshot::channel();
 
 		let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024)

polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs

@@ -36,6 +36,9 @@ seccompiler = "0.4.0"
 assert_matches = "1.4.0"
 tempfile = "3.3.0"
 
+[build-dependencies]
+substrate-build-script-utils = { path = "../../../../../substrate/utils/build-script-utils" }
+
 [features]
 # This feature is used to export test code to other crates without putting it in the production build.
 test-utils = []

polkadot/node/core/pvf/common/Cargo.toml

@@ -0,0 +1,19 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+fn main() {
+	substrate_build_script_utils::generate_wasmtime_version();
+}

polkadot/node/core/pvf/common/build.rs

@@ -14,16 +14,24 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::prepare::PrepareStats;
+use crate::prepare::{PrepareSuccess, PrepareWorkerSuccess};
 use parity_scale_codec::{Decode, Encode};
 use std::fmt;
 
-/// Result of PVF preparation performed by the validation host. Contains stats about the preparation
-/// if successful
-pub type PrepareResult = Result<PrepareStats, PrepareError>;
+/// Result of PVF preparation from a worker, with checksum of the compiled PVF and stats of the
+/// preparation if successful.
+pub type PrepareWorkerResult = Result<PrepareWorkerSuccess, PrepareError>;
+
+/// Result of PVF preparation propagated all the way back to the host, with path to the concluded
+/// artifact and stats of the preparation if successful.
+pub type PrepareResult = Result<PrepareSuccess, PrepareError>;
+
+/// Result of prechecking PVF performed by the validation host. Contains stats about the preparation
+/// if successful.
+pub type PrecheckResult = Result<(), PrepareError>;
 
 /// An error that occurred during the prepare part of the PVF pipeline.
-// Codec indexes are intended to stabilize pre-encoded payloads (see `OOM_PAYLOAD` below)
+// Codec indexes are intended to stabilize pre-encoded payloads (see `OOM_PAYLOAD`)
 #[derive(Debug, Clone, Encode, Decode)]
 pub enum PrepareError {
 	/// During the prevalidation stage of preparation an issue was found with the PVF.

polkadot/node/core/pvf/common/src/error.rs

@@ -31,6 +31,8 @@ pub use sp_tracing;
 
 const LOG_TARGET: &str = "parachain::pvf-common";
 
+pub const RUNTIME_VERSION: &str = env!("SUBSTRATE_WASMTIME_VERSION");
+
 use std::{
 	io::{self, Read, Write},
 	mem,

polkadot/node/core/pvf/common/src/lib.rs

@@ -15,6 +15,25 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use parity_scale_codec::{Decode, Encode};
+use std::path::PathBuf;
+
+/// Result from prepare worker if successful.
+#[derive(Debug, Clone, Default, Encode, Decode)]
+pub struct PrepareWorkerSuccess {
+	/// Checksum of the compiled PVF.
+	pub checksum: String,
+	/// Stats of the current preparation run.
+	pub stats: PrepareStats,
+}
+
+/// Result of PVF preparation if successful.
+#[derive(Debug, Clone, Default)]
+pub struct PrepareSuccess {
+	/// Canonical path to the compiled artifact.
+	pub path: PathBuf,
+	/// Stats of the current preparation run.
+	pub stats: PrepareStats,
+}
 
 /// Preparation statistics, including the CPU time and memory taken.
 #[derive(Debug, Clone, Default, Encode, Decode)]