98.6% OF DEVELOPERS CANNOT REVIEW THIS PR! [read more...]

Cargo.toml

@@ -12,6 +12,14 @@ authors.workspace = true
 edition.workspace = true
 version.workspace = true
 
+[[bin]]
+name = "polkadot-execute-worker"
+path = "node/core/pvf/execute-worker/src/main.rs"
+
+[[bin]]
+name = "polkadot-prepare-worker"
+path = "node/core/pvf/prepare-worker/src/main.rs"
+
 [workspace.package]
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2021"
@@ -28,6 +36,11 @@ polkadot-node-core-pvf = { path = "node/core/pvf" }
 polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" }
 polkadot-overseer = { path = "node/overseer" }
 
+# Needed for worker binaries.
+polkadot-node-core-pvf-common = { path = "node/core/pvf/common" }
+polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" }
+sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
 [dev-dependencies]
 assert_cmd = "2.0.4"
 nix = { version = "0.26.1", features = ["signal"] }
@@ -226,6 +239,8 @@ license-file = ["LICENSE", "0"]
 maintainer-scripts = "scripts/packaging/deb-maintainer-scripts"
 assets = [
 	["target/release/polkadot", "/usr/bin/", "755"],
+	["target/release/polkadot-prepare-worker", "/usr/lib/polkadot/", "755"],
+	["target/release/polkadot-execute-worker", "/usr/lib/polkadot/", "755"],
 	["scripts/packaging/polkadot.service", "/lib/systemd/system/", "644"]
 ]
 conf-files = [

cli/build.rs

@@ -19,4 +19,7 @@ fn main() {
 		println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
 	}
 	substrate_build_script_utils::generate_cargo_keys();
+	// For the node/worker version check, make sure we always rebuild the node when the version
+	// changes.
+	substrate_build_script_utils::rerun_if_git_head_changed();
 }

cli/src/cli.rs

@@ -17,6 +17,7 @@
 //! Polkadot CLI library.
 
 use clap::Parser;
+use std::path::PathBuf;
 
 #[allow(missing_docs)]
 #[derive(Debug, Parser)]
@@ -148,6 +149,13 @@ pub struct RunCmd {
 	/// **Dangerous!** Do not touch unless explicitly adviced to.
 	#[arg(long)]
 	pub overseer_channel_capacity_override: Option<usize>,
+
+	/// Path to the directory where auxiliary worker binaries reside. If not specified, the main
+	/// binary's directory is searched first, then `/usr/lib/polkadot` is searched. If the path
+	/// points to an executable rather then directory, that executable is used both as preparation
+	/// and execution worker (supposed to be used for tests only).
+	#[arg(long, value_name = "PATH")]
+	pub workers_path: Option<PathBuf>,
 }
 
 #[allow(missing_docs)]

cli/src/command.rs

@@ -283,16 +283,24 @@ where
 		let database_source = config.database.clone();
 		let task_manager = service::build_full(
 			config,
-			service::IsCollator::No,
-			grandpa_pause,
-			enable_beefy,
-			jaeger_agent,
-			None,
-			false,
-			overseer_gen,
-			cli.run.overseer_channel_capacity_override,
-			maybe_malus_finality_delay,
-			hwbench,
+			service::NewFullParams {
+				is_collator: service::IsCollator::No,
+				grandpa_pause,
+				enable_beefy,
+				jaeger_agent,
+				telemetry_worker_handle: None,
+				node_version: Some(env!("SUBSTRATE_CLI_IMPL_VERSION").to_string()),
+				workers_path: cli.run.workers_path,
+				workers_names: None,
+				dont_use_external_workers: false,
+				overseer_enable_anyways: false,
+				overseer_gen,
+				overseer_message_channel_capacity_override: cli
+					.run
+					.overseer_channel_capacity_override,
+				malus_finality_delay: maybe_malus_finality_delay,
+				hwbench,
+			},
 		)
 		.map(|full| full.task_manager)?;
 
@@ -437,6 +445,7 @@ pub fn run() -> Result<()> {
 				polkadot_node_core_pvf_prepare_worker::worker_entrypoint(
 					&cmd.socket_path,
 					Some(&cmd.node_impl_version),
+					Some(&cmd.node_impl_version),
 				);
 				Ok(())
 			}
@@ -459,6 +468,7 @@ pub fn run() -> Result<()> {
 				polkadot_node_core_pvf_execute_worker::worker_entrypoint(
 					&cmd.socket_path,
 					Some(&cmd.node_impl_version),
+					Some(&cmd.node_impl_version),
 				);
 				Ok(())
 			}

node/core/candidate-validation/src/lib.rs

@@ -93,9 +93,12 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12);
 pub struct Config {
 	/// The path where candidate validation can store compiled artifacts for PVFs.
 	pub artifacts_cache_path: PathBuf,
-	/// The path to the executable which can be used for spawning PVF compilation & validation
-	/// workers.
-	pub program_path: PathBuf,
+	/// The version of the node. `None` can be passed to skip the version check (only for tests).
+	pub node_version: Option<String>,
+	/// Path to the preparation worker binary
+	pub prep_worker_path: PathBuf,
+	/// Path to the execution worker binary
+	pub exec_worker_path: PathBuf,
 }
 
 /// The candidate validation subsystem.
@@ -124,15 +127,9 @@ impl CandidateValidationSubsystem {
 #[overseer::subsystem(CandidateValidation, error=SubsystemError, prefix=self::overseer)]
 impl<Context> CandidateValidationSubsystem {
 	fn start(self, ctx: Context) -> SpawnedSubsystem {
-		let future = run(
-			ctx,
-			self.metrics,
-			self.pvf_metrics,
-			self.config.artifacts_cache_path,
-			self.config.program_path,
-		)
-		.map_err(|e| SubsystemError::with_origin("candidate-validation", e))
-		.boxed();
+		let future = run(ctx, self.metrics, self.pvf_metrics, self.config)
+			.map_err(|e| SubsystemError::with_origin("candidate-validation", e))
+			.boxed();
 		SpawnedSubsystem { name: "candidate-validation-subsystem", future }
 	}
 }
@@ -142,11 +139,15 @@ async fn run<Context>(
 	mut ctx: Context,
 	metrics: Metrics,
 	pvf_metrics: polkadot_node_core_pvf::Metrics,
-	cache_path: PathBuf,
-	program_path: PathBuf,
+	Config { artifacts_cache_path, node_version, prep_worker_path, exec_worker_path }: Config,
 ) -> SubsystemResult<()> {
 	let (validation_host, task) = polkadot_node_core_pvf::start(
-		polkadot_node_core_pvf::Config::new(cache_path, program_path),
+		polkadot_node_core_pvf::Config::new(
+			artifacts_cache_path,
+			node_version,
+			prep_worker_path,
+			exec_worker_path,
+		),
 		pvf_metrics,
 	);
 	ctx.spawn_blocking("pvf-validation-host", task.boxed())?;

node/core/pvf/common/Cargo.toml

@@ -31,6 +31,3 @@ landlock = "0.2.0"
 [dev-dependencies]
 assert_matches = "1.4.0"
 tempfile = "3.3.0"
-
-[build-dependencies]
-substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" }