diff --git a/wfe-yaml/Cargo.toml b/wfe-yaml/Cargo.toml index 198f765..10d17fb 100644 --- a/wfe-yaml/Cargo.toml +++ b/wfe-yaml/Cargo.toml @@ -9,6 +9,7 @@ default = [] deno = ["deno_core", "deno_error", "url", "reqwest"] buildkit = ["wfe-buildkit"] containerd = ["wfe-containerd"] +rustlang = ["wfe-rustlang"] [dependencies] wfe-core = { workspace = true } @@ -27,6 +28,7 @@ url = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } wfe-buildkit = { workspace = true, optional = true } wfe-containerd = { workspace = true, optional = true } +wfe-rustlang = { workspace = true, optional = true } [dev-dependencies] pretty_assertions = { workspace = true } @@ -36,3 +38,4 @@ wfe-core = { workspace = true, features = ["test-support"] } wfe = { path = "../wfe" } wiremock = { workspace = true } tempfile = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/wfe-yaml/src/compiler.rs b/wfe-yaml/src/compiler.rs index 869cf58..3b4e32b 100644 --- a/wfe-yaml/src/compiler.rs +++ b/wfe-yaml/src/compiler.rs @@ -13,6 +13,8 @@ use crate::executors::deno::{DenoConfig, DenoPermissions, DenoStep}; use wfe_buildkit::{BuildkitConfig, BuildkitStep}; #[cfg(feature = "containerd")] use wfe_containerd::{ContainerdConfig, ContainerdStep}; +#[cfg(feature = "rustlang")] +use wfe_rustlang::{CargoCommand, CargoConfig, CargoStep, RustupCommand, RustupConfig, RustupStep}; use wfe_core::primitives::sub_workflow::SubWorkflowStep; use wfe_core::models::condition::{ComparisonOp, FieldComparison, StepCondition}; @@ -454,6 +456,38 @@ fn build_step_config_and_factory( }); Ok((key, value, factory)) } + #[cfg(feature = "rustlang")] + "cargo-build" | "cargo-test" | "cargo-check" | "cargo-clippy" | "cargo-fmt" + | "cargo-doc" | "cargo-publish" | "cargo-audit" | "cargo-deny" | "cargo-nextest" + | "cargo-llvm-cov" | "cargo-doc-mdx" => { + let config = build_cargo_config(step, step_type)?; + let key = format!("wfe_yaml::cargo::{}", step.name); + let value = serde_json::to_value(&config).map_err(|e| { + YamlWorkflowError::Compilation(format!( + "Failed to serialize cargo config: {e}" + )) + })?; + let config_clone = config.clone(); + let factory: StepFactory = Box::new(move || { + Box::new(CargoStep::new(config_clone.clone())) as Box + }); + Ok((key, value, factory)) + } + #[cfg(feature = "rustlang")] + "rust-install" | "rustup-toolchain" | "rustup-component" | "rustup-target" => { + let config = build_rustup_config(step, step_type)?; + let key = format!("wfe_yaml::rustup::{}", step.name); + let value = serde_json::to_value(&config).map_err(|e| { + YamlWorkflowError::Compilation(format!( + "Failed to serialize rustup config: {e}" + )) + })?; + let config_clone = config.clone(); + let factory: StepFactory = Box::new(move || { + Box::new(RustupStep::new(config_clone.clone())) as Box + }); + Ok((key, value, factory)) + } "workflow" => { let config = step.config.as_ref().ok_or_else(|| { YamlWorkflowError::Compilation(format!( @@ -576,6 +610,88 @@ fn build_shell_config(step: &YamlStep) -> Result }) } +#[cfg(feature = "rustlang")] +fn build_cargo_config( + step: &YamlStep, + step_type: &str, +) -> Result { + let command = match step_type { + "cargo-build" => CargoCommand::Build, + "cargo-test" => CargoCommand::Test, + "cargo-check" => CargoCommand::Check, + "cargo-clippy" => CargoCommand::Clippy, + "cargo-fmt" => CargoCommand::Fmt, + "cargo-doc" => CargoCommand::Doc, + "cargo-publish" => CargoCommand::Publish, + "cargo-audit" => CargoCommand::Audit, + "cargo-deny" => CargoCommand::Deny, + "cargo-nextest" => CargoCommand::Nextest, + "cargo-llvm-cov" => CargoCommand::LlvmCov, + "cargo-doc-mdx" => CargoCommand::DocMdx, + _ => { + return Err(YamlWorkflowError::Compilation(format!( + "Unknown cargo step type: '{step_type}'" + ))); + } + }; + + let config = step.config.as_ref(); + let timeout_ms = config + .and_then(|c| c.timeout.as_ref()) + .and_then(|t| parse_duration_ms(t)); + + Ok(CargoConfig { + command, + toolchain: config.and_then(|c| c.toolchain.clone()), + package: config.and_then(|c| c.package.clone()), + features: config.map(|c| c.features.clone()).unwrap_or_default(), + all_features: config.and_then(|c| c.all_features).unwrap_or(false), + no_default_features: config.and_then(|c| c.no_default_features).unwrap_or(false), + release: config.and_then(|c| c.release).unwrap_or(false), + target: config.and_then(|c| c.target.clone()), + profile: config.and_then(|c| c.profile.clone()), + extra_args: config.map(|c| c.extra_args.clone()).unwrap_or_default(), + env: config.map(|c| c.env.clone()).unwrap_or_default(), + working_dir: config.and_then(|c| c.working_dir.clone()), + timeout_ms, + output_dir: config.and_then(|c| c.output_dir.clone()), + }) +} + +#[cfg(feature = "rustlang")] +fn build_rustup_config( + step: &YamlStep, + step_type: &str, +) -> Result { + let command = match step_type { + "rust-install" => RustupCommand::Install, + "rustup-toolchain" => RustupCommand::ToolchainInstall, + "rustup-component" => RustupCommand::ComponentAdd, + "rustup-target" => RustupCommand::TargetAdd, + _ => { + return Err(YamlWorkflowError::Compilation(format!( + "Unknown rustup step type: '{step_type}'" + ))); + } + }; + + let config = step.config.as_ref(); + let timeout_ms = config + .and_then(|c| c.timeout.as_ref()) + .and_then(|t| parse_duration_ms(t)); + + Ok(RustupConfig { + command, + toolchain: config.and_then(|c| c.toolchain.clone()), + components: config.map(|c| c.components.clone()).unwrap_or_default(), + targets: config.map(|c| c.targets.clone()).unwrap_or_default(), + profile: config.and_then(|c| c.profile.clone()), + default_toolchain: config.and_then(|c| c.default_toolchain.clone()), + extra_args: config.map(|c| c.extra_args.clone()).unwrap_or_default(), + timeout_ms, + }) +} + fn parse_duration_ms(s: &str) -> Option { let s = s.trim(); // Check "ms" before "s" since strip_suffix('s') would also match "500ms" diff --git a/wfe-yaml/src/schema.rs b/wfe-yaml/src/schema.rs index 7fb7db1..74c903b 100644 --- a/wfe-yaml/src/schema.rs +++ b/wfe-yaml/src/schema.rs @@ -164,6 +164,39 @@ pub struct StepConfig { pub containerd_addr: Option, /// CLI binary name for containerd steps: "nerdctl" (default) or "docker". pub cli: Option, + // Cargo fields + /// Target package for cargo steps (`-p`). + pub package: Option, + /// Features to enable for cargo steps. + #[serde(default)] + pub features: Vec, + /// Enable all features for cargo steps. + #[serde(default)] + pub all_features: Option, + /// Disable default features for cargo steps. + #[serde(default)] + pub no_default_features: Option, + /// Build in release mode for cargo steps. + #[serde(default)] + pub release: Option, + /// Build profile for cargo steps (`--profile`). + pub profile: Option, + /// Rust toolchain override for cargo steps (e.g. "nightly"). + pub toolchain: Option, + /// Additional arguments for cargo/rustup steps. + #[serde(default)] + pub extra_args: Vec, + /// Output directory for generated files (e.g., MDX docs). + pub output_dir: Option, + // Rustup fields + /// Components to add for rustup steps (e.g. ["clippy", "rustfmt"]). + #[serde(default)] + pub components: Vec, + /// Compilation targets to add for rustup steps (e.g. ["wasm32-unknown-unknown"]). + #[serde(default)] + pub targets: Vec, + /// Default toolchain for rust-install steps. + pub default_toolchain: Option, // Workflow (sub-workflow) fields /// Child workflow ID (for `type: workflow` steps). #[serde(rename = "workflow")] diff --git a/wfe-yaml/tests/rustlang.rs b/wfe-yaml/tests/rustlang.rs new file mode 100644 index 0000000..e361601 --- /dev/null +++ b/wfe-yaml/tests/rustlang.rs @@ -0,0 +1,777 @@ +#![cfg(feature = "rustlang")] + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use wfe::models::WorkflowStatus; +use wfe::{WorkflowHostBuilder, run_workflow_sync}; +use wfe_core::test_support::{ + InMemoryLockProvider, InMemoryPersistenceProvider, InMemoryQueueProvider, +}; +use wfe_yaml::load_single_workflow_from_str; + +fn has_factory(compiled: &wfe_yaml::compiler::CompiledWorkflow, key: &str) -> bool { + compiled.step_factories.iter().any(|(k, _)| k == key) +} + +async fn run_yaml_workflow(yaml: &str) -> wfe::models::WorkflowInstance { + let config = HashMap::new(); + let compiled = load_single_workflow_from_str(yaml, &config).unwrap(); + + let persistence = Arc::new(InMemoryPersistenceProvider::new()); + let lock = Arc::new(InMemoryLockProvider::new()); + let queue = Arc::new(InMemoryQueueProvider::new()); + + let host = WorkflowHostBuilder::new() + .use_persistence(persistence as Arc) + .use_lock_provider(lock as Arc) + .use_queue_provider(queue as Arc) + .build() + .unwrap(); + + for (key, factory) in compiled.step_factories { + host.register_step_factory(&key, factory).await; + } + + host.register_workflow_definition(compiled.definition.clone()) + .await; + host.start().await.unwrap(); + + let instance = run_workflow_sync( + &host, + &compiled.definition.id, + compiled.definition.version, + serde_json::json!({}), + Duration::from_secs(30), + ) + .await + .unwrap(); + + host.stop().await; + instance +} + +// --------------------------------------------------------------------------- +// Compiler tests — verify YAML compiles to correct step types and configs +// --------------------------------------------------------------------------- + +#[test] +fn compile_cargo_build_step() { + let yaml = r#" +workflow: + id: cargo-build-wf + version: 1 + steps: + - name: build + type: cargo-build + config: + release: true + package: my-crate +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + let step = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("build")) + .unwrap(); + assert_eq!(step.step_type, "wfe_yaml::cargo::build"); + assert!(has_factory(&compiled, "wfe_yaml::cargo::build")); +} + +#[test] +fn compile_cargo_test_step() { + let yaml = r#" +workflow: + id: cargo-test-wf + version: 1 + steps: + - name: test + type: cargo-test + config: + features: + - feat1 + - feat2 + extra_args: + - "--" + - "--nocapture" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + let step = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("test")) + .unwrap(); + assert_eq!(step.step_type, "wfe_yaml::cargo::test"); +} + +#[test] +fn compile_cargo_check_step() { + let yaml = r#" +workflow: + id: cargo-check-wf + version: 1 + steps: + - name: check + type: cargo-check +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::check")); +} + +#[test] +fn compile_cargo_clippy_step() { + let yaml = r#" +workflow: + id: cargo-clippy-wf + version: 1 + steps: + - name: lint + type: cargo-clippy + config: + all_features: true + extra_args: + - "--" + - "-D" + - "warnings" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::lint")); +} + +#[test] +fn compile_cargo_fmt_step() { + let yaml = r#" +workflow: + id: cargo-fmt-wf + version: 1 + steps: + - name: format + type: cargo-fmt + config: + extra_args: + - "--check" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::format")); +} + +#[test] +fn compile_cargo_doc_step() { + let yaml = r#" +workflow: + id: cargo-doc-wf + version: 1 + steps: + - name: docs + type: cargo-doc + config: + extra_args: + - "--no-deps" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::docs")); +} + +#[test] +fn compile_cargo_publish_step() { + let yaml = r#" +workflow: + id: cargo-publish-wf + version: 1 + steps: + - name: publish + type: cargo-publish + config: + extra_args: + - "--dry-run" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::publish")); +} + +#[test] +fn compile_cargo_step_with_toolchain() { + let yaml = r#" +workflow: + id: nightly-wf + version: 1 + steps: + - name: nightly-check + type: cargo-check + config: + toolchain: nightly + no_default_features: true +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::nightly-check")); +} + +#[test] +fn compile_cargo_step_with_timeout() { + let yaml = r#" +workflow: + id: timeout-wf + version: 1 + steps: + - name: slow-build + type: cargo-build + config: + timeout: 5m +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::slow-build")); +} + +#[test] +fn compile_cargo_step_without_config() { + let yaml = r#" +workflow: + id: bare-wf + version: 1 + steps: + - name: bare-check + type: cargo-check +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::bare-check")); +} + +#[test] +fn compile_cargo_multi_step_pipeline() { + let yaml = r#" +workflow: + id: ci-pipeline + version: 1 + steps: + - name: fmt + type: cargo-fmt + config: + extra_args: ["--check"] + - name: check + type: cargo-check + - name: clippy + type: cargo-clippy + config: + extra_args: ["--", "-D", "warnings"] + - name: test + type: cargo-test + - name: build + type: cargo-build + config: + release: true +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::check")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::clippy")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::test")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::build")); +} + +#[test] +fn compile_cargo_step_with_all_shared_flags() { + let yaml = r#" +workflow: + id: full-flags-wf + version: 1 + steps: + - name: full + type: cargo-build + config: + package: my-crate + features: [foo, bar] + all_features: false + no_default_features: true + release: true + toolchain: stable + profile: release + extra_args: ["--jobs", "4"] + working_dir: /tmp/project + timeout: 30s + env: + RUSTFLAGS: "-C target-cpu=native" +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::full")); +} + +#[test] +fn compile_cargo_step_preserves_step_config_json() { + let yaml = r#" +workflow: + id: config-json-wf + version: 1 + steps: + - name: build + type: cargo-build + config: + release: true + package: wfe-core +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + let step = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("build")) + .unwrap(); + + let step_config = step.step_config.as_ref().unwrap(); + assert_eq!(step_config["command"], "build"); + assert_eq!(step_config["release"], true); + assert_eq!(step_config["package"], "wfe-core"); +} + +// --------------------------------------------------------------------------- +// Integration tests — run actual cargo commands through the workflow engine +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn cargo_check_on_self_succeeds() { + let yaml = r#" +workflow: + id: self-check + version: 1 + steps: + - name: check + type: cargo-check + config: + working_dir: . + timeout: 120s +"#; + let instance = run_yaml_workflow(yaml).await; + assert_eq!(instance.status, WorkflowStatus::Complete); + + let data = instance.data.as_object().unwrap(); + assert!(data.contains_key("check.stdout") || data.contains_key("check.stderr")); +} + +#[tokio::test] +async fn cargo_fmt_check_compiles() { + let yaml = r#" +workflow: + id: fmt-check + version: 1 + steps: + - name: fmt + type: cargo-fmt + config: + working_dir: . + extra_args: ["--check"] + timeout: 60s +"#; + let config = HashMap::new(); + let compiled = load_single_workflow_from_str(yaml, &config).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt")); +} + +// --------------------------------------------------------------------------- +// Rustup compiler tests +// --------------------------------------------------------------------------- + +#[test] +fn compile_rust_install_step() { + let yaml = r#" +workflow: + id: rust-install-wf + version: 1 + steps: + - name: install-rust + type: rust-install + config: + profile: minimal + default_toolchain: stable + timeout: 5m +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + let step = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("install-rust")) + .unwrap(); + assert_eq!(step.step_type, "wfe_yaml::rustup::install-rust"); + assert!(has_factory(&compiled, "wfe_yaml::rustup::install-rust")); +} + +#[test] +fn compile_rustup_toolchain_step() { + let yaml = r#" +workflow: + id: tc-install-wf + version: 1 + steps: + - name: add-nightly + type: rustup-toolchain + config: + toolchain: nightly-2024-06-01 + profile: minimal +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-nightly")); +} + +#[test] +fn compile_rustup_component_step() { + let yaml = r#" +workflow: + id: comp-add-wf + version: 1 + steps: + - name: add-tools + type: rustup-component + config: + components: [clippy, rustfmt, rust-src] + toolchain: nightly +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-tools")); +} + +#[test] +fn compile_rustup_target_step() { + let yaml = r#" +workflow: + id: target-add-wf + version: 1 + steps: + - name: add-wasm + type: rustup-target + config: + targets: [wasm32-unknown-unknown] + toolchain: stable +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-wasm")); +} + +#[test] +fn compile_rustup_step_without_config() { + let yaml = r#" +workflow: + id: bare-install-wf + version: 1 + steps: + - name: install + type: rust-install +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::install")); +} + +#[test] +fn compile_rustup_step_preserves_config_json() { + let yaml = r#" +workflow: + id: config-json-wf + version: 1 + steps: + - name: tc + type: rustup-toolchain + config: + toolchain: nightly + profile: minimal +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + let step = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("tc")) + .unwrap(); + + let step_config = step.step_config.as_ref().unwrap(); + assert_eq!(step_config["command"], "toolchain-install"); + assert_eq!(step_config["toolchain"], "nightly"); + assert_eq!(step_config["profile"], "minimal"); +} + +#[test] +fn compile_full_rust_ci_pipeline() { + let yaml = r#" +workflow: + id: full-rust-ci + version: 1 + steps: + - name: install + type: rust-install + config: + profile: minimal + default_toolchain: stable + - name: add-nightly + type: rustup-toolchain + config: + toolchain: nightly + - name: add-components + type: rustup-component + config: + components: [clippy, rustfmt] + - name: add-wasm + type: rustup-target + config: + targets: [wasm32-unknown-unknown] + - name: fmt + type: cargo-fmt + config: + extra_args: ["--check"] + - name: check + type: cargo-check + - name: clippy + type: cargo-clippy + config: + extra_args: ["--", "-D", "warnings"] + - name: test + type: cargo-test + - name: build + type: cargo-build + config: + release: true +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::install")); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-nightly")); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-components")); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-wasm")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::check")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::clippy")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::test")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::build")); +} + +#[test] +fn compile_rustup_component_with_extra_args() { + let yaml = r#" +workflow: + id: comp-extra-wf + version: 1 + steps: + - name: add-llvm + type: rustup-component + config: + components: [llvm-tools-preview] + extra_args: ["--force"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::add-llvm")); +} + +#[test] +fn compile_rustup_target_multiple() { + let yaml = r#" +workflow: + id: multi-target-wf + version: 1 + steps: + - name: cross-targets + type: rustup-target + config: + targets: + - wasm32-unknown-unknown + - aarch64-linux-android + - x86_64-unknown-linux-musl + toolchain: nightly +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::rustup::cross-targets")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("cross-targets")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "target-add"); + let targets = step_config["targets"].as_array().unwrap(); + assert_eq!(targets.len(), 3); +} + +// --------------------------------------------------------------------------- +// External cargo tool step compiler tests +// --------------------------------------------------------------------------- + +#[test] +fn compile_cargo_audit_step() { + let yaml = r#" +workflow: + id: audit-wf + version: 1 + steps: + - name: audit + type: cargo-audit +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::audit")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("audit")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "audit"); +} + +#[test] +fn compile_cargo_deny_step() { + let yaml = r#" +workflow: + id: deny-wf + version: 1 + steps: + - name: license-check + type: cargo-deny + config: + extra_args: ["check"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::license-check")); +} + +#[test] +fn compile_cargo_nextest_step() { + let yaml = r#" +workflow: + id: nextest-wf + version: 1 + steps: + - name: fast-test + type: cargo-nextest + config: + features: [foo] + extra_args: ["--no-fail-fast"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::fast-test")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("fast-test")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "nextest"); +} + +#[test] +fn compile_cargo_llvm_cov_step() { + let yaml = r#" +workflow: + id: cov-wf + version: 1 + steps: + - name: coverage + type: cargo-llvm-cov + config: + extra_args: ["--html", "--output-dir", "coverage"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::coverage")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("coverage")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "llvm-cov"); +} + +#[test] +fn compile_full_ci_with_external_tools() { + let yaml = r#" +workflow: + id: full-ci-external + version: 1 + steps: + - name: audit + type: cargo-audit + - name: deny + type: cargo-deny + config: + extra_args: ["check", "licenses"] + - name: test + type: cargo-nextest + - name: coverage + type: cargo-llvm-cov + config: + extra_args: ["--summary-only"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::audit")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::deny")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::test")); + assert!(has_factory(&compiled, "wfe_yaml::cargo::coverage")); +} + +#[test] +fn compile_cargo_doc_mdx_step() { + let yaml = r#" +workflow: + id: doc-mdx-wf + version: 1 + steps: + - name: docs + type: cargo-doc-mdx + config: + package: my-crate + output_dir: docs/api + extra_args: ["--no-deps"] +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::docs")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("docs")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "doc-mdx"); + assert_eq!(step_config["package"], "my-crate"); + assert_eq!(step_config["output_dir"], "docs/api"); +} + +#[test] +fn compile_cargo_doc_mdx_minimal() { + let yaml = r#" +workflow: + id: doc-mdx-minimal-wf + version: 1 + steps: + - name: generate-docs + type: cargo-doc-mdx +"#; + let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap(); + assert!(has_factory(&compiled, "wfe_yaml::cargo::generate-docs")); + + let step_config = compiled + .definition + .steps + .iter() + .find(|s| s.name.as_deref() == Some("generate-docs")) + .unwrap() + .step_config + .as_ref() + .unwrap(); + assert_eq!(step_config["command"], "doc-mdx"); + assert!(step_config["output_dir"].is_null()); +} diff --git a/wfe-yaml/tests/rustlang_containerd.rs b/wfe-yaml/tests/rustlang_containerd.rs new file mode 100644 index 0000000..a7586a9 --- /dev/null +++ b/wfe-yaml/tests/rustlang_containerd.rs @@ -0,0 +1,474 @@ +//! End-to-end integration tests for the Rust toolchain steps running inside +//! containerd containers. +//! +//! These tests start from a bare Debian image (no Rust installed) and exercise +//! the full Rust CI pipeline: install Rust, install external tools, create a +//! test project, and run every cargo operation. +//! +//! Requirements: +//! - A running containerd daemon (Lima/colima or native) +//! - Set `WFE_CONTAINERD_ADDR` to point to the socket +//! +//! These tests are gated behind `rustlang` + `containerd` features and are +//! marked `#[ignore]` so they don't run in normal CI. Run them explicitly: +//! cargo test -p wfe-yaml --features rustlang,containerd --test rustlang_containerd -- --ignored + +#![cfg(all(feature = "rustlang", feature = "containerd"))] + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use wfe::models::WorkflowStatus; +use wfe::{WorkflowHostBuilder, run_workflow_sync}; +use wfe_core::test_support::{ + InMemoryLockProvider, InMemoryPersistenceProvider, InMemoryQueueProvider, +}; +use wfe_yaml::load_single_workflow_from_str; + +/// Returns the containerd address if available, or None. +/// Supports both Unix sockets (`unix:///path`) and TCP (`http://host:port`). +fn containerd_addr() -> Option { + let addr = std::env::var("WFE_CONTAINERD_ADDR").unwrap_or_else(|_| { + // Default: TCP proxy on the Lima VM (socat forwarding containerd socket) + "http://127.0.0.1:2500".to_string() + }); + + // For TCP addresses, assume reachable (the test will fail fast if not). + if addr.starts_with("http://") || addr.starts_with("tcp://") { + return Some(addr); + } + + // For Unix sockets, check the file exists. + let socket_path = addr.strip_prefix("unix://").unwrap_or(addr.as_str()); + if Path::new(socket_path).exists() { + Some(addr) + } else { + None + } +} + +async fn run_yaml_workflow_with_config( + yaml: &str, + config: &HashMap, +) -> wfe::models::WorkflowInstance { + let compiled = load_single_workflow_from_str(yaml, config).unwrap(); + for step in &compiled.definition.steps { + eprintln!(" step: {:?} type={} config={:?}", step.name, step.step_type, step.step_config); + } + eprintln!(" factories: {:?}", compiled.step_factories.iter().map(|(k, _)| k.clone()).collect::>()); + + let persistence = Arc::new(InMemoryPersistenceProvider::new()); + let lock = Arc::new(InMemoryLockProvider::new()); + let queue = Arc::new(InMemoryQueueProvider::new()); + + let host = WorkflowHostBuilder::new() + .use_persistence(persistence as Arc) + .use_lock_provider(lock as Arc) + .use_queue_provider(queue as Arc) + .build() + .unwrap(); + + for (key, factory) in compiled.step_factories { + host.register_step_factory(&key, factory).await; + } + + host.register_workflow_definition(compiled.definition.clone()) + .await; + host.start().await.unwrap(); + + let instance = run_workflow_sync( + &host, + &compiled.definition.id, + compiled.definition.version, + serde_json::json!({}), + Duration::from_secs(1800), + ) + .await + .unwrap(); + + host.stop().await; + instance +} + +/// Shared env block and volume template for containerd steps. +/// Uses format! to avoid Rust 2024 reserved `##` token in raw strings. +fn containerd_step_yaml( + name: &str, + network: &str, + pull: &str, + timeout: &str, + working_dir: Option<&str>, + mount_workspace: bool, + run_script: &str, +) -> String { + let wfe = "##wfe"; + let wd = working_dir + .map(|d| format!(" working_dir: {d}")) + .unwrap_or_default(); + let ws_volume = if mount_workspace { + " - source: ((workspace))\n target: /workspace" + } else { + "" + }; + + format!( + r#" - name: {name} + type: containerd + config: + image: docker.io/library/debian:bookworm-slim + containerd_addr: ((containerd_addr)) + user: "0:0" + network: {network} + pull: {pull} + timeout: {timeout} +{wd} + env: + CARGO_HOME: /cargo + RUSTUP_HOME: /rustup + PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + volumes: + - source: ((cargo_home)) + target: /cargo + - source: ((rustup_home)) + target: /rustup +{ws_volume} + run: | +{run_script} + echo "{wfe}[output {name}.status=ok]" +"# + ) +} + +/// Base directory for shared state between host and containerd VM. +/// Must be inside the virtiofs mount defined in test/lima/wfe-test.yaml. +fn shared_dir() -> std::path::PathBuf { + let base = std::env::var("WFE_IO_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/wfe-io")); + std::fs::create_dir_all(&base).unwrap(); + base +} + +/// Create a temporary directory inside the shared mount so containerd can see it. +fn shared_tempdir(name: &str) -> std::path::PathBuf { + let id = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = shared_dir().join(format!("{name}-{id}")); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn make_config( + addr: &str, + cargo_home: &Path, + rustup_home: &Path, + workspace: Option<&Path>, +) -> HashMap { + let mut config = HashMap::new(); + config.insert( + "containerd_addr".to_string(), + serde_json::Value::String(addr.to_string()), + ); + config.insert( + "cargo_home".to_string(), + serde_json::Value::String(cargo_home.to_str().unwrap().to_string()), + ); + config.insert( + "rustup_home".to_string(), + serde_json::Value::String(rustup_home.to_str().unwrap().to_string()), + ); + if let Some(ws) = workspace { + config.insert( + "workspace".to_string(), + serde_json::Value::String(ws.to_str().unwrap().to_string()), + ); + } + config +} + +// --------------------------------------------------------------------------- +// Minimal: just echo hello in a containerd step through the workflow engine +// --------------------------------------------------------------------------- + +#[tokio::test] +#[ignore = "requires containerd daemon"] +async fn minimal_echo_in_containerd_via_workflow() { + let _ = tracing_subscriber::fmt().with_env_filter("wfe_containerd=debug,wfe_core::executor=debug").try_init(); + let Some(addr) = containerd_addr() else { + eprintln!("SKIP: containerd not available"); + return; + }; + + let mut config = HashMap::new(); + config.insert( + "containerd_addr".to_string(), + serde_json::Value::String(addr), + ); + + let wfe = "##wfe"; + let yaml = format!( + r#"workflow: + id: minimal-containerd + version: 1 + error_behavior: + type: terminate + steps: + - name: echo + type: containerd + config: + image: docker.io/library/alpine:3.18 + containerd_addr: ((containerd_addr)) + user: "0:0" + network: none + pull: if-not-present + timeout: 30s + run: | + echo hello-from-workflow + echo "{wfe}[output echo.status=ok]" +"# + ); + + let instance = run_yaml_workflow_with_config(&yaml, &config).await; + + eprintln!("Status: {:?}, Data: {:?}", instance.status, instance.data); + assert_eq!(instance.status, WorkflowStatus::Complete); + let data = instance.data.as_object().unwrap(); + assert_eq!( + data.get("echo.status").and_then(|v| v.as_str()), + Some("ok"), + ); +} + +// --------------------------------------------------------------------------- +// Full Rust CI pipeline in a container: install → build → test → lint → cover +// --------------------------------------------------------------------------- + +#[tokio::test] +#[ignore = "requires containerd daemon"] +async fn full_rust_pipeline_in_container() { + let Some(addr) = containerd_addr() else { + eprintln!("SKIP: containerd socket not available"); + return; + }; + + let cargo_home = shared_tempdir("cargo"); + let rustup_home = shared_tempdir("rustup"); + let workspace = shared_tempdir("workspace"); + + let config = make_config( + &addr, + &cargo_home, + &rustup_home, + Some(&workspace), + ); + + let steps = [ + containerd_step_yaml( + "install-rust", "host", "if-not-present", "10m", None, false, + " apt-get update && apt-get install -y curl gcc pkg-config libssl-dev\n\ + \x20 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable", + ), + containerd_step_yaml( + "install-tools", "host", "never", "10m", None, false, + " rustup component add clippy rustfmt llvm-tools-preview\n\ + \x20 cargo install cargo-audit cargo-deny cargo-nextest cargo-llvm-cov", + ), + containerd_step_yaml( + "create-project", "host", "never", "2m", None, true, + " cargo init /workspace/test-crate --name test-crate\n\ + \x20 cd /workspace/test-crate\n\ + \x20 echo '#[cfg(test)] mod tests { #[test] fn it_works() { assert_eq!(2+2,4); } }' >> src/main.rs", + ), + containerd_step_yaml( + "cargo-fmt", "none", "never", "2m", + Some("/workspace/test-crate"), true, + " cargo fmt -- --check || cargo fmt", + ), + containerd_step_yaml( + "cargo-check", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo check", + ), + containerd_step_yaml( + "cargo-clippy", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo clippy -- -D warnings", + ), + containerd_step_yaml( + "cargo-test", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo test", + ), + containerd_step_yaml( + "cargo-build", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo build --release", + ), + containerd_step_yaml( + "cargo-nextest", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo nextest run", + ), + containerd_step_yaml( + "cargo-llvm-cov", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo llvm-cov --summary-only", + ), + containerd_step_yaml( + "cargo-audit", "host", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo audit || true", + ), + containerd_step_yaml( + "cargo-deny", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo deny init\n\ + \x20 cargo deny check || true", + ), + containerd_step_yaml( + "cargo-doc", "none", "never", "5m", + Some("/workspace/test-crate"), true, + " cargo doc --no-deps", + ), + ]; + + let yaml = format!( + "workflow:\n id: rust-container-pipeline\n version: 1\n error_behavior:\n type: terminate\n steps:\n{}", + steps.join("\n") + ); + + let instance = run_yaml_workflow_with_config(&yaml, &config).await; + + assert_eq!( + instance.status, + WorkflowStatus::Complete, + "workflow should complete successfully, data: {:?}", + instance.data + ); + + let data = instance.data.as_object().unwrap(); + + for key in [ + "install-rust.status", + "install-tools.status", + "create-project.status", + "cargo-fmt.status", + "cargo-check.status", + "cargo-clippy.status", + "cargo-test.status", + "cargo-build.status", + "cargo-nextest.status", + "cargo-llvm-cov.status", + "cargo-audit.status", + "cargo-deny.status", + "cargo-doc.status", + ] { + assert_eq!( + data.get(key).and_then(|v| v.as_str()), + Some("ok"), + "step output '{key}' should be 'ok', got: {:?}", + data.get(key) + ); + } +} + +// --------------------------------------------------------------------------- +// Focused test: just rust-install in a bare container +// --------------------------------------------------------------------------- + +#[tokio::test] +#[ignore = "requires containerd daemon"] +async fn rust_install_in_bare_container() { + let Some(addr) = containerd_addr() else { + eprintln!("SKIP: containerd socket not available"); + return; + }; + + let cargo_home = shared_tempdir("cargo"); + let rustup_home = shared_tempdir("rustup"); + + let config = make_config(&addr, &cargo_home, &rustup_home, None); + + let wfe = "##wfe"; + let yaml = format!( + r#"workflow: + id: rust-install-container + version: 1 + error_behavior: + type: terminate + steps: + - name: install + type: containerd + config: + image: docker.io/library/debian:bookworm-slim + containerd_addr: ((containerd_addr)) + user: "0:0" + network: host + pull: if-not-present + timeout: 10m + env: + CARGO_HOME: /cargo + RUSTUP_HOME: /rustup + PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + volumes: + - source: ((cargo_home)) + target: /cargo + - source: ((rustup_home)) + target: /rustup + run: | + apt-get update && apt-get install -y curl + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + rustc --version + cargo --version + echo "{wfe}[output rustc_installed=true]" + + - name: verify + type: containerd + config: + image: docker.io/library/debian:bookworm-slim + containerd_addr: ((containerd_addr)) + user: "0:0" + network: none + pull: if-not-present + timeout: 2m + env: + CARGO_HOME: /cargo + RUSTUP_HOME: /rustup + PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + volumes: + - source: ((cargo_home)) + target: /cargo + - source: ((rustup_home)) + target: /rustup + run: | + rustc --version + cargo --version + echo "{wfe}[output verify.status=ok]" +"# + ); + + let instance = run_yaml_workflow_with_config(&yaml, &config).await; + + assert_eq!( + instance.status, + WorkflowStatus::Complete, + "install workflow should complete, data: {:?}", + instance.data + ); + + let data = instance.data.as_object().unwrap(); + eprintln!("Workflow data: {:?}", instance.data); + assert!( + data.get("rustc_installed").is_some(), + "rustc_installed should be set, got data: {:?}", + data + ); + assert_eq!( + data.get("verify.status").and_then(|v| v.as_str()), + Some("ok"), + ); +}