studio/wfe
2
0
Fork 0
Code Issues Pull Requests Actions Packages Releases 2 Activity

feat(wfe-yaml): wire rustlang step types and containerd integration tests

Browse Source 
Add rustlang feature flag to wfe-yaml with support for all cargo and
rustup step types (15 total), including cargo-doc-mdx.

Schema additions: output_dir, package, features, all_features,
no_default_features, release, profile, toolchain, extra_args,
components, targets, default_toolchain fields on StepConfig.

Integration tests for compiling all step types from YAML, and
containerd-based end-to-end tests for running Rust toolchain
inside containers from bare Debian images.
This commit is contained in:
Sienna Meridian Satterwhite
2026-03-29 16:57:50 +01:00
parent 272ddf17c2
commit 60e8c7f9a8
5 changed files with 1403 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
wfe-yaml/Cargo.toml
View File

@@ -9,6 +9,7 @@ default = []
deno = ["deno_core", "deno_error", "url", "reqwest"]
buildkit = ["wfe-buildkit"]
containerd = ["wfe-containerd"]
rustlang = ["wfe-rustlang"]
[dependencies]
wfe-core = { workspace = true }
@@ -27,6 +28,7 @@ url = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
wfe-buildkit = { workspace = true, optional = true }
wfe-containerd = { workspace = true, optional = true }
wfe-rustlang = { workspace = true, optional = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
@@ -36,3 +38,4 @@ wfe-core = { workspace = true, features = ["test-support"] }
wfe = { path = "../wfe" }
wiremock = { workspace = true }
tempfile = { workspace = true }
tracing-subscriber = { workspace = true }

116
wfe-yaml/src/compiler.rs
View File

@@ -13,6 +13,8 @@ use crate::executors::deno::{DenoConfig, DenoPermissions, DenoStep};
use wfe_buildkit::{BuildkitConfig, BuildkitStep};
#[cfg(feature = "containerd")]
use wfe_containerd::{ContainerdConfig, ContainerdStep};
#[cfg(feature = "rustlang")]
use wfe_rustlang::{CargoCommand, CargoConfig, CargoStep, RustupCommand, RustupConfig, RustupStep};
use wfe_core::primitives::sub_workflow::SubWorkflowStep;
use wfe_core::models::condition::{ComparisonOp, FieldComparison, StepCondition};
@@ -454,6 +456,38 @@ fn build_step_config_and_factory(
});
Ok((key, value, factory))
}
#[cfg(feature = "rustlang")]
"cargo-build" | "cargo-test" | "cargo-check" | "cargo-clippy" | "cargo-fmt"
| "cargo-doc" | "cargo-publish" | "cargo-audit" | "cargo-deny" | "cargo-nextest"
| "cargo-llvm-cov" | "cargo-doc-mdx" => {
let config = build_cargo_config(step, step_type)?;
let key = format!("wfe_yaml::cargo::{}", step.name);
let value = serde_json::to_value(&config).map_err(|e| {
YamlWorkflowError::Compilation(format!(
"Failed to serialize cargo config: {e}"
))
})?;
let config_clone = config.clone();
let factory: StepFactory = Box::new(move || {
Box::new(CargoStep::new(config_clone.clone())) as Box<dyn StepBody>
});
Ok((key, value, factory))
}
#[cfg(feature = "rustlang")]
"rust-install" | "rustup-toolchain" | "rustup-component" | "rustup-target" => {
let config = build_rustup_config(step, step_type)?;
let key = format!("wfe_yaml::rustup::{}", step.name);
let value = serde_json::to_value(&config).map_err(|e| {
YamlWorkflowError::Compilation(format!(
"Failed to serialize rustup config: {e}"
))
})?;
let config_clone = config.clone();
let factory: StepFactory = Box::new(move || {
Box::new(RustupStep::new(config_clone.clone())) as Box<dyn StepBody>
});
Ok((key, value, factory))
}
"workflow" => {
let config = step.config.as_ref().ok_or_else(|| {
YamlWorkflowError::Compilation(format!(
@@ -576,6 +610,88 @@ fn build_shell_config(step: &YamlStep) -> Result<ShellConfig, YamlWorkflowError>
})
}
#[cfg(feature = "rustlang")]
fn build_cargo_config(
step: &YamlStep,
step_type: &str,
) -> Result<CargoConfig, YamlWorkflowError> {
let command = match step_type {
"cargo-build" => CargoCommand::Build,
"cargo-test" => CargoCommand::Test,
"cargo-check" => CargoCommand::Check,
"cargo-clippy" => CargoCommand::Clippy,
"cargo-fmt" => CargoCommand::Fmt,
"cargo-doc" => CargoCommand::Doc,
"cargo-publish" => CargoCommand::Publish,
"cargo-audit" => CargoCommand::Audit,
"cargo-deny" => CargoCommand::Deny,
"cargo-nextest" => CargoCommand::Nextest,
"cargo-llvm-cov" => CargoCommand::LlvmCov,
"cargo-doc-mdx" => CargoCommand::DocMdx,
_ => {
return Err(YamlWorkflowError::Compilation(format!(
"Unknown cargo step type: '{step_type}'"
)));
}
};
let config = step.config.as_ref();
let timeout_ms = config
.and_then(|c| c.timeout.as_ref())
.and_then(|t| parse_duration_ms(t));
Ok(CargoConfig {
command,
toolchain: config.and_then(|c| c.toolchain.clone()),
package: config.and_then(|c| c.package.clone()),
features: config.map(|c| c.features.clone()).unwrap_or_default(),
all_features: config.and_then(|c| c.all_features).unwrap_or(false),
no_default_features: config.and_then(|c| c.no_default_features).unwrap_or(false),
release: config.and_then(|c| c.release).unwrap_or(false),
target: config.and_then(|c| c.target.clone()),
profile: config.and_then(|c| c.profile.clone()),
extra_args: config.map(|c| c.extra_args.clone()).unwrap_or_default(),
env: config.map(|c| c.env.clone()).unwrap_or_default(),
working_dir: config.and_then(|c| c.working_dir.clone()),
timeout_ms,
output_dir: config.and_then(|c| c.output_dir.clone()),
})
}
#[cfg(feature = "rustlang")]
fn build_rustup_config(
step: &YamlStep,
step_type: &str,
) -> Result<RustupConfig, YamlWorkflowError> {
let command = match step_type {
"rust-install" => RustupCommand::Install,
"rustup-toolchain" => RustupCommand::ToolchainInstall,
"rustup-component" => RustupCommand::ComponentAdd,
"rustup-target" => RustupCommand::TargetAdd,
_ => {
return Err(YamlWorkflowError::Compilation(format!(
"Unknown rustup step type: '{step_type}'"
)));
}
};
let config = step.config.as_ref();
let timeout_ms = config
.and_then(|c| c.timeout.as_ref())
.and_then(|t| parse_duration_ms(t));
Ok(RustupConfig {
command,
toolchain: config.and_then(|c| c.toolchain.clone()),
components: config.map(|c| c.components.clone()).unwrap_or_default(),
targets: config.map(|c| c.targets.clone()).unwrap_or_default(),
profile: config.and_then(|c| c.profile.clone()),
default_toolchain: config.and_then(|c| c.default_toolchain.clone()),
extra_args: config.map(|c| c.extra_args.clone()).unwrap_or_default(),
timeout_ms,
})
}
fn parse_duration_ms(s: &str) -> Option<u64> {
let s = s.trim();
// Check "ms" before "s" since strip_suffix('s') would also match "500ms"

33
wfe-yaml/src/schema.rs
View File

@@ -164,6 +164,39 @@ pub struct StepConfig {
pub containerd_addr: Option<String>,
/// CLI binary name for containerd steps: "nerdctl" (default) or "docker".
pub cli: Option<String>,
// Cargo fields
/// Target package for cargo steps (`-p`).
pub package: Option<String>,
/// Features to enable for cargo steps.
#[serde(default)]
pub features: Vec<String>,
/// Enable all features for cargo steps.
#[serde(default)]
pub all_features: Option<bool>,
/// Disable default features for cargo steps.
#[serde(default)]
pub no_default_features: Option<bool>,
/// Build in release mode for cargo steps.
#[serde(default)]
pub release: Option<bool>,
/// Build profile for cargo steps (`--profile`).
pub profile: Option<String>,
/// Rust toolchain override for cargo steps (e.g. "nightly").
pub toolchain: Option<String>,
/// Additional arguments for cargo/rustup steps.
#[serde(default)]
pub extra_args: Vec<String>,
/// Output directory for generated files (e.g., MDX docs).
pub output_dir: Option<String>,
// Rustup fields
/// Components to add for rustup steps (e.g. ["clippy", "rustfmt"]).
#[serde(default)]
pub components: Vec<String>,
/// Compilation targets to add for rustup steps (e.g. ["wasm32-unknown-unknown"]).
#[serde(default)]
pub targets: Vec<String>,
/// Default toolchain for rust-install steps.
pub default_toolchain: Option<String>,
// Workflow (sub-workflow) fields
/// Child workflow ID (for `type: workflow` steps).
#[serde(rename = "workflow")]

777
wfe-yaml/tests/rustlang.rs Normal file
View File

@@ -0,0 +1,777 @@
#![cfg(feature = "rustlang")]
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use wfe::models::WorkflowStatus;
use wfe::{WorkflowHostBuilder, run_workflow_sync};
use wfe_core::test_support::{
InMemoryLockProvider, InMemoryPersistenceProvider, InMemoryQueueProvider,
};
use wfe_yaml::load_single_workflow_from_str;
fn has_factory(compiled: &wfe_yaml::compiler::CompiledWorkflow, key: &str) -> bool {
compiled.step_factories.iter().any(|(k, _)| k == key)
}
async fn run_yaml_workflow(yaml: &str) -> wfe::models::WorkflowInstance {
let config = HashMap::new();
let compiled = load_single_workflow_from_str(yaml, &config).unwrap();
let persistence = Arc::new(InMemoryPersistenceProvider::new());
let lock = Arc::new(InMemoryLockProvider::new());
let queue = Arc::new(InMemoryQueueProvider::new());
let host = WorkflowHostBuilder::new()
.use_persistence(persistence as Arc<dyn wfe_core::traits::PersistenceProvider>)
.use_lock_provider(lock as Arc<dyn wfe_core::traits::DistributedLockProvider>)
.use_queue_provider(queue as Arc<dyn wfe_core::traits::QueueProvider>)
.build()
.unwrap();
for (key, factory) in compiled.step_factories {
host.register_step_factory(&key, factory).await;
}
host.register_workflow_definition(compiled.definition.clone())
.await;
host.start().await.unwrap();
let instance = run_workflow_sync(
&host,
&compiled.definition.id,
compiled.definition.version,
serde_json::json!({}),
Duration::from_secs(30),
)
.await
.unwrap();
host.stop().await;
instance
}
// ---------------------------------------------------------------------------
// Compiler tests — verify YAML compiles to correct step types and configs
// ---------------------------------------------------------------------------
#[test]
fn compile_cargo_build_step() {
let yaml = r#"
workflow:
id: cargo-build-wf
version: 1
steps:
- name: build
type: cargo-build
config:
release: true
package: my-crate
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
let step = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("build"))
.unwrap();
assert_eq!(step.step_type, "wfe_yaml::cargo::build");
assert!(has_factory(&compiled, "wfe_yaml::cargo::build"));
}
#[test]
fn compile_cargo_test_step() {
let yaml = r#"
workflow:
id: cargo-test-wf
version: 1
steps:
- name: test
type: cargo-test
config:
features:
- feat1
- feat2
extra_args:
- "--"
- "--nocapture"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
let step = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("test"))
.unwrap();
assert_eq!(step.step_type, "wfe_yaml::cargo::test");
}
#[test]
fn compile_cargo_check_step() {
let yaml = r#"
workflow:
id: cargo-check-wf
version: 1
steps:
- name: check
type: cargo-check
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::check"));
}
#[test]
fn compile_cargo_clippy_step() {
let yaml = r#"
workflow:
id: cargo-clippy-wf
version: 1
steps:
- name: lint
type: cargo-clippy
config:
all_features: true
extra_args:
- "--"
- "-D"
- "warnings"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::lint"));
}
#[test]
fn compile_cargo_fmt_step() {
let yaml = r#"
workflow:
id: cargo-fmt-wf
version: 1
steps:
- name: format
type: cargo-fmt
config:
extra_args:
- "--check"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::format"));
}
#[test]
fn compile_cargo_doc_step() {
let yaml = r#"
workflow:
id: cargo-doc-wf
version: 1
steps:
- name: docs
type: cargo-doc
config:
extra_args:
- "--no-deps"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::docs"));
}
#[test]
fn compile_cargo_publish_step() {
let yaml = r#"
workflow:
id: cargo-publish-wf
version: 1
steps:
- name: publish
type: cargo-publish
config:
extra_args:
- "--dry-run"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::publish"));
}
#[test]
fn compile_cargo_step_with_toolchain() {
let yaml = r#"
workflow:
id: nightly-wf
version: 1
steps:
- name: nightly-check
type: cargo-check
config:
toolchain: nightly
no_default_features: true
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::nightly-check"));
}
#[test]
fn compile_cargo_step_with_timeout() {
let yaml = r#"
workflow:
id: timeout-wf
version: 1
steps:
- name: slow-build
type: cargo-build
config:
timeout: 5m
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::slow-build"));
}
#[test]
fn compile_cargo_step_without_config() {
let yaml = r#"
workflow:
id: bare-wf
version: 1
steps:
- name: bare-check
type: cargo-check
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::bare-check"));
}
#[test]
fn compile_cargo_multi_step_pipeline() {
let yaml = r#"
workflow:
id: ci-pipeline
version: 1
steps:
- name: fmt
type: cargo-fmt
config:
extra_args: ["--check"]
- name: check
type: cargo-check
- name: clippy
type: cargo-clippy
config:
extra_args: ["--", "-D", "warnings"]
- name: test
type: cargo-test
- name: build
type: cargo-build
config:
release: true
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::check"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::clippy"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::test"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::build"));
}
#[test]
fn compile_cargo_step_with_all_shared_flags() {
let yaml = r#"
workflow:
id: full-flags-wf
version: 1
steps:
- name: full
type: cargo-build
config:
package: my-crate
features: [foo, bar]
all_features: false
no_default_features: true
release: true
toolchain: stable
profile: release
extra_args: ["--jobs", "4"]
working_dir: /tmp/project
timeout: 30s
env:
RUSTFLAGS: "-C target-cpu=native"
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::full"));
}
#[test]
fn compile_cargo_step_preserves_step_config_json() {
let yaml = r#"
workflow:
id: config-json-wf
version: 1
steps:
- name: build
type: cargo-build
config:
release: true
package: wfe-core
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
let step = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("build"))
.unwrap();
let step_config = step.step_config.as_ref().unwrap();
assert_eq!(step_config["command"], "build");
assert_eq!(step_config["release"], true);
assert_eq!(step_config["package"], "wfe-core");
}
// ---------------------------------------------------------------------------
// Integration tests — run actual cargo commands through the workflow engine
// ---------------------------------------------------------------------------
#[tokio::test]
async fn cargo_check_on_self_succeeds() {
let yaml = r#"
workflow:
id: self-check
version: 1
steps:
- name: check
type: cargo-check
config:
working_dir: .
timeout: 120s
"#;
let instance = run_yaml_workflow(yaml).await;
assert_eq!(instance.status, WorkflowStatus::Complete);
let data = instance.data.as_object().unwrap();
assert!(data.contains_key("check.stdout") || data.contains_key("check.stderr"));
}
#[tokio::test]
async fn cargo_fmt_check_compiles() {
let yaml = r#"
workflow:
id: fmt-check
version: 1
steps:
- name: fmt
type: cargo-fmt
config:
working_dir: .
extra_args: ["--check"]
timeout: 60s
"#;
let config = HashMap::new();
let compiled = load_single_workflow_from_str(yaml, &config).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt"));
}
// ---------------------------------------------------------------------------
// Rustup compiler tests
// ---------------------------------------------------------------------------
#[test]
fn compile_rust_install_step() {
let yaml = r#"
workflow:
id: rust-install-wf
version: 1
steps:
- name: install-rust
type: rust-install
config:
profile: minimal
default_toolchain: stable
timeout: 5m
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
let step = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("install-rust"))
.unwrap();
assert_eq!(step.step_type, "wfe_yaml::rustup::install-rust");
assert!(has_factory(&compiled, "wfe_yaml::rustup::install-rust"));
}
#[test]
fn compile_rustup_toolchain_step() {
let yaml = r#"
workflow:
id: tc-install-wf
version: 1
steps:
- name: add-nightly
type: rustup-toolchain
config:
toolchain: nightly-2024-06-01
profile: minimal
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-nightly"));
}
#[test]
fn compile_rustup_component_step() {
let yaml = r#"
workflow:
id: comp-add-wf
version: 1
steps:
- name: add-tools
type: rustup-component
config:
components: [clippy, rustfmt, rust-src]
toolchain: nightly
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-tools"));
}
#[test]
fn compile_rustup_target_step() {
let yaml = r#"
workflow:
id: target-add-wf
version: 1
steps:
- name: add-wasm
type: rustup-target
config:
targets: [wasm32-unknown-unknown]
toolchain: stable
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-wasm"));
}
#[test]
fn compile_rustup_step_without_config() {
let yaml = r#"
workflow:
id: bare-install-wf
version: 1
steps:
- name: install
type: rust-install
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::install"));
}
#[test]
fn compile_rustup_step_preserves_config_json() {
let yaml = r#"
workflow:
id: config-json-wf
version: 1
steps:
- name: tc
type: rustup-toolchain
config:
toolchain: nightly
profile: minimal
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
let step = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("tc"))
.unwrap();
let step_config = step.step_config.as_ref().unwrap();
assert_eq!(step_config["command"], "toolchain-install");
assert_eq!(step_config["toolchain"], "nightly");
assert_eq!(step_config["profile"], "minimal");
}
#[test]
fn compile_full_rust_ci_pipeline() {
let yaml = r#"
workflow:
id: full-rust-ci
version: 1
steps:
- name: install
type: rust-install
config:
profile: minimal
default_toolchain: stable
- name: add-nightly
type: rustup-toolchain
config:
toolchain: nightly
- name: add-components
type: rustup-component
config:
components: [clippy, rustfmt]
- name: add-wasm
type: rustup-target
config:
targets: [wasm32-unknown-unknown]
- name: fmt
type: cargo-fmt
config:
extra_args: ["--check"]
- name: check
type: cargo-check
- name: clippy
type: cargo-clippy
config:
extra_args: ["--", "-D", "warnings"]
- name: test
type: cargo-test
- name: build
type: cargo-build
config:
release: true
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::install"));
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-nightly"));
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-components"));
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-wasm"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::fmt"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::check"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::clippy"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::test"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::build"));
}
#[test]
fn compile_rustup_component_with_extra_args() {
let yaml = r#"
workflow:
id: comp-extra-wf
version: 1
steps:
- name: add-llvm
type: rustup-component
config:
components: [llvm-tools-preview]
extra_args: ["--force"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::add-llvm"));
}
#[test]
fn compile_rustup_target_multiple() {
let yaml = r#"
workflow:
id: multi-target-wf
version: 1
steps:
- name: cross-targets
type: rustup-target
config:
targets:
- wasm32-unknown-unknown
- aarch64-linux-android
- x86_64-unknown-linux-musl
toolchain: nightly
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::rustup::cross-targets"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("cross-targets"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "target-add");
let targets = step_config["targets"].as_array().unwrap();
assert_eq!(targets.len(), 3);
}
// ---------------------------------------------------------------------------
// External cargo tool step compiler tests
// ---------------------------------------------------------------------------
#[test]
fn compile_cargo_audit_step() {
let yaml = r#"
workflow:
id: audit-wf
version: 1
steps:
- name: audit
type: cargo-audit
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::audit"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("audit"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "audit");
}
#[test]
fn compile_cargo_deny_step() {
let yaml = r#"
workflow:
id: deny-wf
version: 1
steps:
- name: license-check
type: cargo-deny
config:
extra_args: ["check"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::license-check"));
}
#[test]
fn compile_cargo_nextest_step() {
let yaml = r#"
workflow:
id: nextest-wf
version: 1
steps:
- name: fast-test
type: cargo-nextest
config:
features: [foo]
extra_args: ["--no-fail-fast"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::fast-test"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("fast-test"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "nextest");
}
#[test]
fn compile_cargo_llvm_cov_step() {
let yaml = r#"
workflow:
id: cov-wf
version: 1
steps:
- name: coverage
type: cargo-llvm-cov
config:
extra_args: ["--html", "--output-dir", "coverage"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::coverage"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("coverage"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "llvm-cov");
}
#[test]
fn compile_full_ci_with_external_tools() {
let yaml = r#"
workflow:
id: full-ci-external
version: 1
steps:
- name: audit
type: cargo-audit
- name: deny
type: cargo-deny
config:
extra_args: ["check", "licenses"]
- name: test
type: cargo-nextest
- name: coverage
type: cargo-llvm-cov
config:
extra_args: ["--summary-only"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::audit"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::deny"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::test"));
assert!(has_factory(&compiled, "wfe_yaml::cargo::coverage"));
}
#[test]
fn compile_cargo_doc_mdx_step() {
let yaml = r#"
workflow:
id: doc-mdx-wf
version: 1
steps:
- name: docs
type: cargo-doc-mdx
config:
package: my-crate
output_dir: docs/api
extra_args: ["--no-deps"]
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::docs"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("docs"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "doc-mdx");
assert_eq!(step_config["package"], "my-crate");
assert_eq!(step_config["output_dir"], "docs/api");
}
#[test]
fn compile_cargo_doc_mdx_minimal() {
let yaml = r#"
workflow:
id: doc-mdx-minimal-wf
version: 1
steps:
- name: generate-docs
type: cargo-doc-mdx
"#;
let compiled = load_single_workflow_from_str(yaml, &HashMap::new()).unwrap();
assert!(has_factory(&compiled, "wfe_yaml::cargo::generate-docs"));
let step_config = compiled
.definition
.steps
.iter()
.find(|s| s.name.as_deref() == Some("generate-docs"))
.unwrap()
.step_config
.as_ref()
.unwrap();
assert_eq!(step_config["command"], "doc-mdx");
assert!(step_config["output_dir"].is_null());
}

474
wfe-yaml/tests/rustlang_containerd.rs Normal file
View File

@@ -0,0 +1,474 @@
//! End-to-end integration tests for the Rust toolchain steps running inside
//! containerd containers.
//!
//! These tests start from a bare Debian image (no Rust installed) and exercise
//! the full Rust CI pipeline: install Rust, install external tools, create a
//! test project, and run every cargo operation.
//!
//! Requirements:
//! - A running containerd daemon (Lima/colima or native)
//! - Set `WFE_CONTAINERD_ADDR` to point to the socket
//!
//! These tests are gated behind `rustlang` + `containerd` features and are
//! marked `#[ignore]` so they don't run in normal CI. Run them explicitly:
//! cargo test -p wfe-yaml --features rustlang,containerd --test rustlang_containerd -- --ignored
#![cfg(all(feature = "rustlang", feature = "containerd"))]
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use wfe::models::WorkflowStatus;
use wfe::{WorkflowHostBuilder, run_workflow_sync};
use wfe_core::test_support::{
InMemoryLockProvider, InMemoryPersistenceProvider, InMemoryQueueProvider,
};
use wfe_yaml::load_single_workflow_from_str;
/// Returns the containerd address if available, or None.
/// Supports both Unix sockets (`unix:///path`) and TCP (`http://host:port`).
fn containerd_addr() -> Option<String> {
let addr = std::env::var("WFE_CONTAINERD_ADDR").unwrap_or_else(|_| {
// Default: TCP proxy on the Lima VM (socat forwarding containerd socket)
"http://127.0.0.1:2500".to_string()
});
// For TCP addresses, assume reachable (the test will fail fast if not).
if addr.starts_with("http://") || addr.starts_with("tcp://") {
return Some(addr);
}
// For Unix sockets, check the file exists.
let socket_path = addr.strip_prefix("unix://").unwrap_or(addr.as_str());
if Path::new(socket_path).exists() {
Some(addr)
} else {
None
}
}
async fn run_yaml_workflow_with_config(
yaml: &str,
config: &HashMap<String, serde_json::Value>,
) -> wfe::models::WorkflowInstance {
let compiled = load_single_workflow_from_str(yaml, config).unwrap();
for step in &compiled.definition.steps {
eprintln!(" step: {:?} type={} config={:?}", step.name, step.step_type, step.step_config);
}
eprintln!(" factories: {:?}", compiled.step_factories.iter().map(|(k, _)| k.clone()).collect::<Vec<_>>());
let persistence = Arc::new(InMemoryPersistenceProvider::new());
let lock = Arc::new(InMemoryLockProvider::new());
let queue = Arc::new(InMemoryQueueProvider::new());
let host = WorkflowHostBuilder::new()
.use_persistence(persistence as Arc<dyn wfe_core::traits::PersistenceProvider>)
.use_lock_provider(lock as Arc<dyn wfe_core::traits::DistributedLockProvider>)
.use_queue_provider(queue as Arc<dyn wfe_core::traits::QueueProvider>)
.build()
.unwrap();
for (key, factory) in compiled.step_factories {
host.register_step_factory(&key, factory).await;
}
host.register_workflow_definition(compiled.definition.clone())
.await;
host.start().await.unwrap();
let instance = run_workflow_sync(
&host,
&compiled.definition.id,
compiled.definition.version,
serde_json::json!({}),
Duration::from_secs(1800),
)
.await
.unwrap();
host.stop().await;
instance
}
/// Shared env block and volume template for containerd steps.
/// Uses format! to avoid Rust 2024 reserved `##` token in raw strings.
fn containerd_step_yaml(
name: &str,
network: &str,
pull: &str,
timeout: &str,
working_dir: Option<&str>,
mount_workspace: bool,
run_script: &str,
) -> String {
let wfe = "##wfe";
let wd = working_dir
.map(|d| format!(" working_dir: {d}"))
.unwrap_or_default();
let ws_volume = if mount_workspace {
" - source: ((workspace))\n target: /workspace"
} else {
""
};
format!(
r#" - name: {name}
type: containerd
config:
image: docker.io/library/debian:bookworm-slim
containerd_addr: ((containerd_addr))
user: "0:0"
network: {network}
pull: {pull}
timeout: {timeout}
{wd}
env:
CARGO_HOME: /cargo
RUSTUP_HOME: /rustup
PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
volumes:
- source: ((cargo_home))
target: /cargo
- source: ((rustup_home))
target: /rustup
{ws_volume}
run: |
{run_script}
echo "{wfe}[output {name}.status=ok]"
"#
)
}
/// Base directory for shared state between host and containerd VM.
/// Must be inside the virtiofs mount defined in test/lima/wfe-test.yaml.
fn shared_dir() -> std::path::PathBuf {
let base = std::env::var("WFE_IO_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|_| std::path::PathBuf::from("/tmp/wfe-io"));
std::fs::create_dir_all(&base).unwrap();
base
}
/// Create a temporary directory inside the shared mount so containerd can see it.
fn shared_tempdir(name: &str) -> std::path::PathBuf {
let id = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let dir = shared_dir().join(format!("{name}-{id}"));
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn make_config(
addr: &str,
cargo_home: &Path,
rustup_home: &Path,
workspace: Option<&Path>,
) -> HashMap<String, serde_json::Value> {
let mut config = HashMap::new();
config.insert(
"containerd_addr".to_string(),
serde_json::Value::String(addr.to_string()),
);
config.insert(
"cargo_home".to_string(),
serde_json::Value::String(cargo_home.to_str().unwrap().to_string()),
);
config.insert(
"rustup_home".to_string(),
serde_json::Value::String(rustup_home.to_str().unwrap().to_string()),
);
if let Some(ws) = workspace {
config.insert(
"workspace".to_string(),
serde_json::Value::String(ws.to_str().unwrap().to_string()),
);
}
config
}
// ---------------------------------------------------------------------------
// Minimal: just echo hello in a containerd step through the workflow engine
// ---------------------------------------------------------------------------
#[tokio::test]
#[ignore = "requires containerd daemon"]
async fn minimal_echo_in_containerd_via_workflow() {
let _ = tracing_subscriber::fmt().with_env_filter("wfe_containerd=debug,wfe_core::executor=debug").try_init();
let Some(addr) = containerd_addr() else {
eprintln!("SKIP: containerd not available");
return;
};
let mut config = HashMap::new();
config.insert(
"containerd_addr".to_string(),
serde_json::Value::String(addr),
);
let wfe = "##wfe";
let yaml = format!(
r#"workflow:
id: minimal-containerd
version: 1
error_behavior:
type: terminate
steps:
- name: echo
type: containerd
config:
image: docker.io/library/alpine:3.18
containerd_addr: ((containerd_addr))
user: "0:0"
network: none
pull: if-not-present
timeout: 30s
run: |
echo hello-from-workflow
echo "{wfe}[output echo.status=ok]"
"#
);
let instance = run_yaml_workflow_with_config(&yaml, &config).await;
eprintln!("Status: {:?}, Data: {:?}", instance.status, instance.data);
assert_eq!(instance.status, WorkflowStatus::Complete);
let data = instance.data.as_object().unwrap();
assert_eq!(
data.get("echo.status").and_then(|v| v.as_str()),
Some("ok"),
);
}
// ---------------------------------------------------------------------------
// Full Rust CI pipeline in a container: install → build → test → lint → cover
// ---------------------------------------------------------------------------
#[tokio::test]
#[ignore = "requires containerd daemon"]
async fn full_rust_pipeline_in_container() {
let Some(addr) = containerd_addr() else {
eprintln!("SKIP: containerd socket not available");
return;
};
let cargo_home = shared_tempdir("cargo");
let rustup_home = shared_tempdir("rustup");
let workspace = shared_tempdir("workspace");
let config = make_config(
&addr,
&cargo_home,
&rustup_home,
Some(&workspace),
);
let steps = [
containerd_step_yaml(
"install-rust", "host", "if-not-present", "10m", None, false,
" apt-get update && apt-get install -y curl gcc pkg-config libssl-dev\n\
\x20 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable",
),
containerd_step_yaml(
"install-tools", "host", "never", "10m", None, false,
" rustup component add clippy rustfmt llvm-tools-preview\n\
\x20 cargo install cargo-audit cargo-deny cargo-nextest cargo-llvm-cov",
),
containerd_step_yaml(
"create-project", "host", "never", "2m", None, true,
" cargo init /workspace/test-crate --name test-crate\n\
\x20 cd /workspace/test-crate\n\
\x20 echo '#[cfg(test)] mod tests { #[test] fn it_works() { assert_eq!(2+2,4); } }' >> src/main.rs",
),
containerd_step_yaml(
"cargo-fmt", "none", "never", "2m",
Some("/workspace/test-crate"), true,
" cargo fmt -- --check || cargo fmt",
),
containerd_step_yaml(
"cargo-check", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo check",
),
containerd_step_yaml(
"cargo-clippy", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo clippy -- -D warnings",
),
containerd_step_yaml(
"cargo-test", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo test",
),
containerd_step_yaml(
"cargo-build", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo build --release",
),
containerd_step_yaml(
"cargo-nextest", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo nextest run",
),
containerd_step_yaml(
"cargo-llvm-cov", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo llvm-cov --summary-only",
),
containerd_step_yaml(
"cargo-audit", "host", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo audit || true",
),
containerd_step_yaml(
"cargo-deny", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo deny init\n\
\x20 cargo deny check || true",
),
containerd_step_yaml(
"cargo-doc", "none", "never", "5m",
Some("/workspace/test-crate"), true,
" cargo doc --no-deps",
),
];
let yaml = format!(
"workflow:\n id: rust-container-pipeline\n version: 1\n error_behavior:\n type: terminate\n steps:\n{}",
steps.join("\n")
);
let instance = run_yaml_workflow_with_config(&yaml, &config).await;
assert_eq!(
instance.status,
WorkflowStatus::Complete,
"workflow should complete successfully, data: {:?}",
instance.data
);
let data = instance.data.as_object().unwrap();
for key in [
"install-rust.status",
"install-tools.status",
"create-project.status",
"cargo-fmt.status",
"cargo-check.status",
"cargo-clippy.status",
"cargo-test.status",
"cargo-build.status",
"cargo-nextest.status",
"cargo-llvm-cov.status",
"cargo-audit.status",
"cargo-deny.status",
"cargo-doc.status",
] {
assert_eq!(
data.get(key).and_then(|v| v.as_str()),
Some("ok"),
"step output '{key}' should be 'ok', got: {:?}",
data.get(key)
);
}
}
// ---------------------------------------------------------------------------
// Focused test: just rust-install in a bare container
// ---------------------------------------------------------------------------
#[tokio::test]
#[ignore = "requires containerd daemon"]
async fn rust_install_in_bare_container() {
let Some(addr) = containerd_addr() else {
eprintln!("SKIP: containerd socket not available");
return;
};
let cargo_home = shared_tempdir("cargo");
let rustup_home = shared_tempdir("rustup");
let config = make_config(&addr, &cargo_home, &rustup_home, None);
let wfe = "##wfe";
let yaml = format!(
r#"workflow:
id: rust-install-container
version: 1
error_behavior:
type: terminate
steps:
- name: install
type: containerd
config:
image: docker.io/library/debian:bookworm-slim
containerd_addr: ((containerd_addr))
user: "0:0"
network: host
pull: if-not-present
timeout: 10m
env:
CARGO_HOME: /cargo
RUSTUP_HOME: /rustup
PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
volumes:
- source: ((cargo_home))
target: /cargo
- source: ((rustup_home))
target: /rustup
run: |
apt-get update && apt-get install -y curl
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
rustc --version
cargo --version
echo "{wfe}[output rustc_installed=true]"
- name: verify
type: containerd
config:
image: docker.io/library/debian:bookworm-slim
containerd_addr: ((containerd_addr))
user: "0:0"
network: none
pull: if-not-present
timeout: 2m
env:
CARGO_HOME: /cargo
RUSTUP_HOME: /rustup
PATH: /cargo/bin:/rustup/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
volumes:
- source: ((cargo_home))
target: /cargo
- source: ((rustup_home))
target: /rustup
run: |
rustc --version
cargo --version
echo "{wfe}[output verify.status=ok]"
"#
);
let instance = run_yaml_workflow_with_config(&yaml, &config).await;
assert_eq!(
instance.status,
WorkflowStatus::Complete,
"install workflow should complete, data: {:?}",
instance.data
);
let data = instance.data.as_object().unwrap();
eprintln!("Workflow data: {:?}", instance.data);
assert!(
data.get("rustc_installed").is_some(),
"rustc_installed should be set, got data: {:?}",
data
);
assert_eq!(
data.get("verify.status").and_then(|v| v.as_str()),
Some("ok"),
);
}