diff --git a/workflows.yaml b/workflows.yaml index 283f482..6fff72c 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -1,368 +1,356 @@ -# workflows.yaml — WFE self-hosting CI pipeline +# workflows.yaml -- WFE self-hosting CI pipeline (Kubernetes-native) # -# Demonstrates every WFE feature. Idempotent — safe to run repeatedly. +# Designed to run on every push via wfe-server webhook → ci workflow. +# Every step runs as a Kubernetes Job in the wfe namespace; every service +# (postgres, valkey, opensearch) is a K8s Pod + Service in the workflow's +# scoped namespace. # -# Usage: -# cargo run --example run_pipeline -p wfe -- workflows.yaml +# === Prerequisites === # -# With config: -# WFE_CONFIG='{"workspace_dir":"/path/to/wfe","registry":"sunbeam","git_remote":"origin","coverage_threshold":85}' \ -# cargo run --example run_pipeline -p wfe -- workflows.yaml +# 1. wfe-server deployed to the K8s cluster (see ../sbbb/base/wfe/) +# Reads webhooks at /webhooks/gitea, persists to postgres, locks via valkey. # -# TODO: Support multi-file merging — individual task files (e.g., lint.yaml, -# test.yaml, publish.yaml) that compose into a single pipeline definition. +# 2. wfe-credentials Secret in the wfe namespace (via Vault Secrets Operator). +# Required keys: +# sccache-s3-endpoint: S3-compatible endpoint for sccache +# sccache-bucket: S3 bucket name +# sccache-region: S3 region +# aws-access-key-id: S3 credentials for sccache +# aws-secret-access-key: S3 credentials for sccache +# cargo-registry-token: sunbeam registry publish token +# gitea-token: tea CLI token for releases +# buildkit-ca-cert: PEM-encoded CA cert (mTLS to buildkitd) +# buildkit-client-cert: PEM client cert +# buildkit-client-key: PEM client key +# +# 3. wfe-ci image at src.sunbeam.pt/studio/wfe-ci:latest +# Built from Dockerfile.ci -- contains rust, nextest, llvm-cov, sccache, +# buildctl, kubectl, tea, git. +# +# 4. buildkitd running in the build namespace at buildkitd.build.svc:1234 +# Uses mTLS; client cert mounted from wfe-credentials secret. -# ─── Shared Templates ─────────────────────────────────────────────── -# The _templates key is ignored by the workflow parser (extra keys are -# skipped). Anchors are resolved by serde_yaml before parsing. +# --- Shared Templates --- _templates: - shell_defaults: &shell_defaults - type: shell - config: - shell: bash - timeout: 5m + # Shared CI environment variables for the prebuilt wfe-ci image. + # Pulls secrets from wfe-credentials so sccache hits the shared S3 cache. + ci_env: &ci_env + SCCACHE_BUCKET: ${WFE_SCCACHE_BUCKET} + SCCACHE_REGION: ${WFE_SCCACHE_REGION} + SCCACHE_ENDPOINT: ${WFE_SCCACHE_S3_ENDPOINT} + SCCACHE_S3_USE_SSL: "true" + AWS_ACCESS_KEY_ID: ${WFE_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${WFE_AWS_SECRET_ACCESS_KEY} + CARGO_HOME: /workspace/.cargo + RUSTC_WRAPPER: /usr/local/cargo/bin/sccache + CARGO_INCREMENTAL: "0" + # Secrets used by individual steps (publish, image, release). Defined here + # so the shared *ci_env / *ci_config anchors stay flat — YAML 1.1 merge + # keys are shallow, so a step can't override `env:` and still inherit. + GITEA_TOKEN: ${WFE_GITEA_TOKEN} + TEA_TOKEN: ${WFE_GITEA_TOKEN} + CARGO_REGISTRIES_SUNBEAM_TOKEN: ${WFE_CARGO_REGISTRY_TOKEN} + BUILDKIT_CA_CERT: ${WFE_BUILDKIT_CA_CERT} + BUILDKIT_CLIENT_CERT: ${WFE_BUILDKIT_CLIENT_CERT} + BUILDKIT_CLIENT_KEY: ${WFE_BUILDKIT_CLIENT_KEY} - long_running: &long_running - type: shell - config: - shell: bash - timeout: 30m + # Default config for short CI steps (4Gi memory, 30min timeout). + ci_config: &ci_config + image: src.sunbeam.pt/studio/wfe-ci:latest + memory: 4Gi + cpu: "2" + timeout: 30m + env: *ci_env -# ─── Workflow: preflight ─────────────────────────────────────────────── + # Default config for long-running CI steps (8Gi memory, 60min timeout). + ci_long_config: &ci_long_config + image: src.sunbeam.pt/studio/wfe-ci:latest + memory: 8Gi + cpu: "4" + timeout: 60m + env: *ci_env + +# --- Workflows --- workflows: - - id: preflight + + # === checkout: clone the repo into a shared workspace === + + - id: checkout + name: Checkout version: 1 inputs: - workspace_dir: string + repo_url: string + commit_sha: string outputs: - cargo_ok: bool - nextest_ok: bool - llvm_cov_ok: bool - docker_ok: bool - lima_ok: bool - buildctl_ok: bool - git_ok: bool + checkout_ok: bool + commit: string steps: - - name: check-tools - type: shell + - name: clone + type: kubernetes outputs: - - name: cargo_ok - - name: nextest_ok - - name: llvm_cov_ok - - name: docker_ok - - name: lima_ok - - name: buildctl_ok - - name: git_ok + - name: checkout_ok + - name: commit config: - shell: bash - timeout: 1m + <<: *ci_config run: | - CARGO_OK=false; NEXTEST_OK=false; LLVM_COV_OK=false - DOCKER_OK=false; LIMA_OK=false; BUILDCTL_OK=false; GIT_OK=false - - command -v cargo >/dev/null 2>&1 && CARGO_OK=true - command -v cargo-nextest >/dev/null 2>&1 && NEXTEST_OK=true - command -v cargo-llvm-cov >/dev/null 2>&1 && LLVM_COV_OK=true - command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 && DOCKER_OK=true - command -v limactl >/dev/null 2>&1 && LIMA_OK=true - command -v buildctl >/dev/null 2>&1 && BUILDCTL_OK=true - command -v git >/dev/null 2>&1 && GIT_OK=true - - echo "Tool availability:" - echo " cargo: $CARGO_OK" - echo " nextest: $NEXTEST_OK" - echo " llvm-cov: $LLVM_COV_OK" - echo " docker: $DOCKER_OK" - echo " lima: $LIMA_OK" - echo " buildctl: $BUILDCTL_OK" - echo " git: $GIT_OK" - - echo "##wfe[output cargo_ok=$CARGO_OK]" - echo "##wfe[output nextest_ok=$NEXTEST_OK]" - echo "##wfe[output llvm_cov_ok=$LLVM_COV_OK]" - echo "##wfe[output docker_ok=$DOCKER_OK]" - echo "##wfe[output lima_ok=$LIMA_OK]" - echo "##wfe[output buildctl_ok=$BUILDCTL_OK]" - echo "##wfe[output git_ok=$GIT_OK]" - - # Fail if essential tools are missing - if [ "$CARGO_OK" = "false" ] || [ "$NEXTEST_OK" = "false" ] || [ "$GIT_OK" = "false" ]; then - echo "ERROR: Essential tools missing (cargo, nextest, or git)" - exit 1 + set -euo pipefail + cd /workspace + if [ ! -d wfe ]; then + git clone "$REPO_URL" wfe fi + cd wfe + git fetch --all + git checkout "$COMMIT_SHA" + COMMIT=$(git rev-parse HEAD) + echo "Checked out: $COMMIT" + echo "##wfe[output checkout_ok=true]" + echo "##wfe[output commit=$COMMIT]" - # ─── Workflow: lint ────────────────────────────────────────────────── + # === lint: fmt + clippy === - id: lint + name: Lint version: 1 - inputs: - workspace_dir: string outputs: fmt_ok: bool clippy_ok: bool steps: - name: fmt-check - <<: *shell_defaults + type: kubernetes outputs: - name: fmt_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" + set -euo pipefail + cd /workspace/wfe cargo fmt --all -- --check echo "##wfe[output fmt_ok=true]" - name: clippy - <<: *shell_defaults + type: kubernetes outputs: - name: clippy_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" - cargo clippy --workspace -- -D warnings + set -euo pipefail + cd /workspace/wfe + cargo clippy --workspace --all-features -- -D warnings echo "##wfe[output clippy_ok=true]" - # ─── Workflow: test-unit ───────────────────────────────────────── + # === test-unit: pure unit tests, no external dependencies === - id: test-unit + name: Unit Tests version: 1 - inputs: - workspace_dir: string outputs: - tests_passed: bool - deno_tests_passed: bool + core_ok: bool + yaml_ok: bool + deno_ok: bool + kubernetes_ok: bool + rustlang_ok: bool steps: - name: core-tests - <<: *long_running + type: kubernetes outputs: - - name: tests_passed + - name: core_ok config: + <<: *ci_long_config run: | - cd "$WORKSPACE_DIR" - cargo nextest run -P ci - echo "##wfe[output tests_passed=true]" + set -euo pipefail + cd /workspace/wfe + cargo nextest run -p wfe-core -p wfe -P ci + echo "##wfe[output core_ok=true]" + + - name: yaml-tests + type: kubernetes + outputs: + - name: yaml_ok + config: + <<: *ci_long_config + run: | + set -euo pipefail + cd /workspace/wfe + cargo nextest run -p wfe-yaml \ + --features deno,buildkit,containerd,rustlang,kubernetes -P ci + echo "##wfe[output yaml_ok=true]" - name: deno-tests - <<: *long_running + type: kubernetes outputs: - - name: deno_tests_passed + - name: deno_ok config: + <<: *ci_long_config run: | - cd "$WORKSPACE_DIR" - cargo nextest run -p wfe-yaml --features deno -P ci - echo "##wfe[output deno_tests_passed=true]" + set -euo pipefail + cd /workspace/wfe + cargo nextest run -p wfe-deno -P ci + echo "##wfe[output deno_ok=true]" - - name: feature-tests - <<: *shell_defaults + - name: kubernetes-unit-tests + type: kubernetes + outputs: + - name: kubernetes_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" - cargo nextest run -p wfe-yaml --features buildkit,containerd,rustlang -P ci + set -euo pipefail + cd /workspace/wfe + cargo nextest run -p wfe-kubernetes --lib -P ci + echo "##wfe[output kubernetes_ok=true]" + + - name: rustlang-tests + type: kubernetes + outputs: + - name: rustlang_ok + config: + <<: *ci_config + run: | + set -euo pipefail + cd /workspace/wfe cargo nextest run -p wfe-rustlang -P ci + echo "##wfe[output rustlang_ok=true]" - # ─── Workflow: test-integration ────────────────────────────────── + # === test-integration: postgres + valkey + opensearch via K8s services === + # + # The wfe ServiceProvider creates Pods + K8s Services in the workflow's + # scoped namespace. Step containers reach services via in-cluster DNS: + # postgres → postgres..svc.cluster.local - id: test-integration + name: Integration Tests version: 1 - inputs: - workspace_dir: string outputs: postgres_ok: bool valkey_ok: bool opensearch_ok: bool + + services: + postgres: + image: postgres:17 + ports: [5432] + env: + POSTGRES_USER: wfe + POSTGRES_PASSWORD: wfe + POSTGRES_DB: wfe_test + readiness: + exec: ["pg_isready", "-U", "wfe"] + interval: 2s + timeout: 60s + retries: 30 + memory: 512Mi + cpu: 500m + + valkey: + image: valkey/valkey:8 + ports: [6379] + readiness: + tcp: 6379 + interval: 2s + timeout: 30s + retries: 15 + memory: 256Mi + cpu: 250m + + opensearch: + image: opensearchproject/opensearch:2 + ports: [9200] + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_INITIAL_ADMIN_PASSWORD: admin + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + readiness: + http: { port: 9200, path: / } + interval: 5s + timeout: 120s + retries: 24 + memory: 1536Mi + cpu: "1" + steps: - - name: docker-up - <<: *long_running - outputs: - - name: docker_started - config: - run: | - # Docker runs inside a lima VM. Start it if needed. - if ! command -v limactl >/dev/null 2>&1; then - echo "limactl not available — skipping integration tests" - echo "##wfe[output docker_started=false]" - exit 0 - fi - - # Start the docker lima VM if not running - if ! limactl list 2>/dev/null | grep -q "docker.*Running"; then - echo "Starting docker lima VM..." - limactl start docker 2>&1 || { - echo "Failed to start docker VM — skipping integration tests" - echo "##wfe[output docker_started=false]" - exit 0 - } - fi - - # Wait for Docker daemon to be ready - for i in $(seq 1 30); do - if docker info >/dev/null 2>&1; then - break - fi - echo "Waiting for Docker daemon... ($i/30)" - sleep 2 - done - - if ! docker info >/dev/null 2>&1; then - echo "Docker daemon not ready after 60s — skipping" - echo "##wfe[output docker_started=false]" - exit 0 - fi - - cd "$WORKSPACE_DIR" - docker compose up -d --wait - echo "##wfe[output docker_started=true]" - on_failure: - name: docker-up-failed - type: shell - config: - run: echo "Failed to start Docker services" - - name: postgres-tests - <<: *shell_defaults - when: - field: .outputs.docker_started - equals: true + type: kubernetes outputs: - name: postgres_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" + set -euo pipefail + cd /workspace/wfe + export DATABASE_URL="postgres://wfe:wfe@postgres:5432/wfe_test" cargo nextest run -p wfe-postgres -P ci echo "##wfe[output postgres_ok=true]" - name: valkey-tests - <<: *shell_defaults - when: - field: .outputs.docker_started - equals: true + type: kubernetes outputs: - name: valkey_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" + set -euo pipefail + cd /workspace/wfe + export VALKEY_URL="redis://valkey:6379" cargo nextest run -p wfe-valkey -P ci echo "##wfe[output valkey_ok=true]" - name: opensearch-tests - <<: *shell_defaults - when: - field: .outputs.docker_started - equals: true + type: kubernetes outputs: - name: opensearch_ok config: + <<: *ci_config run: | - cd "$WORKSPACE_DIR" + set -euo pipefail + cd /workspace/wfe + export OPENSEARCH_URL="http://opensearch:9200" cargo nextest run -p wfe-opensearch -P ci echo "##wfe[output opensearch_ok=true]" - ensure: - - name: docker-down - <<: *shell_defaults - config: - run: | - if docker info >/dev/null 2>&1; then - cd "$WORKSPACE_DIR" - docker compose down 2>/dev/null || true - fi + # === test-kubernetes: K8s executor + service provisioner === + # + # These tests need cluster access to create namespaces, pods, jobs, services. + # The wfe ServiceAccount must have RBAC for the necessary verbs. - # ─── Workflow: test-containers ─────────────────────────────────── - - - id: test-containers + - id: test-kubernetes + name: Kubernetes Tests version: 1 - inputs: - workspace_dir: string outputs: - buildkit_ok: bool - containerd_ok: bool + k8s_integration_ok: bool steps: - - name: lima-up - <<: *long_running + - name: k8s-integration-tests + type: kubernetes outputs: - - name: lima_started + - name: k8s_integration_ok config: + <<: *ci_long_config run: | - if ! command -v limactl >/dev/null 2>&1; then - echo "limactl not available — skipping container tests" - echo "##wfe[output lima_started=false]" - exit 0 - fi + set -euo pipefail + cd /workspace/wfe + # In-cluster auth via the pod's ServiceAccount. + cargo nextest run -p wfe-kubernetes --test integration -P ci + echo "##wfe[output k8s_integration_ok=true]" - # Start the wfe-test VM if not running - if ! limactl list 2>/dev/null | grep -q "wfe-test.*Running"; then - echo "Starting wfe-test lima VM..." - limactl start --name=wfe-test "$WORKSPACE_DIR/test/lima/wfe-test.yaml" 2>&1 || { - echo "Failed to start wfe-test VM — skipping container tests" - echo "##wfe[output lima_started=false]" - exit 0 - } - fi - - # Wait for TCP proxy ports (socat bridges to containerd/buildkit sockets) - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:2500 >/dev/null 2>&1 || [ $? -eq 56 ]; then - break - fi - echo "Waiting for containerd TCP proxy... ($i/30)" - sleep 2 - done - - echo "##wfe[output lima_started=true]" - - - name: buildkit-tests - <<: *shell_defaults - when: - field: .outputs.lima_started - equals: true - outputs: - - name: buildkit_ok - config: - run: | - cd "$WORKSPACE_DIR" - export WFE_BUILDKIT_ADDR="http://127.0.0.1:2501" - cargo nextest run -p wfe-buildkit -P ci - echo "##wfe[output buildkit_ok=true]" - - - name: containerd-tests - <<: *shell_defaults - when: - field: .outputs.lima_started - equals: true - outputs: - - name: containerd_ok - config: - run: | - cd "$WORKSPACE_DIR" - export WFE_CONTAINERD_ADDR="http://127.0.0.1:2500" - export WFE_IO_DIR="/tmp/wfe-io" - mkdir -p "$WFE_IO_DIR" - cargo nextest run -p wfe-containerd -P ci - cargo nextest run -p wfe-yaml --features rustlang,containerd --test rustlang_containerd -P ci -- --ignored - echo "##wfe[output containerd_ok=true]" - - ensure: - - name: lima-down - <<: *shell_defaults - config: - run: | - limactl stop wfe-test 2>/dev/null || true - - # ─── Workflow: test (orchestrator) ─────────────────────────────── + # === test (orchestrator) === - id: test + name: Tests version: 1 - inputs: - workspace_dir: string outputs: all_passed: bool steps: - name: run-unit type: workflow outputs: - - name: tests_passed - - name: deno_tests_passed + - name: core_ok + - name: yaml_ok + - name: deno_ok + - name: kubernetes_ok + - name: rustlang_ok config: workflow: test-unit version: 1 @@ -377,61 +365,85 @@ workflows: workflow: test-integration version: 1 - - name: run-containers + - name: run-kubernetes type: workflow outputs: - - name: buildkit_ok - - name: containerd_ok + - name: k8s_integration_ok config: - workflow: test-containers + workflow: test-kubernetes version: 1 - name: mark-passed - <<: *shell_defaults + type: kubernetes outputs: - name: all_passed config: + <<: *ci_config run: | echo "All test workflows completed" echo "##wfe[output all_passed=true]" - # ─── Workflow: cover ───────────────────────────────────────────── + # === cover: coverage report with threshold gate === - id: cover + name: Coverage version: 1 inputs: - workspace_dir: string - threshold: number? + coverage_threshold: number? outputs: line_coverage: number meets_threshold: bool + + services: + postgres: + image: postgres:17 + ports: [5432] + env: + POSTGRES_USER: wfe + POSTGRES_PASSWORD: wfe + POSTGRES_DB: wfe_test + readiness: + exec: ["pg_isready", "-U", "wfe"] + timeout: 60s + memory: 512Mi + + valkey: + image: valkey/valkey:8 + ports: [6379] + readiness: + tcp: 6379 + memory: 256Mi + + opensearch: + image: opensearchproject/opensearch:2 + ports: [9200] + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + readiness: + http: { port: 9200, path: / } + timeout: 120s + memory: 1536Mi + steps: - name: run-coverage - <<: *shell_defaults + type: kubernetes outputs: - - name: coverage_ok + - name: coverage_json config: + <<: *ci_long_config run: | - cd "$WORKSPACE_DIR" - if ! command -v cargo-llvm-cov >/dev/null 2>&1; then - echo "cargo-llvm-cov not installed — skipping coverage" - echo "##wfe[output coverage_ok=false]" - exit 0 - fi - cargo llvm-cov nextest -P cover --json 2>&1 | grep '^{' > /tmp/wfe-coverage.json || true - if [ -s /tmp/wfe-coverage.json ]; then - echo "##wfe[output coverage_ok=true]" - else - echo "Coverage JSON not produced — llvm-cov may have failed" - echo "##wfe[output coverage_ok=false]" - fi + set -euo pipefail + cd /workspace/wfe + export DATABASE_URL="postgres://wfe:wfe@postgres:5432/wfe_test" + export VALKEY_URL="redis://valkey:6379" + export OPENSEARCH_URL="http://opensearch:9200" + cargo llvm-cov nextest --workspace -P cover --json > /tmp/wfe-coverage.json echo "##wfe[output coverage_json=/tmp/wfe-coverage.json]" - name: assert-threshold type: deno - when: - field: .outputs.coverage_ok - equals: true outputs: - name: line_coverage - name: meets_threshold @@ -440,7 +452,6 @@ workflows: const data = inputs(); const threshold = data.coverage_threshold || 85; - // Read the coverage JSON produced by run-coverage step const text = await readFile("/tmp/wfe-coverage.json"); const report = JSON.parse(text); @@ -458,269 +469,281 @@ workflows: permissions: read: ["/tmp"] - # ─── Workflow: package ─────────────────────────────────────────── - - - id: package - version: 1 - inputs: - workspace_dir: string - outputs: - packages_ok: bool - steps: - - name: package-all - <<: *shell_defaults - outputs: - - name: packages_ok - config: - run: | - echo "Packaging all crates (stub — remove exit 0 for real packaging)" - echo "##wfe[output packages_ok=true]" - exit 0 - cd "$WORKSPACE_DIR" - for crate in wfe-core wfe-sqlite wfe-postgres wfe-opensearch wfe-valkey \ - wfe-buildkit-protos wfe-containerd-protos wfe-buildkit wfe-containerd \ - wfe-rustlang wfe wfe-yaml; do - echo "Packaging $crate..." - cargo package -p "$crate" --no-verify --allow-dirty 2>&1 || exit 1 - done - echo "##wfe[output packages_ok=true]" - - # ─── Workflow: tag ─────────────────────────────────────────────── + # === tag: read version, create git tag (only on mainline) === - id: tag + name: Tag Release version: 1 - inputs: - workspace_dir: string outputs: version: string tag_created: bool tag_already_existed: bool steps: - name: read-version - type: deno + type: kubernetes outputs: - name: version config: - script: | - // Stub — remove the early return for real tagging - log("Reading version (stub)"); - output("version", "1.0.0"); - permissions: - read: ["((workspace_dir))"] + <<: *ci_config + run: | + set -euo pipefail + cd /workspace/wfe + VERSION=$(grep -m1 '^version' Cargo.toml | sed -E 's/.*"([^"]+)".*/\1/') + echo "Version: $VERSION" + echo "##wfe[output version=$VERSION]" - name: check-tag-exists - <<: *shell_defaults + type: kubernetes outputs: - name: tag_already_existed - - name: tag_created config: + <<: *ci_config run: | - echo "Checking tag (stub — remove exit 0 for real tagging)" - echo "##wfe[output tag_already_existed=true]" - echo "##wfe[output tag_created=false]" - exit 0 - VERSION=$(echo "$VERSION" | tr -d '[:space:]') + set -euo pipefail + cd /workspace/wfe + git fetch --tags TAG="v${VERSION}" if git tag -l "$TAG" | grep -q "$TAG"; then - echo "Tag $TAG already exists — skipping" + echo "Tag $TAG already exists" echo "##wfe[output tag_already_existed=true]" - echo "##wfe[output tag_created=false]" else - echo "Tag $TAG does not exist — will create" + echo "Tag $TAG does not exist" echo "##wfe[output tag_already_existed=false]" fi - name: create-tag - <<: *shell_defaults + type: kubernetes + when: + field: .outputs.tag_already_existed + equals: false outputs: - name: tag_created config: + <<: *ci_config run: | - echo "Creating tag (stub — remove exit 0 for real tagging)" - echo "##wfe[output tag_created=false]" - exit 0 - if [ "$TAG_ALREADY_EXISTED" = "true" ]; then - echo "Skipping tag creation (already exists)" - echo "##wfe[output tag_created=false]" - exit 0 - fi - VERSION=$(echo "$VERSION" | tr -d '[:space:]') + set -euo pipefail + cd /workspace/wfe TAG="v${VERSION}" + git config user.email "wfe-ci@sunbeam.pt" + git config user.name "wfe-ci" git tag -a "$TAG" -m "$TAG" + git push origin "$TAG" echo "##wfe[output tag_created=true]" - # ─── Workflow: publish ─────────────────────────────────────────── + # === publish: publish all crates to sunbeam registry === - id: publish + name: Publish Crates version: 1 - inputs: - workspace_dir: string - registry: string? outputs: all_published: bool steps: - - name: publish-protos - <<: *shell_defaults + - name: publish-tier-1 + type: kubernetes config: + <<: *ci_long_config run: | - echo "Publishing protos (stub — remove exit 0 for real publish)" - exit 0 - cd "$WORKSPACE_DIR" - REGISTRY="${REGISTRY:-sunbeam}" - PUBLISHED="" - for crate in wfe-buildkit-protos wfe-containerd-protos; do - echo "Publishing $crate..." - if cargo publish -p "$crate" --registry "$REGISTRY" 2>&1; then - PUBLISHED="$PUBLISHED $crate" - else - echo "Already published or failed: $crate (continuing)" - fi + set -euo pipefail + cd /workspace/wfe + for crate in wfe-core wfe-containerd-protos wfe-buildkit-protos wfe-server-protos; do + echo "--- Publishing $crate ---" + cargo publish -p "$crate" --registry sunbeam 2>&1 || echo "Already published: $crate" done - echo "##wfe[output published_protos=$PUBLISHED]" error_behavior: type: retry interval: 10s max_retries: 2 - - name: publish-core - <<: *shell_defaults + - name: publish-tier-2 + type: kubernetes config: + <<: *ci_long_config run: | - echo "Publishing core (stub — remove exit 0 for real publish)" - exit 0 - cd "$WORKSPACE_DIR" - REGISTRY="${REGISTRY:-sunbeam}" - cargo publish -p wfe-core --registry "$REGISTRY" 2>&1 || echo "Already published" - echo "##wfe[output core_published=true]" + set -euo pipefail + cd /workspace/wfe + for crate in wfe-sqlite wfe-postgres wfe-opensearch wfe-valkey \ + wfe-buildkit wfe-containerd wfe-rustlang wfe-kubernetes; do + echo "--- Publishing $crate ---" + cargo publish -p "$crate" --registry sunbeam 2>&1 || echo "Already published: $crate" + done error_behavior: type: retry interval: 10s max_retries: 2 - - name: publish-providers - <<: *shell_defaults + - name: publish-tier-3 + type: kubernetes config: + <<: *ci_long_config run: | - echo "Publishing providers (stub — remove exit 0 for real publish)" - exit 0 - cd "$WORKSPACE_DIR" - REGISTRY="${REGISTRY:-sunbeam}" - for crate in wfe-sqlite wfe-postgres wfe-opensearch wfe-valkey; do - echo "Publishing $crate..." - cargo publish -p "$crate" --registry "$REGISTRY" 2>&1 || echo "Already published: $crate" - done - echo "##wfe[output providers_published=true]" - error_behavior: - type: retry - interval: 10s - max_retries: 2 - - - name: publish-executors - <<: *shell_defaults - config: - run: | - echo "Publishing executors (stub — remove exit 0 for real publish)" - exit 0 - cd "$WORKSPACE_DIR" - REGISTRY="${REGISTRY:-sunbeam}" - for crate in wfe-buildkit wfe-containerd wfe-rustlang; do - echo "Publishing $crate..." - cargo publish -p "$crate" --registry "$REGISTRY" 2>&1 || echo "Already published: $crate" - done - echo "##wfe[output executors_published=true]" - - - name: publish-framework - <<: *shell_defaults - outputs: - - name: all_published - config: - run: | - echo "Publishing framework (stub — remove exit 0 for real publish)" - echo "##wfe[output all_published=true]" - exit 0 - cd "$WORKSPACE_DIR" - REGISTRY="${REGISTRY:-sunbeam}" + set -euo pipefail + cd /workspace/wfe + sleep 10 for crate in wfe wfe-yaml; do - echo "Publishing $crate..." - cargo publish -p "$crate" --registry "$REGISTRY" 2>&1 || echo "Already published: $crate" + echo "--- Publishing $crate ---" + cargo publish -p "$crate" --registry sunbeam 2>&1 || echo "Already published: $crate" done - echo "##wfe[output all_published=true]" + error_behavior: + type: retry + interval: 10s + max_retries: 2 - on_failure: - - name: log-partial-publish - <<: *shell_defaults + - name: publish-tier-4 + type: kubernetes outputs: - name: all_published config: + <<: *ci_long_config run: | - echo "WARNING: Publish partially failed. Check logs above." - echo "##wfe[output all_published=false]" + set -euo pipefail + cd /workspace/wfe + sleep 10 + for crate in wfe-server wfe-deno; do + echo "--- Publishing $crate ---" + cargo publish -p "$crate" --registry sunbeam 2>&1 || echo "Already published: $crate" + done + echo "##wfe[output all_published=true]" + error_behavior: + type: retry + interval: 10s + max_retries: 2 - # ─── Workflow: release ─────────────────────────────────────────── + # === image: build wfe-server Docker image via in-cluster buildkitd === + # + # Connects to buildkitd.build.svc:1234 over mTLS using certs mounted from + # the wfe-credentials secret. The wfe-buildkit step type handles all the + # client-side details. - - id: release + - id: image + name: Build Image version: 1 inputs: - workspace_dir: string version: string - git_remote: string? outputs: - pushed: bool - notes: string + image_pushed: bool + image_digest: string steps: - - name: push-tags - <<: *shell_defaults - outputs: - - name: pushed + - name: write-buildkit-certs + type: kubernetes config: + <<: *ci_config run: | - echo "Pushing tags (stub — remove exit 0 for real release)" - echo "##wfe[output pushed=true]" - exit 0 - REMOTE="${GIT_REMOTE:-origin}" - git push "$REMOTE" --tags - echo "##wfe[output pushed=true]" + set -euo pipefail + mkdir -p /workspace/buildkit-certs + echo "$BUILDKIT_CA_CERT" > /workspace/buildkit-certs/ca.pem + echo "$BUILDKIT_CLIENT_CERT" > /workspace/buildkit-certs/client.pem + echo "$BUILDKIT_CLIENT_KEY" > /workspace/buildkit-certs/client-key.pem + chmod 600 /workspace/buildkit-certs/client-key.pem - - name: generate-notes + - name: build-and-push + type: buildkit + outputs: + - name: image_pushed + - name: image_digest + config: + dockerfile: /workspace/wfe/Dockerfile + context: /workspace/wfe + tags: + - "src.sunbeam.pt/studio/wfe:${VERSION}" + - "src.sunbeam.pt/studio/wfe:latest" + push: true + buildkit_addr: tcp://buildkitd.build.svc:1234 + tls: + ca: /workspace/buildkit-certs/ca.pem + cert: /workspace/buildkit-certs/client.pem + key: /workspace/buildkit-certs/client-key.pem + timeout: 30m + + # === release: create Gitea release with changelog notes === + + - id: release + name: Gitea Release + version: 1 + inputs: + version: string + outputs: + release_created: bool + release_url: string + steps: + - name: extract-changelog type: deno outputs: - name: notes config: script: | - // Stub — remove the early return for real release - log("Generating release notes (stub)"); - output("notes", "stub release notes"); - permissions: - run: true + const data = inputs(); + const version = data.version; + const text = await readFile("/workspace/wfe/CHANGELOG.md"); - # ─── Workflow: ci (top-level orchestrator) ─────────────────────── + const lines = text.split("\n"); + const notes = []; + let inSection = false; + for (const line of lines) { + if (line.startsWith("## [")) { + if (inSection) break; + if (line.includes(`[${version}]`)) inSection = true; + continue; + } + if (inSection) notes.push(line); + } + const body = notes.join("\n").trim(); + log(`Extracted ${body.length} chars of changelog notes`); + output("notes", body); + permissions: + read: ["/workspace/wfe"] + + - name: create-gitea-release + type: kubernetes + outputs: + - name: release_created + - name: release_url + config: + <<: *ci_config + run: | + set -euo pipefail + cd /workspace/wfe + TAG="v${VERSION}" + + # tea login from env + tea login add --name sunbeam --url https://src.sunbeam.pt --token "$TEA_TOKEN" 2>/dev/null || true + + if tea release create --tag "$TAG" --title "$TAG" --note "$NOTES" 2>&1; then + echo "##wfe[output release_created=true]" + else + echo "Release may already exist" + echo "##wfe[output release_created=false]" + fi + echo "##wfe[output release_url=https://src.sunbeam.pt/studio/wfe/releases/tag/${TAG}]" + + # === ci: top-level orchestrator -- runs on every push === + # + # Triggered by Gitea webhook → /webhooks/gitea on wfe-server. + # The webhook handler maps push events to this workflow with: + # inputs: { repo_url, commit_sha, branch } - id: ci + name: Continuous Integration version: 1 inputs: - workspace_dir: string - registry: string? - git_remote: string? + repo_url: string + commit_sha: string + branch: string? coverage_threshold: number? outputs: version: string all_tests_passed: bool coverage: number published: bool + image_pushed: bool released: bool + steps: - - name: run-preflight + - name: run-checkout type: workflow outputs: - - name: cargo_ok - - name: nextest_ok - - name: llvm_cov_ok - - name: docker_ok - - name: lima_ok - - name: buildctl_ok - - name: git_ok + - name: commit config: - workflow: preflight + workflow: checkout version: 1 - name: run-lint @@ -748,16 +771,13 @@ workflows: workflow: cover version: 1 - - name: run-package - type: workflow - outputs: - - name: packages_ok - config: - workflow: package - version: 1 + # Below this line: only run on mainline (release branch). - name: run-tag type: workflow + when: + field: .inputs.branch + equals: mainline outputs: - name: version - name: tag_created @@ -767,16 +787,34 @@ workflows: - name: run-publish type: workflow + when: + field: .outputs.tag_created + equals: true outputs: - name: published config: workflow: publish version: 1 + - name: run-image + type: workflow + when: + field: .outputs.tag_created + equals: true + outputs: + - name: image_pushed + config: + workflow: image + version: 1 + - name: run-release type: workflow + when: + field: .outputs.tag_created + equals: true outputs: - name: released + - name: release_url config: workflow: release version: 1