From 83ceda808cbf8f76db020145292da747d62663d2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 16 Jun 2025 03:19:34 +0000 Subject: [PATCH] ci: Add multi-runner/multi-arch to matrices. (closes #55) Signed-off-by: Jason Volk --- .github/workflows/bake.yml | 6 ++- .github/workflows/lint.yml | 15 ++++-- .github/workflows/main.yml | 73 +++++++++++++++++++++++------ .github/workflows/package.yml | 10 ++-- .github/workflows/publish.yml | 4 +- .github/workflows/test.yml | 40 ++++++++++++---- docker/Dockerfile.rocksdb | 2 +- docker/Dockerfile.rust | 5 +- docker/bake.hcl | 87 ++++++++++++++++++++++------------- 9 files changed, 172 insertions(+), 70 deletions(-) diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index a08da907..08f08c92 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -30,6 +30,10 @@ on: machines: type: string description: Hardware platforms + runner: + type: string + description: Runner mask + default: 'self-hosted' excludes: type: string default: '[]' @@ -86,7 +90,7 @@ jobs: ${{matrix.sys_version}} ${{matrix.sys_target}} - runs-on: ${{matrix.machine}} + runs-on: ["${{matrix.machine}}", "${{inputs.runner}}"] strategy: fail-fast: ${{inputs.fail_fast}} max-parallel: ${{inputs.max-parallel}} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b39c9392..f2c52ad0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,6 +43,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Format uses: ./.github/workflows/bake.yml @@ -55,7 +56,7 @@ jobs: sys_names: ${{inputs.sys_names}} sys_targets: ${{inputs.sys_targets}} sys_versions: ${{inputs.sys_versions}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} @@ -65,6 +66,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Typos uses: ./.github/workflows/bake.yml @@ -77,7 +79,7 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} @@ -87,6 +89,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Audit uses: ./.github/workflows/bake.yml @@ -99,7 +102,7 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} @@ -109,6 +112,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Lychee uses: ./.github/workflows/bake.yml @@ -121,7 +125,7 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} @@ -131,6 +135,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Check uses: ./.github/workflows/bake.yml @@ -143,7 +148,7 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b99a2bdb..289b6449 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ concurrency: jobs: init: name: Init - runs-on: self-hosted + runs-on: ${{matrix.runner}} env: default_cargo_profiles: '["test", "release"]' default_feat_sets: '["none", "default", "all"]' @@ -27,7 +27,7 @@ jobs: default_sys_versions: '["testing-slim"]' default_rust_targets: '["x86_64-unknown-linux-gnu"]' default_sys_targets: '["x86_64-linux-gnu"]' - default_machines: '["x86_64"]' + default_machines: '["X64"]' outputs: cargo_profiles: ${{vars.CARGO_PROFILES || env.default_cargo_profiles}} @@ -40,8 +40,19 @@ jobs: machines: ${{vars.MACHINES || env.default_machines}} release_url: ${{steps.release.outputs.upload_url}} + strategy: + fail-fast: true + matrix: + runner: ${{fromJSON(vars.RUNNERS)}} + machine: ${{fromJSON(vars.MACHINES)}} + exclude: ${{fromJSON(vars.RUNNERS_EXCLUDES)}} + steps: - name: Initialize Builder + env: + runner: ${{matrix.runner}} + reserved_space: '{"het": "256GB", "aws": "48GB", "gcp": "160GB"}' + max_used_space: '{"het": "448GB", "aws": "64GB", "gcp": "192GB"}' run: | set +e docker buildx inspect "${GITHUB_ACTOR}" @@ -49,7 +60,9 @@ jobs: exit 0 fi - set -e + set -eux + reserved_space=$(echo -n "$reserved_space" | jq -r ".$runner") + max_used_space=$(echo -n "$max_used_space" | jq -r ".$runner") cat < ./buildkitd.toml [system] platformsCacheMaxAge = "504h" @@ -57,11 +70,11 @@ jobs: enabled = true rootless = false gc = true - reservedSpace = "256GB" - maxUsedSpace = "576GB" + reservedSpace = "${reserved_space}" + maxUsedSpace = "${max_used_space}" [[worker.oci.gcpolicy]] - reservedSpace = "256GB" - maxUsedSpace = "576GB" + reservedSpace = "${reserved_space}" + maxUsedSpace = "${max_used_space}" filters = ["label!=cache==pin"] all = true EOF @@ -76,6 +89,7 @@ jobs: - if: > !failure() && !cancelled() && contains(github.ref, 'refs/tags/v') + && matrix.runner == 'het' id: release name: Create Release @@ -108,13 +122,22 @@ jobs: machines: ${{needs.init.outputs.machines}} excludes: > [ + {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"}, + {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"}, + {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "none"}, + {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "default"}, {"cargo_profile": "release-debuginfo"}, {"cargo_profile": "release-native", "rust_toolchain": "stable"}, {"cargo_profile": "release-native", "feat_set": "none"}, {"cargo_profile": "release-native", "feat_set": "default"}, - {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"}, - {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "none"}, - {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "default"} + {"cargo_profile": "release-native", "rust_target": "aarch64-unknown-linux-gnu"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "none"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"}, + {"rust_target": "aarch64-unknown-linux-gnu", "rust_toolchain": "nightly"}, + {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-linux-gnu"}, + {"rust_target": "x86_64-unknown-linux-gnu", "sys_target": "aarch64-linux-gnu"}, + {"sys_target": "x86_64-linux-gnu", "machine": "ARM64"}, + {"sys_target": "aarch64-linux-gnu", "machine": "X64"}, ] test: @@ -135,13 +158,23 @@ jobs: sys_targets: ${{needs.init.outputs.sys_targets}} sys_versions: ${{needs.init.outputs.sys_versions}} machines: ${{needs.init.outputs.machines}} + complement_runner: 'het' excludes: > [ {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"}, + {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"}, + {"cargo_profile": "release-debuginfo"}, {"cargo_profile": "release-native", "rust_toolchain": "stable"}, {"cargo_profile": "release-native", "feat_set": "default"}, {"cargo_profile": "release-native", "feat_set": "none"}, - {"cargo_profile": "release-debuginfo"} + {"cargo_profile": "release-native", "rust_target": "aarch64-unknown-linux-gnu"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "none"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"}, + {"rust_target": "aarch64-unknown-linux-gnu", "rust_toolchain": "nightly"}, + {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-linux-gnu"}, + {"rust_target": "x86_64-unknown-linux-gnu", "sys_target": "aarch64-linux-gnu"}, + {"sys_target": "x86_64-linux-gnu", "machine": "ARM64"}, + {"sys_target": "aarch64-linux-gnu", "machine": "X64"}, ] package: @@ -168,12 +201,19 @@ jobs: {"cargo_profile": "release-debuginfo", "feat_set": "default"}, {"cargo_profile": "release-debuginfo", "rust_toolchain": "nightly"}, {"cargo_profile": "release-debuginfo", "bake_target": "oci"}, + {"cargo_profile": "release-debuginfo", "rust_target": "aarch64-unknown-linux-gnu"}, {"cargo_profile": "release-native", "feat_set": "default"}, {"cargo_profile": "release-native", "rust_toolchain": "stable"}, {"cargo_profile": "release-native", "feat_set": "default"}, {"cargo_profile": "release-native", "bake_target": "docker"}, {"cargo_profile": "release-native", "bake_target": "oci"}, - {"cargo_profile": "release", "rust_toolchain": "nightly"} + {"cargo_profile": "release-native", "rust_target": "aarch64-unknown-linux-gnu"}, + {"cargo_profile": "release", "rust_toolchain": "nightly"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"}, + {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-linux-gnu"}, + {"rust_target": "x86_64-unknown-linux-gnu", "sys_target": "aarch64-linux-gnu"}, + {"sys_target": "x86_64-linux-gnu", "machine": "ARM64"}, + {"sys_target": "aarch64-linux-gnu", "machine": "X64"}, ] publish: @@ -198,9 +238,14 @@ jobs: [ {"feat_set": "none"}, {"cargo_profile": "test"}, - {"rust_toolchain": "nightly", "cargo_profile": "release"}, {"cargo_profile": "release-debuginfo"}, - {"cargo_profile": "release-native"} + {"cargo_profile": "release-native"}, + {"cargo_profile": "release", "rust_toolchain": "nightly"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"}, + {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-linux-gnu"}, + {"rust_target": "x86_64-unknown-linux-gnu", "sys_target": "aarch64-linux-gnu"}, + {"sys_target": "x86_64-linux-gnu", "machine": "ARM64"}, + {"sys_target": "aarch64-linux-gnu", "machine": "X64"}, ] secrets: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index f97b409d..d782ef7b 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -46,6 +46,7 @@ jobs: && contains(fromJSON(inputs.cargo_profiles), fromJSON('["release"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["default"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["stable"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Book uses: ./.github/workflows/bake.yml @@ -58,7 +59,8 @@ jobs: sys_names: ${{inputs.sys_names}} sys_targets: ${{inputs.sys_targets}} sys_versions: ${{inputs.sys_versions}} - machines: ${{inputs.machines}} + machines: '["X64"]' + runner: 'het' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} release_url: ${{inputs.release_url}} @@ -142,6 +144,7 @@ jobs: && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.cargo_profiles), fromJSON('["release"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["stable"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) && github.ref == 'refs/heads/main' name: Distro Packages @@ -155,7 +158,7 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} release_url: ${{inputs.release_url}} @@ -179,6 +182,7 @@ jobs: && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.cargo_profiles), fromJSON('["release"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["stable"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) && github.ref == 'refs/heads/main' name: Check Packages @@ -193,6 +197,6 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a3d0c6c8..089942ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,12 +49,12 @@ jobs: documents: if: > !failure() && !cancelled() - && inputs.machines + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) && (github.ref == 'refs/heads/main' || contains(github.ref, 'refs/tags/v')) && !contains(github.ref, '-draft') name: Documents - runs-on: self-hosted + runs-on: ['X64', 'het'] permissions: pages: write id-token: write diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4471dc8b..a76da669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,8 @@ on: complement_feat_sets: type: string default: '["all"]' + complement_runner: + type: string jobs: unit: @@ -48,6 +50,7 @@ jobs: contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Unit uses: ./.github/workflows/bake.yml @@ -60,12 +63,16 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} smoke: - if: inputs.cargo_profiles + if: > + !failure() && !cancelled() + && inputs.cargo_profiles + && inputs.machines + name: Smoke uses: ./.github/workflows/bake.yml with: @@ -82,26 +89,37 @@ jobs: excludes: > [ {"cargo_profile": "test", "feat_set": "default"}, - {"cargo_profile": "test", "rust_toolchain": "stable"}, {"cargo_profile": "test", "feat_set": "none", "bake_target": "smoke-valgrind"}, + {"cargo_profile": "test", "rust_toolchain": "stable"}, + {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"}, {"cargo_profile": "release", "rust_toolchain": "nightly"}, {"cargo_profile": "release", "rust_toolchain": "stable", "feat_set": "none"}, {"cargo_profile": "release", "bake_target": "smoke-valgrind"}, {"cargo_profile": "release-debuginfo", "rust_toolchain": "nightly"}, {"cargo_profile": "release-debuginfo", "rust_toolchain": "stable", "feat_set": "none"}, {"cargo_profile": "release-debuginfo", "bake_target": "smoke"}, + {"cargo_profile": "release-debuginfo", "rust_target": "aarch64-unknown-linux-gnu"}, {"cargo_profile": "release-native", "rust_toolchain": "stable"}, {"cargo_profile": "release-native", "feat_set": "none"}, {"cargo_profile": "release-native", "feat_set": "default"}, - {"cargo_profile": "release-native", "bake_target": "smoke-valgrind"} + {"cargo_profile": "release-native", "bake_target": "smoke-valgrind"}, + {"cargo_profile": "release-native", "rust_target": "aarch64-unknown-linux-gnu"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "none"}, + {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"}, + {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-linux-gnu"}, + {"rust_target": "x86_64-unknown-linux-gnu", "sys_target": "aarch64-linux-gnu"}, + {"sys_target": "x86_64-linux-gnu", "machine": "ARM64"}, + {"sys_target": "aarch64-linux-gnu", "machine": "X64"}, ] complement: if: > - inputs.complement + !failure() && !cancelled() + && inputs.complement && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.complement_feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Complement uses: ./.github/workflows/bake.yml @@ -114,20 +132,24 @@ jobs: sys_versions: ${{inputs.sys_versions}} rust_targets: ${{inputs.rust_targets}} sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} + machines: '["X64"]' + runner: ${{inputs.complement_runner}} excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} compliance: if: > - inputs.complement + !failure() && !cancelled() + && inputs.complement + && inputs.machines && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) + && contains(fromJSON(inputs.machines), fromJSON('["X64"]')[0]) name: Matrix Compliance needs: [complement] - runs-on: ${{matrix.machine}} + runs-on: ["${{matrix.machine}}", "${{inputs.complement_runner}}"] concurrency: group: complement-cant-walk-and-chew-bubblegum cancel-in-progress: false @@ -142,7 +164,7 @@ jobs: sys_version: ${{fromJSON(inputs.sys_versions)}} rust_target: ${{fromJSON(inputs.rust_targets)}} sys_target: ${{fromJSON(inputs.sys_targets)}} - machine: ${{fromJSON(inputs.machines)}} + machine: ${{fromJSON('["X64"]')}} exclude: ${{fromJSON(inputs.excludes)}} include: ${{fromJSON(inputs.includes)}} diff --git a/docker/Dockerfile.rocksdb b/docker/Dockerfile.rocksdb index 9546aa8a..28452c47 100644 --- a/docker/Dockerfile.rocksdb +++ b/docker/Dockerfile.rocksdb @@ -29,7 +29,7 @@ ARG rocksdb_shared=0 ARG rocksdb_portable=1 ARG rocksdb_opt_level="3" ARG rocksdb_build_type="Release" -ARG rocksdb_cxx_flags="-ftls-model=initial-exec" +ARG rocksdb_cxx_flags="-ftls-model=initial-exec -mno-outline-atomics" ARG rocksdb_make_verbose="ON" ARG rocksdb_make_rule_messages="OFF" ARG rocksdb_jemalloc=1 diff --git a/docker/Dockerfile.rust b/docker/Dockerfile.rust index d98dc38b..f69a4717 100644 --- a/docker/Dockerfile.rust +++ b/docker/Dockerfile.rust @@ -3,7 +3,7 @@ FROM input AS rustup ARG RUST_HOME ARG rust_target -ARG rustup_version="1.28.1" +ARG rustup_version="1.28.2" ARG rustup_profile="minimal" WORKDIR / @@ -13,7 +13,8 @@ WORKDIR ${RUST_HOME} RUN <