From c5de46e3e1e8485537289b34e54a94c63e5a1281 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Mar 2025 04:23:24 +0000 Subject: [PATCH] Docker Bake Harness Co-authored-by: Nineko Signed-off-by: Jason Volk --- .dockerignore | 20 +- .github/workflows/bake.yml | 90 ++ .github/workflows/build.yml | 165 +++ Cargo.toml | 26 +- docker/Dockerfile.cargo | 37 + docker/Dockerfile.complement | 269 ++++ docker/Dockerfile.cookware | 63 + docker/Dockerfile.deps | 47 + docker/Dockerfile.diner | 53 + docker/Dockerfile.ingredients | 83 ++ docker/Dockerfile.install | 75 ++ docker/Dockerfile.kitchen | 30 + docker/Dockerfile.rocksdb | 77 ++ docker/Dockerfile.smoketest | 69 + docker/bake.hcl | 1188 +++++++++++++++++ docker/bake.sh | 118 ++ docker/run.sh | 31 + .../complement/test_results.jsonl | 4 +- 18 files changed, 2418 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/bake.yml create mode 100644 .github/workflows/build.yml create mode 100644 docker/Dockerfile.cargo create mode 100644 docker/Dockerfile.complement create mode 100644 docker/Dockerfile.cookware create mode 100644 docker/Dockerfile.deps create mode 100644 docker/Dockerfile.diner create mode 100644 docker/Dockerfile.ingredients create mode 100644 docker/Dockerfile.install create mode 100644 docker/Dockerfile.kitchen create mode 100644 docker/Dockerfile.rocksdb create mode 100644 docker/Dockerfile.smoketest create mode 100644 docker/bake.hcl create mode 100755 docker/bake.sh create mode 100755 docker/run.sh diff --git a/.dockerignore b/.dockerignore index 35d35e1b..dcdd5d15 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,27 +1,31 @@ # Local build and dev artifacts -target -tests +target/ +#tests # Docker files Dockerfile* +docker/ # IDE files .vscode .idea *.iml +. + # Git folder -.git -.gitea -.gitlab -.github +#.git +#.gitea +#.gitlab +#.github # Dot files .env -.gitignore +#.gitignore # Toml files -rustfmt.toml +#rustfmt.toml # Documentation #*.md +*.hcl diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml new file mode 100644 index 00000000..79f6d01f --- /dev/null +++ b/.github/workflows/bake.yml @@ -0,0 +1,90 @@ +name: Bakery + +on: + workflow_call: + inputs: + bake_targets: + type: string + required: false + default: '["default"]' + description: Bake targets + cargo_profiles: + type: string + required: false + default: '["test", "bench"]' + description: Cargo profiles + docker_id: + type: string + required: false + description: Dockerhub acct/repo identity. + feat_sets: + type: string + required: false + default: '["none", "default", "all"]' + description: Cargo feature groups + machines: + type: string + required: false + default: '["x86_64"]' + description: Hardware platform vector + rust_targets: + type: string + required: false + default: '["x86_64-unknown-linux-gnu"]' + description: Rust targets + rust_toolchains: + type: string + required: false + default: '["nightly", "stable"]' + description: Rust toolchains + sys_names: + type: string + required: false + default: '["debian"]' + description: System names + sys_targets: + type: string + required: false + default: '["x86_64-linux-gnu"]' + description: System targets + sys_versions: + type: string + required: false + default: '["testing-slim"]' + description: System versions + +env: + docker_id: ${{inputs.docker_id}} + +jobs: + task: + runs-on: ${{matrix.machine}} + strategy: + fail-fast: false + matrix: + bake_target: ${{fromJSON(inputs.bake_targets)}} + cargo_profile: ${{fromJSON(inputs.cargo_profiles)}} + feat_set: ${{fromJSON(inputs.feat_sets)}} + machine: ${{fromJSON(inputs.machines)}} + rust_target: ${{fromJSON(inputs.rust_targets)}} + rust_toolchain: ${{fromJSON(inputs.rust_toolchains)}} + sys_name: ${{fromJSON(inputs.sys_names)}} + sys_target: ${{fromJSON(inputs.sys_targets)}} + sys_version: ${{fromJSON(inputs.sys_versions)}} + + steps: + - uses: actions/checkout@v3 + - name: Task + env: + bake_target: ${{matrix.bake_target}} + cargo_profile: ${{matrix.cargo_profile}} + feat_set: ${{matrix.feat_set}} + machine: ${{matrix.machine}} + rust_target: ${{matrix.rust_target}} + rust_toolchain: ${{matrix.rust_toolchain}} + sys_name: ${{matrix.sys_name}} + sys_target: ${{matrix.sys_target}} + sys_version: ${{matrix.sys_version}} + + run: | + docker/bake.sh ${{matrix.bake_target}} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..36b42b9c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,165 @@ +name: Build + +on: + push: + branches: + - main + + workflow_dispatch: + inputs: + bake: + type: string + required: false + description: JSON Object of inputs passed to the environment + +concurrency: + group: ${{github.workflow}}-${{github.ref}} + cancel-in-progress: true + +env: + docker_id: ${{vars.DOCKER_ID}} + inputs: ${{github.event.inputs}} + +jobs: + systems: + name: Base Environment + uses: ./.github/workflows/bake.yml + with: + bake_targets: '["systems"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + buildsys: + name: Build Environment + uses: ./.github/workflows/bake.yml + needs: [systems] + with: + bake_targets: '["buildsys"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + sources: + name: Acquire Source + uses: ./.github/workflows/bake.yml + needs: [buildsys] + with: + bake_targets: '["sources"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + rocksdb: + name: Build RocksDB + uses: ./.github/workflows/bake.yml + needs: [sources] + with: + bake_targets: '["rocksdb"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + deps: + name: Build Dependencies + uses: ./.github/workflows/bake.yml + needs: [rocksdb] + with: + bake_targets: '["deps"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + clippy: + name: Clippy Lints + uses: ./.github/workflows/bake.yml + needs: [deps] + with: + bake_targets: '["clippy"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + install: + name: Install + uses: ./.github/workflows/bake.yml + needs: [deps] + with: + bake_targets: '["install"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + tests-unit: + name: Unit Tests + uses: ./.github/workflows/bake.yml + needs: [deps] + with: + bake_targets: '["tests-unit"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} + + smoketest: + name: Smoke Tests + uses: ./.github/workflows/bake.yml + needs: [install] + with: + bake_targets: '["tests-smoke"]' + cargo_profiles: ${{vars.CARGO_PROFILES}} + docker_id: ${{github.env.docker_id}} + feat_sets: ${{vars.FEAT_SETS}} + machines: ${{vars.MACHINES}} + rust_targets: ${{vars.RUST_TARGETS}} + rust_toolchains: ${{vars.RUST_TOOLCHAINS}} + sys_names: ${{vars.SYS_NAMES}} + sys_targets: ${{vars.SYS_TARGETS}} + sys_versions: ${{vars.SYS_VERSIONS}} diff --git a/Cargo.toml b/Cargo.toml index 36ac8e8f..15135c23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -668,7 +668,8 @@ lto = "fat" codegen-units = 1 panic = "abort" -# do not use without profile-rustflags enabled +# do not use without profile-rustflags enabled. uncomment ALL the sections for +# profile.release-max-perf [profile.release-max-perf] inherits = "release" strip = "symbols" @@ -710,7 +711,8 @@ inherits = "release-max-perf.build-override" #] [profile.bench] -inherits = "release" +debug = "limited" +strip = false #rustflags = [ # "-Cremark=all", # '-Ztime-passes', @@ -731,10 +733,6 @@ inherits = "release" [profile.dev] debug = "full" -opt-level = 0 -panic = "unwind" -debug-assertions = true -incremental = true #rustflags = [ # '--cfg', 'tuwunel_mods', # '-Ztime-passes', @@ -794,7 +792,7 @@ inherits = "dev" [profile.dev.package.'*'] inherits = "dev" -debug = 'limited' +debug = "limited" incremental = false codegen-units = 1 opt-level = 'z' @@ -813,19 +811,13 @@ opt-level = 'z' # primarily used for CI [profile.test] -inherits = "dev" -strip = false -opt-level = 0 -codegen-units = 16 -incremental = false +debug = "limited" [profile.test.package.'*'] -inherits = "dev" -debug = 0 -strip = false -opt-level = 0 -codegen-units = 16 +inherits = "test" incremental = false +codegen-units = 1 +opt-level = 'z' ############################################################################### # diff --git a/docker/Dockerfile.cargo b/docker/Dockerfile.cargo new file mode 100644 index 00000000..9ff7a325 --- /dev/null +++ b/docker/Dockerfile.cargo @@ -0,0 +1,37 @@ +# syntax = docker/dockerfile:1.11-labs + +FROM input AS cargo +ARG sys_target +ARG rust_toolchain +ARG RUSTUP_HOME +ARG CARGO_HOME +ARG CARGO_TARGET +ARG CARGO_TARGET_DIR +ARG cargo_profile +ARG cargo_features +ARG cargo_cmd +ARG cargo_args="" + +WORKDIR /usr/lib/${sys_target} +COPY --link --from=rocksdb . . + +WORKDIR /usr/src/tuwunel +RUN \ +--mount=type=cache,dst=${RUSTUP_HOME},sharing=locked \ +--mount=type=cache,dst=${CARGO_HOME},sharing=locked \ +--mount=type=cache,dst=${CARGO_TARGET_DIR},sharing=shared \ +< new_results.jsonl; + + wc -l old_results.jsonl new_results.jsonl; + + diff -w -y -t --width=275 --suppress-common-lines old_results.jsonl new_results.jsonl; +EOF +RUN echo $(tr -d '\n' < uwu.sh) > uwu.sh +ENTRYPOINT ["/bin/bash", "/usr/src/complement/uwu.sh"] + + +FROM input AS complement-tester-valgrind +ARG complement_debug=0 +ARG complement_count=1 +ARG complement_parallel=16 +ARG complement_shuffle=1337 +ARG complement_timeout="1h" +ARG complement_run=".*" +ARG complement_skip="" +ARG complement_tags="tuwunel_blacklist" +ARG complement_tests="./tests/..." +ARG complement_base_image + +ENV COMPLEMENT_DEBUG=$complement_debug +ENV complement_parallel="$complement_parallel" +ENV complement_shuffle="$complement_shuffle" +ENV complement_tags="$complement_tags" +ENV complement_timeout="$complement_timeout" +ENV complement_count="$complement_count" +ENV complement_tests="$complement_tests" +ENV complement_skip="$complement_skip" +ENV complement_run="$complement_run" +ENV complement_tests="$complement_tests" +ENV COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT="host.docker.internal" +ENV COMPLEMENT_HOST_MOUNTS="/var/run/docker.sock:/var/run/docker.sock" +WORKDIR /usr/src/complement +COPY < valgrind.sh +ENTRYPOINT ["/bin/bash", "/usr/src/complement/valgrind.sh"] diff --git a/docker/Dockerfile.cookware b/docker/Dockerfile.cookware new file mode 100644 index 00000000..0f0d1e24 --- /dev/null +++ b/docker/Dockerfile.cookware @@ -0,0 +1,63 @@ +# syntax = docker/dockerfile:1.11-labs + +FROM input AS cookware +ARG rust_toolchain +ARG RUSTUP_HOME +ARG CARGO_HOME +ARG CARGO_TARGET +ARG rustup_version="1.28.1" + +WORKDIR /opt +RUN \ +--mount=type=cache,dst=${RUSTUP_HOME},sharing=locked \ +--mount=type=cache,dst=${CARGO_HOME},sharing=locked \ +< /etc/apt/apt.conf.d/keep-downloaded + rm -f /etc/apt/apt.conf.d/docker-clean + + apt-get -y -U install --no-install-recommends ${packages} +EOF + + +FROM input AS valgrind +ARG var_cache +ARG var_lib_apt + +RUN \ +--mount=type=cache,dst=${var_cache},sharing=locked \ +--mount=type=cache,dst=${var_lib_apt},sharing=locked \ +< rocksdb.url + + sha1sum recipe.json rocksdb.url +EOF + + +FROM scratch AS recipe + +WORKDIR / +COPY --from=preparing /usr/src/tuwunel/recipe.json . +COPY --from=preparing /usr/src/tuwunel/rocksdb.url . diff --git a/docker/Dockerfile.install b/docker/Dockerfile.install new file mode 100644 index 00000000..b0eb5c96 --- /dev/null +++ b/docker/Dockerfile.install @@ -0,0 +1,75 @@ +# syntax = docker/dockerfile:1.11-labs + +FROM input AS installer +ARG sys_target +ARG rust_toolchain +ARG RUSTUP_HOME +ARG CARGO_HOME +ARG CARGO_TARGET +ARG CARGO_TARGET_DIR +ARG cargo_profile +ARG cargo_features +ARG cargo_args="" +ARG crate_path="src/main" +ARG crate_ident="" +ARG install_temp="/usr/src/tuwunel/install" + +WORKDIR /usr/lib/${sys_target} +COPY --link --from=rocksdb . . + +WORKDIR /usr/src/tuwunel +RUN \ +--mount=type=cache,dst=${RUSTUP_HOME},sharing=locked \ +--mount=type=cache,dst=${CARGO_HOME},sharing=locked \ +--mount=type=cache,dst=${CARGO_TARGET_DIR},sharing=shared \ +</dev/null +echo -e "\033[1;42;37mPASS\033[0m" diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 97c2e1b1..37907dd3 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -502,6 +502,7 @@ {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Deleting_a_non-existent_alias_should_return_a_404"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_in_the_default_room_configuration"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_can't_delete_other's_aliases"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other's_aliases"} {"Action":"fail","Test":"TestRoomForget"} {"Action":"fail","Test":"TestRoomForget/Parallel"} {"Action":"pass","Test":"TestRoomForget/Parallel/Can't_forget_room_you're_still_in"} @@ -606,10 +607,9 @@ {"Action":"fail","Test":"TestThreadedReceipts"} {"Action":"fail","Test":"TestThreadsEndpoint"} {"Action":"pass","Test":"TestToDeviceMessages"} -{"Action":"fail","Test":"TestToDeviceMessagesOverFederation"} +{"Action":"pass","Test":"TestToDeviceMessagesOverFederation"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/good_connectivity"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity"} -{"Action":"fail","Test":"TestToDeviceMessagesOverFederation/stopped_server"} {"Action":"fail","Test":"TestTxnIdWithRefreshToken"} {"Action":"fail","Test":"TestTxnIdempotency"} {"Action":"pass","Test":"TestTxnIdempotencyScopedToDevice"}