From df55b42d29eb57c8d62fcb2d4860a4c0085add83 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 10 May 2025 04:29:08 +0000 Subject: [PATCH] Add publish to pipeline. Update deployment docs. Signed-off-by: Jason Volk --- .github/workflows/bake.yml | 5 +- .github/workflows/deps.yml | 32 +++++++-- .github/workflows/lint.yml | 19 ------ .github/workflows/main.yml | 9 ++- .github/workflows/package.yml | 19 +++++- .github/workflows/publish.yml | 124 +++++++++++++++++++++++++--------- .github/workflows/test.yml | 23 ++++++- docker/Dockerfile.cargo | 1 + docker/Dockerfile.rocksdb | 7 +- docker/bake.hcl | 66 ++++++++++++------ docker/complement.sh | 18 ++++- docs/deploying/docker.md | 16 +---- 12 files changed, 234 insertions(+), 105 deletions(-) diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index 71c4296f..2ec1464f 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -72,7 +72,10 @@ jobs: machine: ${{fromJSON(inputs.machines)}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Bake env: bake_target: ${{matrix.bake_target}} diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 956a3aec..7c3a6769 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -51,10 +51,14 @@ on: type: boolean default: true description: Pre-build cargo dependency targets + show_docs: + type: boolean + default: true + description: Pre-build cargo rustdoc targets jobs: systems: - if: inputs.show_systems + if: ${{ !failure() && !cancelled() && inputs.show_systems }} name: System uses: ./.github/workflows/bake.yml with: @@ -71,7 +75,7 @@ jobs: includes: ${{inputs.includes}} buildsys: - if: inputs.show_systems + if: ${{ !failure() && !cancelled() && inputs.show_systems }} name: Builder needs: [systems] uses: ./.github/workflows/bake.yml @@ -89,7 +93,7 @@ jobs: includes: ${{inputs.includes}} sources: - if: inputs.show_sources + if: ${{ !failure() && !cancelled() && inputs.show_sources }} name: Acquire needs: [buildsys] uses: ./.github/workflows/bake.yml @@ -107,7 +111,7 @@ jobs: includes: ${{inputs.includes}} rocksdb: - if: inputs.show_rocksdb + if: ${{ !failure() && !cancelled() && inputs.show_rocksdb }} name: RocksDB needs: [sources] uses: ./.github/workflows/bake.yml @@ -125,7 +129,7 @@ jobs: includes: ${{inputs.includes}} deps: - if: inputs.show_cargo + if: ${{ !failure() && !cancelled() && inputs.show_cargo }} name: Build needs: [rocksdb] uses: ./.github/workflows/bake.yml @@ -141,3 +145,21 @@ jobs: machines: ${{inputs.machines}} excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} + + docs: + if: ${{ !failure() && !cancelled() && inputs.show_docs }} + name: Docs + needs: [deps] + uses: ./.github/workflows/bake.yml + with: + bake_targets: '["docs"]' + cargo_profiles: ${{inputs.cargo_profiles}} + feat_sets: ${{inputs.feat_sets}} + rust_toolchains: ${{inputs.rust_toolchains}} + sys_names: ${{inputs.sys_names}} + sys_versions: ${{inputs.sys_versions}} + rust_targets: ${{inputs.rust_targets}} + sys_targets: ${{inputs.sys_targets}} + machines: ${{inputs.machines}} + excludes: ${{inputs.excludes}} + includes: ${{inputs.includes}} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 41cca7f6..5b189ba3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -91,7 +91,6 @@ jobs: clippy: if: ${{ !failure() && !cancelled() }} name: Clippy - needs: [fmt, audit, lychee] uses: ./.github/workflows/bake.yml with: bake_targets: '["clippy"]' @@ -105,21 +104,3 @@ jobs: machines: ${{inputs.machines}} excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} - - doc: - if: ${{ !failure() && !cancelled() && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0]) }} - name: Docs - needs: [clippy] - uses: ./.github/workflows/bake.yml - with: - bake_targets: '["docs"]' - cargo_profiles: '["test"]' - feat_sets: ${{inputs.feat_sets}} - rust_toolchains: '["nightly"]' - sys_names: ${{inputs.sys_names}} - sys_versions: ${{inputs.sys_versions}} - rust_targets: ${{inputs.rust_targets}} - sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} - excludes: ${{inputs.excludes}} - includes: ${{inputs.includes}} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7fd6bf47..b8a6dc5d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -60,12 +60,12 @@ jobs: sys_targets: ${{vars.sys_TARGETS}} sys_versions: ${{vars.SYS_VERSIONS}} machines: ${{vars.MACHINES}} - complement: true + complement: ${{fromJSON(vars.COMPLEMENT || 'true')}} package: if: ${{ !failure() && !cancelled() }} name: Package - needs: [test] + needs: [lint] uses: ./.github/workflows/package.yml with: cargo_profiles: '["release"]' @@ -80,7 +80,7 @@ jobs: publish: if: ${{ !failure() && !cancelled() }} name: Publish - needs: [package] + needs: [test, package] uses: ./.github/workflows/publish.yml with: cargo_profiles: '["release"]' @@ -91,3 +91,6 @@ jobs: rust_targets: ${{vars.RUST_TARGETS}} sys_targets: ${{vars.sys_TARGETS}} machines: ${{vars.MACHINES}} + secrets: + dockerhub_token: ${{ secrets.dockerhub_token }} + ghcr_token: ${{ secrets.ghcr_token }} diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index b4e36e2e..253336d9 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -43,7 +43,7 @@ jobs: name: Deb uses: ./.github/workflows/bake.yml with: - bake_targets: '["pkg-deb-install"]' + bake_targets: '["pkg-deb"]' cargo_profiles: ${{inputs.cargo_profiles}} feat_sets: '["all"]' rust_toolchains: ${{inputs.rust_toolchains}} @@ -71,3 +71,20 @@ jobs: machines: ${{inputs.machines}} excludes: ${{inputs.excludes}} includes: ${{inputs.includes}} + + static: + if: ${{ !failure() && !cancelled() && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) }} + name: Standalone + uses: ./.github/workflows/bake.yml + with: + bake_targets: '["standalone"]' + cargo_profiles: ${{inputs.cargo_profiles}} + feat_sets: '["all"]' + rust_toolchains: ${{inputs.rust_toolchains}} + sys_names: ${{inputs.sys_names}} + sys_versions: ${{inputs.sys_versions}} + rust_targets: ${{inputs.rust_targets}} + sys_targets: ${{inputs.sys_targets}} + machines: ${{inputs.machines}} + excludes: ${{inputs.excludes}} + includes: ${{inputs.includes}} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4882f174..8d745302 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -36,38 +36,98 @@ on: type: string default: '[]' description: Matrix inclusions + secrets: + dockerhub_token: + ghcr_token: jobs: - dockerhub: - if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }} - name: DockerHub - uses: ./.github/workflows/bake.yml - with: - bake_targets: '["dockerhub"]' - cargo_profiles: ${{inputs.cargo_profiles}} - feat_sets: ${{inputs.feat_sets}} - rust_toolchains: ${{inputs.rust_toolchains}} - sys_names: ${{inputs.sys_names}} - sys_versions: ${{inputs.sys_versions}} - rust_targets: ${{inputs.rust_targets}} - sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} - excludes: ${{inputs.excludes}} - includes: ${{inputs.includes}} + containers: + if: ${{ !failure() && !cancelled() }} + name: Publish via Github + runs-on: ${{matrix.machine}} + permissions: write-all + strategy: + fail-fast: false + matrix: + bake_target: ${{fromJSON('["github", "dockerhub"]')}} + cargo_profile: ${{fromJSON(inputs.cargo_profiles)}} + rust_toolchain: ${{fromJSON(inputs.rust_toolchains)}} + feat_set: ${{fromJSON(inputs.feat_sets)}} + sys_name: ${{fromJSON(inputs.sys_names)}} + sys_version: ${{fromJSON(inputs.sys_versions)}} + rust_target: ${{fromJSON(inputs.rust_targets)}} + sys_target: ${{fromJSON(inputs.sys_targets)}} + machine: ${{fromJSON(inputs.machines)}} + exclude: ${{fromJSON(inputs.excludes)}} + include: ${{fromJSON(inputs.includes)}} - ghcr: - if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }} - name: GitHub Container Registry - uses: ./.github/workflows/bake.yml - with: - bake_targets: '["github"]' - cargo_profiles: ${{inputs.cargo_profiles}} - feat_sets: ${{inputs.feat_sets}} - rust_toolchains: ${{inputs.rust_toolchains}} - sys_names: ${{inputs.sys_names}} - sys_versions: ${{inputs.sys_versions}} - rust_targets: ${{inputs.rust_targets}} - sys_targets: ${{inputs.sys_targets}} - machines: ${{inputs.machines}} - excludes: ${{inputs.excludes}} - includes: ${{inputs.includes}} + steps: + - name: GitHub Login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.ghcr_token }} + + - uses: actions/checkout@v4 + - name: DockerHub Login + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ github.actor }} + password: ${{ secrets.dockerhub_token }} + + - name: Bake + env: + bake_target: ${{matrix.bake_target}} + cargo_profile: ${{matrix.cargo_profile}} + rust_toolchain: ${{matrix.rust_toolchain}} + rust_target: ${{matrix.rust_target}} + feat_set: ${{matrix.feat_set}} + sys_name: ${{matrix.sys_name}} + sys_version: ${{matrix.sys_version}} + sys_target: ${{matrix.sys_target}} + machine: ${{matrix.machine}} + acct: ${{github.actor}} + repo: ${{github.repository}} + CI_VERBOSE_ENV: ${{inputs.verbose_env}} + CI_SILENT_BAKE: ${{inputs.silent_bake}} + CI_PRINT_BAKE: ${{inputs.print_bake}} + + run: | + docker/bake.sh ${{matrix.bake_target}} + + packages: + if: ${{ !failure() && !cancelled() && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) }} + name: Publish packages + runs-on: ${{matrix.machine}} + strategy: + fail-fast: false + matrix: + target: ${{fromJSON('[["pkg-rpm","tuwunel-1.0.0-1.x86_64.rpm"],["pkg-deb","tuwunel_1.0.0-1_amd64.deb"],["standalone", "tuwunel"]]')}} + cargo_profile: ${{fromJSON(inputs.cargo_profiles)}} + rust_toolchain: ${{fromJSON(inputs.rust_toolchains)}} + feat_set: ${{fromJSON('["all"]')}} + sys_name: ${{fromJSON(inputs.sys_names)}} + sys_version: ${{fromJSON(inputs.sys_versions)}} + rust_target: ${{fromJSON(inputs.rust_targets)}} + sys_target: ${{fromJSON(inputs.sys_targets)}} + machine: ${{fromJSON(inputs.machines)}} + exclude: ${{fromJSON(inputs.excludes)}} + include: ${{fromJSON(inputs.includes)}} + + steps: + - name: Extract + env: + iid: ${{matrix.target[0]}}--${{matrix.cargo_profile}}--${{matrix.rust_toolchain}}--${{matrix.rust_target}}--${{matrix.feat_set}}--${{matrix.sys_name}}--${{matrix.sys_version}}--${{matrix.sys_target}} + file: ${{matrix.target[1]}} + + run: | + cid=$(docker create "$iid" /) + docker cp "$cid:$file" . + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{matrix.target[1]}} + path: ${{matrix.target[1]}} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad73b7a8..b079cef2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,7 +94,6 @@ jobs: if: ${{inputs.complement && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0]) && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0]) && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])}} name: Complement Testee uses: ./.github/workflows/bake.yml - needs: [smoke] with: bake_targets: '["complement-testee"]' cargo_profiles: '["test"]' @@ -132,7 +131,9 @@ jobs: include: ${{fromJSON(inputs.includes)}} steps: - - name: Complement + - uses: actions/checkout@v3 + - name: Execute + id: execute env: cargo_profile: ${{matrix.cargo_profile}} rust_toolchain: ${{matrix.rust_toolchain}} @@ -145,3 +146,21 @@ jobs: run: | docker/complement.sh + + - name: Results + id: extract + env: + name: complement_tester__${{matrix.cargo_profile}}__${{matrix.rust_toolchain}}__${{matrix.rust_target}}__${{matrix.feat_set}}__${{matrix.sys_name}}__${{matrix.sys_version}}__${{matrix.sys_target}} + tag: latest + + run: | + cid=$(cat "$name") + docker cp "$cid:/usr/src/complement/new_results.jsonl" tests/test_results/complement/test_results.jsonl + git diff --exit-code --color --color-moved + + - name: Artifacts + id: upload + uses: actions/upload-artifact@v4 + with: + name: test_results.${{matrix.feat_set}}.${{matrix.sys_name}}.${{matrix.sys_target}}.jsonl + path: ./tests/test_results/complement/test_results.jsonl diff --git a/docker/Dockerfile.cargo b/docker/Dockerfile.cargo index 1cca766e..1a42e7f1 100644 --- a/docker/Dockerfile.cargo +++ b/docker/Dockerfile.cargo @@ -33,5 +33,6 @@ RUN \ --target "${CARGO_TARGET}" \ --target-dir "${CARGO_TARGET_DIR}" \ --manifest-path Cargo.toml \ + --color always \ ${cargo_args} EOF diff --git a/docker/Dockerfile.rocksdb b/docker/Dockerfile.rocksdb index 9aa3cb12..b213b597 100644 --- a/docker/Dockerfile.rocksdb +++ b/docker/Dockerfile.rocksdb @@ -48,11 +48,14 @@ RUN \ "-DCMAKE_BUILD_TYPE=${rocksdb_build_type}" \ "-DBUILD_SHARED_LIBS=${rocksdb_shared}" \ "-DCMAKE_CXX_FLAGS:STRING=${rocksdb_cxx_flags}" \ - "-DCMAKE_CXX_FLAGS_RELEASE:STRING=-g0 -O${rocksdb_opt_level}" \ + "-DCMAKE_CXX_FLAGS_RELEASE:STRING=-g0 -O${rocksdb_opt_level} -DNDEBUG" \ "-DPORTABLE=${rocksdb_portable}" \ "-DFAIL_ON_WARNINGS=0" \ - "-DUSE_RTTI=1" \ + "-DUSE_RTTI=0" \ + "-DWITH_RUNTIME_DEBUG=0" \ "-DWITH_JNI=0" \ + "-DWITH_EXAMPLES=0" \ + "-DWITH_BENCHMARK=0" \ "-DWITH_BENCHMARK_TOOLS=0" \ "-DWITH_TRACE_TOOLS=0" \ "-DWITH_CORE_TOOLS=0" \ diff --git a/docker/bake.hcl b/docker/bake.hcl index 1a80b3bb..124e29a9 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -194,10 +194,8 @@ group "publish" { target "github" { name = elem("github", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]) tags = [ - "ghcr.io/matrix-construct/tuwunel:${cargo_profile}-${feat_set}-${sys_target}", - (GITHUB_REF_NAME == "main" && cargo_profile == "release" && feat_set == "all")? - "ghcr.io/matrix-construct/tuwunel:main": "", - (GITHUB_REF_NAME == "main" && cargo_profile == "release" && feat_set == "all")? + "ghcr.io/matrix-construct/tuwunel:${GITHUB_REF_NAME}-${cargo_profile}-${feat_set}-${sys_target}", + (cargo_profile == "release" && feat_set == "all")? "ghcr.io/matrix-construct/tuwunel:${GITHUB_REF_NAME}": "", (GITHUB_REF_NAME == "main" && cargo_profile == "release" && feat_set == "all")? "ghcr.io/matrix-construct/tuwunel:latest": "", @@ -205,32 +203,33 @@ target "github" { output = ["type=registry,compression=zstd,mode=min"] matrix = cargo_rust_feat_sys inherits = [ - elem("install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]), + elem("tuwunel", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]), ] - contexts = { - input = elem("target:install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]) - } - target = "tuwunel" - dockerfile-inline =</dev/null + +trap 'set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' ERR + set -x -e cid=$(docker run $arg) set +x +if test "$CI" = "true"; then + echo -n "$cid" > "$name" +fi + trap 'docker container stop $cid; set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' INT docker logs -f "$cid" diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index d1760751..520bea06 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -12,29 +12,15 @@ OCI images for tuwunel are available in the registries listed below. | Registry | Image | Size | Notes | | --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- | | GitHub Registry | [ghcr.io/matrix-construct/tuwunel:latest][gh] | ![Image Size][shield-latest] | Stable latest tagged image. | -| GitLab Registry | [registry.gitlab.com/jevolk/tuwunel:latest][gl] | ![Image Size][shield-latest] | Stable latest tagged image. | | Docker Hub | [docker.io/jevolk/tuwunel:latest][dh] | ![Image Size][shield-latest] | Stable latest tagged image. | | GitHub Registry | [ghcr.io/matrix-construct/tuwunel:main][gh] | ![Image Size][shield-main] | Stable main branch. | -| GitLab Registry | [registry.gitlab.com/jevolk/tuwunel:main][gl] | ![Image Size][shield-main] | Stable main branch. | | Docker Hub | [docker.io/jevolk/tuwunel:main][dh] | ![Image Size][shield-main] | Stable main branch. | [dh]: https://hub.docker.com/r/jevolk/tuwunel -[gh]: https://github.com/jevolk/tuwunel/pkgs/container/tuwunel -[gl]: https://gitlab.com/tuwunel/tuwunel/container_registry/6369729 +[gh]: https://github.com/matrix-construct/tuwunel/pkgs/container/tuwunel [shield-latest]: https://img.shields.io/docker/image-size/jevolk/tuwunel/latest [shield-main]: https://img.shields.io/docker/image-size/jevolk/tuwunel/main -OCI image `.tar.gz` files are also hosted directly at when uploaded by CI with a -commit hash/revision or a tagged release: - -Use - -```bash -docker image pull $LINK -``` - -to pull it to your machine. - ### Run When you have the image you can simply run it with