Combine Base and Deps workflows.

Add workflow arguments for verbosity.

Add missing ldap to full features.

Superpose --all-features as backstop for unlisted features.

Fix hardened_malloc requiring gcc

Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk
2025-05-03 02:32:49 +00:00
parent 09680fc86b
commit 481b67f1f9
15 changed files with 147 additions and 190 deletions

View File

@@ -5,49 +5,52 @@ on:
inputs:
bake_targets:
type: string
required: false
default: '["default"]'
description: Bake targets
cargo_profiles:
type: string
required: false
default: '["test", "bench"]'
default: ${{vars.CARGO_PROFILES || '["test", "bench", "release"]'}}
description: Cargo profiles
feat_sets:
type: string
required: false
default: '["none", "default", "all"]'
default: ${{vars.FEAT_SETS || '["none", "default", "all"]'}}
description: Cargo feature groups
machines:
type: string
required: false
default: '["x86_64"]'
default: ${{vars.MACHINES || '["x86_64"]'}}
description: Hardware platform vector
rust_targets:
type: string
required: false
default: '["x86_64-unknown-linux-gnu"]'
default: ${{vars.RUST_TARGETS || '["x86_64-unknown-linux-gnu"]'}}
description: Rust targets
rust_toolchains:
type: string
required: false
default: '["nightly", "stable"]'
default: ${{vars.RUST_TOOLCHAINS || '["nightly", "stable"]'}}
description: Rust toolchains
sys_names:
type: string
required: false
default: '["debian"]'
default: ${{vars.SYS_NAMES || '["debian"]'}}
description: System names
sys_targets:
type: string
required: false
default: '["x86_64-linux-gnu"]'
default: ${{vars.SYS_TARGETS || '["x86_64-linux-gnu"]'}}
description: System targets
sys_versions:
type: string
required: false
default: '["testing-slim"]'
default: ${{vars.SYS_VERSIONS || '["testing-slim"]'}}
description: System versions
verbose_env:
type: string
default: ${{vars.CI_VERBOSE_ENV || false}}
description: Print the bake environment.
silent_bake:
type: string
default: ${{vars.CI_SILENT_BAKE || false}}
description: Print as little as possible.
print_bake:
type: string
default: ${{vars.CI_PRINT_BAKE || false}}
description: Print the computed receipe.
jobs:
bake:
@@ -79,6 +82,9 @@ jobs:
sys_name: ${{matrix.sys_name}}
sys_target: ${{matrix.sys_target}}
sys_version: ${{matrix.sys_version}}
CI_VERBOSE_ENV: ${{inputs.verbose_env}}
CI_SILENT_BAKE: ${{inputs.silent_bake}}
CI_PRINT_BAKE: ${{inputs.print_bake}}
run: |
docker/bake.sh ${{matrix.bake_target}}

View File

@@ -1,49 +0,0 @@
name: Base
on:
workflow_call:
jobs:
systems:
name: System
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["systems"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
buildsys:
name: Builder
uses: ./.github/workflows/bake.yml
needs: [systems]
with:
bake_targets: '["buildsys"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
complement:
name: Tester
uses: ./.github/workflows/bake.yml
needs: [systems]
with:
bake_targets: '["complement-tester"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}

View File

@@ -9,14 +9,6 @@ jobs:
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["install"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
complement:
name: Complement Testee
@@ -24,11 +16,3 @@ jobs:
needs: [install]
with:
bake_targets: '["complement-testee"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}

View File

@@ -1,49 +1,46 @@
name: Dependency Build
name: Dependencies
on:
workflow_call:
jobs:
systems:
name: System
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["systems"]'
buildsys:
name: Builder
needs: [systems]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["buildsys"]'
tester:
name: Tester
needs: [systems]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["complement-tester"]'
sources:
name: Acquire
needs: [buildsys]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["sources"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
rocksdb:
name: RocksDB
uses: ./.github/workflows/bake.yml
needs: [sources]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["rocksdb"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
deps:
name: Build
uses: ./.github/workflows/bake.yml
needs: [rocksdb]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["deps-clippy", "deps-build-tests", "deps-build-bins"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}

View File

@@ -11,12 +11,7 @@ jobs:
bake_targets: '["fmt"]'
cargo_profiles: '["test"]'
feat_sets: '["none"]'
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: '["nightly"]'
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
audit:
name: Audit
@@ -25,12 +20,7 @@ jobs:
bake_targets: '["audit"]'
cargo_profiles: '["test"]'
feat_sets: '["none"]'
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: '["nightly"]'
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
lychee:
name: Lychee
@@ -39,26 +29,13 @@ jobs:
bake_targets: '["lychee"]'
cargo_profiles: '["test"]'
feat_sets: '["none"]'
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: '["nightly"]'
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
clippy:
name: Clippy
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["clippy"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
doc:
name: Docs
@@ -67,10 +44,4 @@ jobs:
with:
bake_targets: '["docs"]'
cargo_profiles: '["test"]'
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: '["nightly"]'
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}

View File

@@ -17,14 +17,9 @@ env:
inputs: ${{github.event.inputs}}
jobs:
base:
name: Base
uses: ./.github/workflows/base.yml
deps:
name: Deps
uses: ./.github/workflows/deps.yml
needs: [base]
lint:
name: Lint

View File

@@ -1,7 +1,40 @@
name: Test
name: Testing
on:
workflow_call:
inputs:
cargo_profiles:
type: string
default: ${{vars.CARGO_PROFILES || '["test", "bench", "release"]'}}
description: Cargo profiles
feat_sets:
type: string
default: ${{vars.FEAT_SETS || '["none", "default", "all"]'}}
description: Cargo feature groups
machines:
type: string
default: ${{vars.MACHINES || '["x86_64"]'}}
description: Hardware platform vector
rust_targets:
type: string
default: ${{vars.RUST_TARGETS || '["x86_64-unknown-linux-gnu"]'}}
description: Rust targets
rust_toolchains:
type: string
default: ${{vars.RUST_TOOLCHAINS || '["nightly", "stable"]'}}
description: Rust toolchains
sys_names:
type: string
default: ${{vars.SYS_NAMES || '["debian"]'}}
description: System names
sys_targets:
type: string
default: ${{vars.SYS_TARGETS || '["x86_64-linux-gnu"]'}}
description: System targets
sys_versions:
type: string
default: ${{vars.SYS_VERSIONS || '["testing-slim"]'}}
description: System versions
jobs:
unit:
@@ -9,28 +42,15 @@ jobs:
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["tests-unit"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
cargo_profiles: '["test", "bench"]'
feat_sets: '["all"]'
rust_toolchains: '["nightly"]'
smoke:
name: Smoke
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["tests-smoke"]'
cargo_profiles: ${{vars.CARGO_PROFILES}}
feat_sets: ${{vars.FEAT_SETS}}
machines: ${{vars.MACHINES}}
rust_targets: ${{vars.RUST_TARGETS}}
rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
sys_names: ${{vars.SYS_NAMES}}
sys_targets: ${{vars.SYS_TARGETS}}
sys_versions: ${{vars.SYS_VERSIONS}}
complement:
name: Compliance
@@ -43,12 +63,12 @@ jobs:
matrix:
cargo_profile: ${{fromJSON('["test"]')}}
feat_set: ${{fromJSON('["all"]')}}
machine: ${{fromJSON(vars.MACHINES)}}
rust_target: ${{fromJSON(vars.RUST_TARGETS)}}
machine: ${{fromJSON(inputs.machines)}}
rust_target: ${{fromJSON(inputs.rust_targets)}}
rust_toolchain: ${{fromJSON('["nightly"]')}}
sys_name: ${{fromJSON(vars.SYS_NAMES)}}
sys_target: ${{fromJSON(vars.SYS_TARGETS)}}
sys_version: ${{fromJSON(vars.SYS_VERSIONS)}}
sys_name: ${{fromJSON(inputs.sys_names)}}
sys_target: ${{fromJSON(inputs.sys_targets)}}
sys_version: ${{fromJSON(inputs.sys_versions)}}
steps:
- name: Complement

View File

@@ -9,6 +9,7 @@ ARG CARGO_TARGET
ARG CARGO_TARGET_DIR
ARG cargo_profile
ARG cargo_features
ARG cargo_spec_features
ARG cargo_cmd
ARG cargo_args=""
@@ -27,7 +28,7 @@ RUN \
cargo ${cargo_cmd} \
--frozen \
--workspace \
--no-default-features \
"${cargo_spec_features}" \
--features "${cargo_features}" \
--profile "${cargo_profile}" \
--target "${CARGO_TARGET}" \

View File

@@ -9,6 +9,7 @@ ARG CARGO_TARGET
ARG CARGO_TARGET_DIR
ARG cargo_profile
ARG cargo_features
ARG cargo_spec_features
ARG cook_args
ARG git_checkout
@@ -29,7 +30,7 @@ RUN \
cargo chef cook ${cook_args} \
--frozen \
--workspace \
--no-default-features \
"${cargo_spec_features}" \
--features "${cargo_features}" \
--profile "${cargo_profile}" \
--target "${CARGO_TARGET}" \

View File

@@ -9,6 +9,7 @@ ARG CARGO_TARGET
ARG CARGO_TARGET_DIR
ARG cargo_profile
ARG cargo_features
ARG cargo_spec_features
ARG cargo_args=""
ARG crate_path="src/main"
ARG crate_ident=""
@@ -37,7 +38,7 @@ RUN \
cargo install \
--locked \
--no-track \
--no-default-features \
"${cargo_spec_features}" \
--features "${cargo_features}" \
--profile "${cargo_profile}" \
--target "${CARGO_TARGET}" \

View File

@@ -5,7 +5,6 @@ ARG var_cache
ARG var_lib_apt
ARG packages
ENV DEBIAN_FRONTEND="noninteractive"
ENV packages="\
bzip2 \
ca-certificates \

View File

@@ -4,7 +4,7 @@ variable "repo" {}
cargo_feat_sets = {
none = ""
default = "brotli_compression,element_hacks,gzip_compression,io_uring,jemalloc,jemalloc_conf,media_thumbnail,release_max_log_level,systemd,url_preview,zstd_compression"
all = "blurhashing,brotli_compression,tuwunel_mods,console,default,direct_tls,element_hacks,gzip_compression,io_uring,jemalloc,jemalloc_conf,jemalloc_prof,jemalloc_stats,media_thumbnail,perf_measurements,release_max_log_level,sentry_telemetry,systemd,tokio_console,url_preview,zstd_compression"
all = "blurhashing,brotli_compression,tuwunel_mods,console,default,direct_tls,element_hacks,gzip_compression,hardened_malloc,io_uring,jemalloc,jemalloc_conf,jemalloc_prof,jemalloc_stats,ldap,media_thumbnail,perf_measurements,release_max_log_level,sentry_telemetry,systemd,tokio_console,url_preview,zstd_compression"
}
variable "cargo_features_always" {
@@ -708,7 +708,7 @@ target "audit" {
tags = [
elem_tag("audit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "audit"
target = "audit"
dockerfile = "docker/Dockerfile.cargo.audit"
matrix = cargo_rust_feat_sys
inherits = [
@@ -1018,6 +1018,10 @@ target "ingredients" {
cargo_feat_sets[feat_set],
cargo_features_always,
])
cargo_spec_features = (
feat_set == "all"?
"--all-features": "--no-default-features"
)
CARGO_TARGET_DIR = "/usr/src/tuwunel/target/${sys_name}/${sys_version}/${rust_toolchain}"
CARGO_BUILD_RUSTFLAGS = (
rust_toolchain == "nightly"?
@@ -1135,10 +1139,17 @@ target "kitchen" {
}
args = {
packages = join(" ", [
"bzip2",
contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? "liburing-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "libzstd-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? "libjemalloc-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "io_uring")?
"liburing-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")?
"libzstd-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")?
"libjemalloc-dev": "",
contains(split(",", cargo_feat_sets[feat_set]), "hardened_malloc")?
"g++": "",
])
}
}

View File

@@ -1,9 +1,14 @@
#!/bin/bash
set -eo pipefail
CI="${CI:-true}"
BASEDIR=$(dirname "$0")
CI="${CI:-false}"
CI_VERBOSE="${CI_VERBOSE:-false}"
CI_VERBOSE_ENV="${CI_VERBOSE_ENV:-$CI_VERBOSE}"
CI_SILENT_BAKE="${CI_SILENT_BAKE:-false}"
CI_PRINT_BAKE="${CI_PRINT_BAKE:-$CI_VERBOSE}"
default_cargo_profiles='["test", "bench"]'
default_feat_sets='["none", "default", "all"]'
default_rust_toolchains='["nightly", "stable"]'
@@ -52,6 +57,7 @@ sys_versions="${env_sys_versions:-$default_sys_versions}"
runner_name=$(echo $RUNNER_NAME | cut -d"." -f1)
runner_num=$(echo $RUNNER_NAME | cut -d"." -f2)
builder_name="owo"
rocksdb_opt_level=3
rocksdb_portable=1
git_checkout="HEAD"
@@ -72,10 +78,9 @@ if test "$CI" = "true"; then
export BUILDKIT_PROGRESS="plain"
fi
uwu_docker_build_args=""
args="$uwu_docker_build_args"
args="$args --builder owo"
args="$args --set *.platform=${sys_platform}"
args=""
args="$args --builder ${builder_name}"
#args="$args --set *.platform=${sys_platform}"
if test ! -z "$runner_num"; then
#cpu_num=$(expr $runner_num % $(nproc))
@@ -89,12 +94,19 @@ else
:
fi
trap 'set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' ERR
env
date
if test "$CI_SILENT_BAKE" = "true"; then
args="$args --progress=quiet"
fi
arg="$args -f $BASEDIR/bake.hcl"
if test "$BUILDKIT_PROGRESS" = "plain"; then
trap 'set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' ERR
if test "$CI_VERBOSE_ENV" = "true"; then
date
env
fi
if test "$CI_PRINT_BAKE" = "true"; then
docker buildx bake --print $arg $bake_target
fi
@@ -104,8 +116,6 @@ fi
trap '' ERR
set -ux
docker buildx bake $arg $bake_target
set +x
echo -e "\033[1;42;30mPASS\033[0m"

View File

@@ -1,9 +1,12 @@
#!/bin/bash
set -eo pipefail
CI="${CI:-true}"
BASEDIR=$(dirname "$0")
CI="${CI:-false}"
CI_VERBOSE="${CI_VERBOSE_ENV:-false}"
CI_VERBOSE_ENV="${CI_VERBOSE_ENV:-$CI_VERBOSE}"
default_cargo_profile="test"
default_feat_set="all"
default_rust_toolchain="nightly"
@@ -34,11 +37,18 @@ sock="/var/run/docker.sock"
arg="--rm --name $name -v $sock:$sock --network=host $tester_image ${testee_image}"
trap 'set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' ERR
date
env
if test "$CI_VERBOSE_ENV" = "true"; then
date
env
fi
set -x -e
cid=$(docker run -d $arg)
set +x
trap 'docker container stop $cid; set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' INT
docker logs -f "$cid"
docker wait "$cid" 2>/dev/null
echo -e "\033[1;42;30mPASS\033[0m"

View File

@@ -4,7 +4,7 @@ extern crate test;
#[cfg(tuwunel_bench)]
#[cfg_attr(tuwunel_bench, bench)]
fn ser_str(b: &mut test::Bencher) {
use tuwunel::ruma::{RoomId, UserId};
use tuwunel_core::ruma::{RoomId, UserId};
use crate::ser::serialize_to_vec;