Add cargo audit, lychee to workflow.

Rename / tweak some workflow jobs.

Signed-off-by: Jason Volk <jason@zemos.net>

.github/workflows/bake.yml

@@ -13,10 +13,6 @@ on:
         required: false
         default: '["test", "bench"]'
         description: Cargo profiles
-      docker_id:
-        type: string
-        required: false
-        description: Dockerhub acct/repo identity.
       feat_sets:
         type: string
         required: false
@@ -53,11 +49,8 @@ on:
         default: '["testing-slim"]'
         description: System versions
 
-env:
-  docker_id: ${{inputs.docker_id}}
-
 jobs:
-  task:
+  bake:
     name: Bake
     runs-on: ${{matrix.machine}}
     strategy:
@@ -75,7 +68,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Task
+    - name: Execute
       env:
         bake_target: ${{matrix.bake_target}}
         cargo_profile: ${{matrix.cargo_profile}}

.github/workflows/base.yml

@@ -1,16 +1,15 @@
-name: Base Environment
+name: Base
 
 on:
   workflow_call:
 
 jobs:
   systems:
-    name: Base Environment
+    name: System
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["systems"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -20,13 +19,12 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   buildsys:
-    name: Build Environment
+    name: Builder
     uses: ./.github/workflows/bake.yml
     needs: [systems]
     with:
       bake_targets: '["buildsys"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -36,13 +34,12 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   complement:
-    name: Test Environment
+    name: Tester
     uses: ./.github/workflows/bake.yml
     needs: [systems]
     with:
       bake_targets: '["complement-tester"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}

.github/workflows/build.yml

@@ -10,7 +10,6 @@ jobs:
     with:
       bake_targets: '["install"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -26,7 +25,6 @@ jobs:
     with:
       bake_targets: '["complement-testee"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}

.github/workflows/deps.yml

@@ -5,12 +5,11 @@ on:
 
 jobs:
   sources:
-    name: Acquire Source
+    name: Acquire
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["sources"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -20,13 +19,12 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   rocksdb:
-    name: Build RocksDB
+    name: RocksDB
     uses: ./.github/workflows/bake.yml
     needs: [sources]
     with:
       bake_targets: '["rocksdb"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -36,13 +34,12 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   deps:
-    name: Build Dependencies
+    name: Build
     uses: ./.github/workflows/bake.yml
     needs: [rocksdb]
     with:
-      bake_targets: '["deps"]'
+      bake_targets: '["deps-clippy", "deps-build-tests", "deps-build-bins"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}

.github/workflows/lint.yml

@@ -9,9 +9,36 @@ jobs:
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["fmt"]'
-      cargo_profiles: ${{vars.CARGO_PROFILES}}
+      cargo_profiles: '["test"]'
-      docker_id: ${{vars.DOCKER_ID}}
+      feat_sets: '["none"]'
-      feat_sets: ${{vars.FEAT_SETS}}
+      machines: ${{vars.MACHINES}}
+      rust_targets: ${{vars.RUST_TARGETS}}
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{vars.SYS_NAMES}}
+      sys_targets: ${{vars.SYS_TARGETS}}
+      sys_versions: ${{vars.SYS_VERSIONS}}
+
+  audit:
+    name: Audit
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["audit"]'
+      cargo_profiles: '["test"]'
+      feat_sets: '["none"]'
+      machines: ${{vars.MACHINES}}
+      rust_targets: ${{vars.RUST_TARGETS}}
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{vars.SYS_NAMES}}
+      sys_targets: ${{vars.SYS_TARGETS}}
+      sys_versions: ${{vars.SYS_VERSIONS}}
+
+  lychee:
+    name: Lychee
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["lychee"]'
+      cargo_profiles: '["test"]'
+      feat_sets: '["none"]'
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
       rust_toolchains: '["nightly"]'
@@ -25,7 +52,6 @@ jobs:
     with:
       bake_targets: '["clippy"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -40,12 +66,11 @@ jobs:
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["docs"]'
-      cargo_profiles: ${{vars.CARGO_PROFILES}}
+      cargo_profiles: '["test"]'
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
-      rust_toolchains: ${{vars.RUST_TOOLCHAINS}}
+      rust_toolchains: '["nightly"]'
       sys_names: ${{vars.SYS_NAMES}}
       sys_targets: ${{vars.SYS_TARGETS}}
       sys_versions: ${{vars.SYS_VERSIONS}}

.github/workflows/main.yml

@@ -7,20 +7,13 @@ on:
   pull_request:
     branches:
       - "**"
-
   workflow_dispatch:
-    inputs:
-      bake:
-        type: string
-        required: false
-        description: JSON Object of inputs passed to the environment
 
 concurrency:
   group: ${{github.workflow}}-${{github.ref}}
   cancel-in-progress: true
 
 env:
-  docker_id: ${{vars.DOCKER_ID}}
   inputs: ${{github.event.inputs}}
 
 jobs:
@@ -29,12 +22,12 @@ jobs:
     uses: ./.github/workflows/base.yml
 
   deps:
-    name: Dependencies
+    name: Deps
     uses: ./.github/workflows/deps.yml
     needs: [base]
 
   lint:
-    name: Linting
+    name: Lint
     uses: ./.github/workflows/lint.yml
     needs: [deps]
 

.github/workflows/test.yml

@@ -5,12 +5,11 @@ on:
 
 jobs:
   unit:
-    name: Unit Tests
+    name: Unit
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["tests-unit"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -20,12 +19,11 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   smoke:
-    name: Smoke Tests
+    name: Smoke
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["tests-smoke"]'
       cargo_profiles: ${{vars.CARGO_PROFILES}}
-      docker_id: ${{vars.DOCKER_ID}}
       feat_sets: ${{vars.FEAT_SETS}}
       machines: ${{vars.MACHINES}}
       rust_targets: ${{vars.RUST_TARGETS}}
@@ -35,18 +33,19 @@ jobs:
       sys_versions: ${{vars.SYS_VERSIONS}}
 
   complement:
-    name: Compliance Tests
+    name: Compliance
     needs: [smoke]
     runs-on: ${{matrix.machine}}
-    concurrency: ${{matrix.feat_set}}-${{matrix.sys_name}}-${{matrix.sys_target}}-${{matrix.sys_version}}
+    #concurrency: ${{matrix.feat_set}}-${{matrix.sys_name}}-${{matrix.sys_target}}-${{matrix.sys_version}}
+    concurrency: complement-cant-walk-and-chew-bubblegum
     strategy:
       fail-fast: false
       matrix:
-        cargo_profile: ${{fromJSON(vars.CARGO_PROFILES)}}
+        cargo_profile: ${{fromJSON('["test"]')}}
-        feat_set: ${{fromJSON(vars.FEAT_SETS)}}
+        feat_set: ${{fromJSON('["all"]')}}
         machine: ${{fromJSON(vars.MACHINES)}}
         rust_target: ${{fromJSON(vars.RUST_TARGETS)}}
-        rust_toolchain: ${{fromJSON(vars.RUST_TOOLCHAINS)}}
+        rust_toolchain: ${{fromJSON('["nightly"]')}}
         sys_name: ${{fromJSON(vars.SYS_NAMES)}}
         sys_target: ${{fromJSON(vars.SYS_TARGETS)}}
         sys_version: ${{fromJSON(vars.SYS_VERSIONS)}}

docker/Dockerfile.cargo.audit

@@ -0,0 +1,26 @@
+# syntax = docker/dockerfile:1.11-labs
+
+FROM input AS audit
+ARG sys_target
+ARG rust_toolchain="nightly"
+ARG RUSTUP_HOME
+ARG CARGO_HOME
+ARG audit_args=""
+
+WORKDIR /usr/src/tuwunel
+RUN \
+--mount=type=cache,dst=${RUSTUP_HOME},sharing=locked \
+--mount=type=cache,dst=${CARGO_HOME},sharing=locked \
+<<EOF
+    env
+    set -eux
+    rustup run ${rust_toolchain} \
+        cargo audit \
+            --stale \
+            --deny yanked \
+            --deny unsound \
+            --deny unmaintained \
+            --deny warnings \
+            --color=always \
+            ${audit_args}
+EOF

docker/Dockerfile.cargo.fmt

@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:1.11-labs
 
-FROM input AS cargo
+FROM input AS fmt
 ARG sys_target
 ARG rust_toolchain="nightly"
 ARG RUSTUP_HOME

docker/Dockerfile.cargo.lychee

@@ -0,0 +1,30 @@
+# syntax = docker/dockerfile:1.11-labs
+
+FROM input AS lychee
+ARG sys_target
+ARG rust_toolchain="nightly"
+ARG RUSTUP_HOME
+ARG CARGO_HOME
+ARG lychee_cache_age="7d"
+ARG lychee_mode="color"
+ARG lychee_args="docs *.md"
+
+WORKDIR /usr/src/tuwunel
+RUN \
+--mount=type=cache,dst=${RUSTUP_HOME},sharing=locked \
+--mount=type=cache,dst=${CARGO_HOME},sharing=locked \
+<<EOF
+    env
+    set -eux
+    rustup run ${rust_toolchain} \
+        lychee \
+            --verbose \
+            --cache \
+            --offline \
+            --max-cache-age ${lychee_cache_age} \
+            --mode ${lychee_mode} \
+            --exclude development.md \
+            --exclude contributing.md \
+            --exclude testing.md \
+            ${lychee_args}
+EOF

docker/Dockerfile.cookware

@@ -39,7 +39,9 @@ RUN \
     rustup component add \
         --toolchain ${rust_toolchain} \
         --target ${CARGO_TARGET} \
-        clippy rustfmt
+        clippy \
+        rustfmt \
+        ;
 EOF
 
 
@@ -59,5 +61,9 @@ RUN \
         cargo install \
             --locked \
             --target ${CARGO_TARGET} \
-            cargo-chef
+            cargo-chef \
+            cargo-audit \
+            cargo-deb \
+            lychee \
+            ;
 EOF

docker/Dockerfile.kitchen

@@ -14,6 +14,7 @@ cmake \
 curl \
 git \
 libc6-dev \
+libssl-dev \
 make \
 pkg-config \
 pkgconf \

docker/bake.hcl

@@ -123,8 +123,12 @@ group "default" {
 
 group "lints" {
     targets = [
+        "audit",
         "check",
         "clippy",
+        "docs",
+        "fmt",
+        "lychee",
     ]
 }
 
@@ -208,6 +212,7 @@ target "complement-testee-valgrind" {
     ]
     contexts = {
         input = elem("target:smoketest-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+        complement-tester = elem("target:complement-tester-valgrind", [feat_set, sys_name, sys_version, sys_target])
     }
 }
 
@@ -229,6 +234,7 @@ target "complement-testee" {
     ]
     contexts = {
         input = elem("target:install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+        complement-tester = elem("target:complement-tester", [feat_set, sys_name, sys_version, sys_target])
         complement-config = elem("target:complement-config", [feat_set, sys_name, sys_version, sys_target])
     }
     args = {
@@ -680,11 +686,46 @@ target "check" {
     }
 }
 
+target "lychee" {
+    name = elem("lychee", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("lychee", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    target = "lychee"
+    dockerfile = "docker/Dockerfile.cargo.lychee"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("deps-base", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+        elem("cargo", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:ingredients", [rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+}
+
+target "audit" {
+    name = elem("audit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("audit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+	target = "audit"
+    dockerfile = "docker/Dockerfile.cargo.audit"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("deps-base", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+        elem("cargo", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:ingredients", [rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+}
+
 target "fmt" {
     name = elem("fmt", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
     tags = [
         elem_tag("fmt", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
     ]
+    target = "fmt"
     dockerfile = "docker/Dockerfile.cargo.fmt"
     matrix = cargo_rust_feat_sys
     inherits = [
@@ -721,11 +762,11 @@ target "cargo" {
 
 group "deps" {
     targets = [
-        #"deps-check",
+        "deps-check",
         "deps-clippy",
-        #"deps-build",
+        "deps-build",
         "deps-build-tests",
-        #"deps-build-bench",
+        "deps-build-bench",
         "deps-build-bins",
     ]
 }

docker/bake.sh

@@ -1,11 +1,6 @@
 #!/bin/bash
 set -eo pipefail
 
-default_docker_id="jevolk/tuwunel"
-docker_id=${docker_id:=$default_docker_id}
-docker_acct=${docker_acct:=$(echo $docker_id | cut -d"/" -f1)}
-docker_repo=${docker_repo:=$(echo $docker_id | cut -d"/" -f2)}
-
 CI="${CI:-true}"
 BASEDIR=$(dirname "$0")
 
@@ -75,7 +70,6 @@ set +a
 export DOCKER_BUILDKIT=1
 if test "$CI" = "true"; then
     export BUILDKIT_PROGRESS="plain"
-    echo "plain"
 fi
 
 uwu_docker_build_args=""
@@ -101,7 +95,6 @@ date
 
 arg="$args -f $BASEDIR/bake.hcl"
 if test "$BUILDKIT_PROGRESS" = "plain"; then
-	echo "PRINTING"
     docker buildx bake --print $arg $bake_target
 fi
 
@@ -115,4 +108,4 @@ set -ux
 docker buildx bake $arg $bake_target
 
 set +x
-echo -e "\033[1;42;37mPASS\033[0m"
+echo -e "\033[1;42;30mPASS\033[0m"

docker/complement.sh

@@ -1,11 +1,6 @@
 #!/bin/bash
 set -eo pipefail
 
-default_docker_id="jevolk/tuwunel"
-docker_id=${docker_id:=$default_docker_id}
-docker_acct=${docker_acct:=$(echo $docker_id | cut -d"/" -f1)}
-docker_repo=${docker_repo:=$(echo $docker_id | cut -d"/" -f2)}
-
 CI="${CI:-true}"
 BASEDIR=$(dirname "$0")
 
@@ -46,4 +41,4 @@ cid=$(docker run -d $arg)
 set +x
 trap 'docker container stop $cid; set +x; date; echo -e "\033[1;41;37mFAIL\033[0m"' INT
 docker wait "$cid" 2>/dev/null
-echo -e "\033[1;42;37mPASS\033[0m"
+echo -e "\033[1;42;30mPASS\033[0m"

engage.toml

@@ -1,189 +0,0 @@
-interpreter = ["bash", "-euo", "pipefail", "-c"]
-
-[[task]]
-name = "engage"
-group = "versions"
-script = "engage --version"
-
-[[task]]
-name = "nix"
-group = "versions"
-script = "nix --version"
-
-[[task]]
-name = "direnv"
-group = "versions"
-script = "direnv --version"
-
-[[task]]
-name = "rustc"
-group = "versions"
-script = "rustc --version -v"
-
-[[task]]
-name = "cargo"
-group = "versions"
-script = "cargo --version -v"
-
-[[task]]
-name = "cargo-fmt"
-group = "versions"
-script = "cargo fmt --version"
-
-[[task]]
-name = "rustdoc"
-group = "versions"
-script = "rustdoc --version"
-
-[[task]]
-name = "cargo-clippy"
-group = "versions"
-script = "cargo clippy -- --version"
-
-[[task]]
-name = "cargo-audit"
-group = "versions"
-script = "cargo audit --version"
-
-[[task]]
-name = "cargo-deb"
-group = "versions"
-script = "cargo deb --version"
-
-[[task]]
-name = "lychee"
-group = "versions"
-script = "lychee --version"
-
-[[task]]
-name = "markdownlint"
-group = "versions"
-script = "markdownlint --version"
-
-[[task]]
-name = "cargo-audit"
-group = "security"
-script = "cargo audit --color=always -D warnings -D unmaintained -D unsound -D yanked"
-
-[[task]]
-name = "cargo-fmt"
-group = "lints"
-script = """
-cargo fmt --check -- --color=always
-"""
-
-[[task]]
-name = "cargo-doc"
-group = "lints"
-script = """
-env DIRENV_DEVSHELL=all-features \
-    RUSTDOCFLAGS="-D warnings" \
-    direnv exec . \
-    cargo doc \
-        --workspace \
-        --locked \
-        --profile test \
-        --all-features \
-        --no-deps \
-        --document-private-items \
-        --color always
-"""
-
-[[task]]
-name = "clippy/default"
-group = "lints"
-script = """
-direnv exec . \
-cargo clippy \
-    --workspace \
-    --locked \
-    --profile test \
-    --color=always \
-    -- \
-    -D warnings
-"""
-
-[[task]]
-name = "clippy/all"
-group = "lints"
-script = """
-env DIRENV_DEVSHELL=all-features \
-    direnv exec . \
-    cargo clippy \
-        --workspace \
-        --locked \
-        --profile test \
-        --all-features \
-        --color=always \
-        -- \
-        -D warnings
-"""
-
-[[task]]
-name = "clippy/no-features"
-group = "lints"
-script = """
-env DIRENV_DEVSHELL=no-features \
-    direnv exec . \
-    cargo clippy \
-        --workspace \
-        --locked \
-        --profile test \
-        --no-default-features \
-        --color=always \
-        -- \
-        -D warnings
-"""
-
-[[task]]
-name = "clippy/other-features"
-group = "lints"
-script = """
-direnv exec . \
-cargo clippy \
-    --workspace \
-    --locked \
-    --profile test \
-    --no-default-features \
-    --features=console,systemd,element_hacks,direct_tls,perf_measurements,brotli_compression,blurhashing \
-    --color=always \
-    -- \
-    -D warnings
-"""
-
-[[task]]
-name = "lychee"
-group = "lints"
-script = "lychee --verbose --offline docs *.md --exclude development.md --exclude contributing.md --exclude testing.md"
-
-[[task]]
-name = "markdownlint"
-group = "lints"
-script = "markdownlint docs *.md || true" # TODO: fix the ton of markdown lints so we can drop `|| true`
-
-[[task]]
-name = "cargo/default"
-group = "tests"
-script = """
-env DIRENV_DEVSHELL=default \
-    direnv exec . \
-    cargo test \
-        --workspace \
-        --locked \
-        --profile test \
-        --all-targets \
-        --no-fail-fast \
-        --color=always \
-        -- \
-        --color=always
-"""
-
-# Checks if the generated example config differs from the checked in repo's
-# example config.
-[[task]]
-name = "example-config"
-group = "tests"
-depends = ["cargo/default"]
-script = """
-git diff --exit-code conduwuit-example.toml
-"""