From 09680fc86b8081650777805b637935897504fe98 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 5 May 2025 00:58:05 +0000 Subject: [PATCH] Restore the bin/ contents under nix/ (for now) [ci skip] Signed-off-by: Jason Volk --- nix/pkgs/complement/bin/complement | 95 +++++++++++++++++ nix/pkgs/complement/bin/nix-build-and-cache | 110 ++++++++++++++++++++ nix/pkgs/complement/default.nix | 2 +- 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 nix/pkgs/complement/bin/complement create mode 100644 nix/pkgs/complement/bin/nix-build-and-cache diff --git a/nix/pkgs/complement/bin/complement b/nix/pkgs/complement/bin/complement new file mode 100644 index 00000000..c437503e --- /dev/null +++ b/nix/pkgs/complement/bin/complement @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Path to Complement's source code +# +# The `COMPLEMENT_SRC` environment variable is set in the Nix dev shell, which +# points to a store path containing the Complement source code. It's likely you +# want to just pass that as the first argument to use it here. +COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}" + +# A `.jsonl` file to write test logs to +LOG_FILE="${2:-complement_test_logs.jsonl}" + +# A `.jsonl` file to write test results to +RESULTS_FILE="${3:-complement_test_results.jsonl}" + +COMPLEMENT_BASE_IMAGE="${COMPLEMENT_BASE_IMAGE:-complement-conduwuit:main}" + +# Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time +SKIPPED_COMPLEMENT_TESTS='TestPartialStateJoin.*|TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.*|TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias|TestUnbanViaInvite.*|TestRoomState/Parallel/GET_/publicRooms_lists.*"|TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other.*' + +# $COMPLEMENT_SRC needs to be a directory to Complement source code +if [ -f "$COMPLEMENT_SRC" ]; then + echo "\$COMPLEMENT_SRC must be a directory/path to Complement source code" + exit 1 +fi + +# quick test to make sure we can actually write to $LOG_FILE and $RESULTS_FILE +touch $LOG_FILE && rm -v $LOG_FILE +touch $RESULTS_FILE && rm -v $RESULTS_FILE + +toplevel="$(git rev-parse --show-toplevel)" + +pushd "$toplevel" > /dev/null + +if [ ! -f "complement_oci_image.tar.gz" ]; then + echo "building complement conduwuit image" + + # if using macOS, use linux-complement + #bin/nix-build-and-cache just .#linux-complement + bin/nix-build-and-cache just .#complement + #nix build -L .#complement + + echo "complement conduwuit image tar.gz built at \"result\"" + + echo "loading into docker" + docker load < result + popd > /dev/null +else + echo "skipping building a complement conduwuit image as complement_oci_image.tar.gz was already found, loading this" + + docker load < complement_oci_image.tar.gz + popd > /dev/null +fi + +echo "" +echo "running go test with:" +echo "\$COMPLEMENT_SRC: $COMPLEMENT_SRC" +echo "\$COMPLEMENT_BASE_IMAGE: $COMPLEMENT_BASE_IMAGE" +echo "\$RESULTS_FILE: $RESULTS_FILE" +echo "\$LOG_FILE: $LOG_FILE" +echo "" + +# It's okay (likely, even) that `go test` exits nonzero +# `COMPLEMENT_ENABLE_DIRTY_RUNS=1` reuses the same complement container for faster complement, at the possible expense of test environment pollution +set +o pipefail +env \ + -C "$COMPLEMENT_SRC" \ + COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \ + go test -tags="conduwuit_blacklist" -skip="$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" +set -o pipefail + +# Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results +cat "$LOG_FILE" | jq -s -c 'sort_by(.Test)[]' | jq -c ' + select( + (.Action == "pass" or .Action == "fail" or .Action == "skip") + and .Test != null + ) | {Action: .Action, Test: .Test} + ' > "$RESULTS_FILE" + +#if command -v gotestfmt &> /dev/null; then +# echo "using gotestfmt on $LOG_FILE" +# grep '{"Time":' "$LOG_FILE" | gotestfmt > "complement_test_logs_gotestfmt.log" +#fi + +echo "" +echo "" +echo "complement logs saved at $LOG_FILE" +echo "complement results saved at $RESULTS_FILE" +#if command -v gotestfmt &> /dev/null; then +# echo "complement logs in gotestfmt pretty format outputted at complement_test_logs_gotestfmt.log (use an editor/terminal/pager that interprets ANSI colours and UTF-8 emojis)" +#fi +echo "" +echo "" diff --git a/nix/pkgs/complement/bin/nix-build-and-cache b/nix/pkgs/complement/bin/nix-build-and-cache new file mode 100644 index 00000000..ac64ff23 --- /dev/null +++ b/nix/pkgs/complement/bin/nix-build-and-cache @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +set -eo pipefail + +toplevel="$(git rev-parse --show-toplevel)" + +# Build just the single installable and forward any other arguments too +just() { + # uses nix-output-monitor (nom) if available + if command -v nom &> /dev/null; then + nom build "$@" + else + nix build -L "$@" + fi + + if [ -z "$ATTIC_TOKEN" ]; then + echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" + return + fi + + # historical "conduit" store for compatibility purposes, same as conduwuit + nix run --inputs-from "$toplevel" attic -- \ + login \ + conduit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \ + "$ATTIC_TOKEN" + + # Find all output paths of the installables and their build dependencies + #readarray -t derivations < <(nix path-info --derivation "$@") + derivations=() + while IFS=$'\n' read derivation; do + derivations+=("$derivation") + done < <(nix path-info --derivation "$@") + + cache=() + for derivation in "${derivations[@]}"; do + cache+=( + "$(nix-store --query --requisites --include-outputs "$derivation")" + ) + done + + withattic() { + nix shell --inputs-from "$toplevel" attic --command xargs attic push "$@" <<< "${cache[*]}" + } + # Upload them to Attic (conduit store) + # + # Use `xargs` and a here-string because something would probably explode if + # several thousand arguments got passed to a command at once. Hopefully no + # store paths include a newline in them. + ( + IFS=$'\n' + withattic conduit || withattic conduit || withattic conduit || true + ) + + # main "conduwuit" store + nix run --inputs-from "$toplevel" attic -- \ + login \ + conduwuit \ + "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \ + "$ATTIC_TOKEN" + + # Upload them to Attic (conduwuit store) and Cachix + # + # Use `xargs` and a here-string because something would probably explode if + # several thousand arguments got passed to a command at once. Hopefully no + # store paths include a newline in them. + ( + IFS=$'\n' + withattic conduwuit || withattic conduwuit || withattic conduwuit || true + + # push to cachix if available + if [ "$CACHIX_AUTH_TOKEN" ]; then + nix shell --inputs-from "$toplevel" cachix -c xargs \ + cachix push conduwuit <<< "${cache[*]}" + fi + ) +} + +# Build and cache things needed for CI +ci() { + cache=( + --inputs-from "$toplevel" + + # Keep sorted + #"$toplevel#devShells.x86_64-linux.default" + #"$toplevel#devShells.x86_64-linux.all-features" + attic#default + cachix#default + nixpkgs#direnv + nixpkgs#jq + nixpkgs#nix-direnv + ) + + just "${cache[@]}" +} + +# Build and cache *all* the package outputs from the flake.nix +packages() { + declare -a cache="($( + nix flake show --json 2> /dev/null | + nix run --inputs-from "$toplevel" nixpkgs#jq -- \ + -r \ + '.packages."x86_64-linux" | keys | map("'"$toplevel"'#" + .) | @sh' + ))" + + just "${cache[@]}" +} + + +eval "$@" diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 9b010e14..a2d57152 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -53,7 +53,7 @@ dockerTools.buildImage { copyToRoot = buildEnv { name = "root"; pathsToLink = [ - "/bin" + "/nix/pkgs/complement/bin" ]; paths = [ bashInteractive