ci: Add matrix-rust-sdk integration testing.

Signed-off-by: Jason Volk <jason@zemos.net>

.github/workflows/main.yml

@@ -47,6 +47,7 @@ jobs:
       docker_repo: ${{vars.DOCKER_REPO}}
       release_url: ${{steps.release.outputs.upload_url}}
       pages_url: 'https://matrix-construct.github.io/tuwunel/'
+      rust_sdk_integration: ${{vars.RUST_SDK_INTEGRATION || 'true'}}
 
     strategy:
       fail-fast: true
@@ -59,8 +60,8 @@ jobs:
     - name: Initialize Builder
       env:
         runner: ${{matrix.runner}}
-        reserved_space: '{"het": "128GB", "aws": "48GB", "gcp": "160GB"}'
+        reserved_space: '{"het": "192GB", "aws": "48GB", "gcp": "160GB"}'
-        max_used_space: '{"het": "256GB", "aws": "64GB", "gcp": "192GB"}'
+        max_used_space: '{"het": "384GB", "aws": "64GB", "gcp": "192GB"}'
       run: |
         set +e
         docker buildx inspect "${GITHUB_ACTOR}"
@@ -167,6 +168,7 @@ jobs:
     needs: [init, lint]
     uses: ./.github/workflows/test.yml
     with:
+      rust_sdk_integration: ${{fromJSON(needs.init.outputs.rust_sdk_integration)}}
       complement: ${{fromJSON(needs.init.outputs.complement)}}
       complement_runner: ${{needs.init.outputs.complement_runner}}
       cargo_profiles: ${{needs.init.outputs.cargo_profiles}}

.github/workflows/test.yml

@@ -43,6 +43,9 @@ on:
         default: '["all"]'
       complement_runner:
         type: string
+      rust_sdk_integration:
+        type: boolean
+        default: true
 
 jobs:
   docs:
@@ -160,6 +163,38 @@ jobs:
           {"sys_target": "x86_64-v4-linux-gnu", "bake_target": "smoke-valgrind"},
         ]
 
+  rust-sdk-integration:
+    if: >
+      !failure() && !cancelled()
+      && inputs.rust_sdk_integration
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
+      && contains(fromJSON(inputs.complement_feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
+
+    name: Rust SDK Integration
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["matrix-rust-sdk-integration"]'
+      cargo_profiles: '["test"]'
+      feat_sets: ${{inputs.complement_feat_sets}}
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: ${{inputs.rust_targets}}
+      sys_targets: '["x86_64-v1-linux-gnu"]'
+      machines: ${{inputs.machines}}
+      runner: ${{inputs.complement_runner}}
+      excludes: ${{inputs.excludes}}
+      includes: ${{inputs.includes}}
+      artifact: >
+        {
+          "matrix-rust-sdk-integration": {
+            "src": "/var/log/tuwunel.log",
+            "dst": "rust-sdk-integration.tuwunel.log",
+          }
+        }
+
   complement:
     if: >
       !failure() && !cancelled()

docker/Dockerfile.complement

@@ -51,7 +51,6 @@ COPY <<EOF complement.toml
     [global]
     address = "0.0.0.0"
     admin_room_notices = false
-    allow_check_for_updates = false
     allow_device_name_federation = true
     allow_guest_registration = true
     allow_invalid_tls_certificates = true
@@ -73,7 +72,6 @@ COPY <<EOF complement.toml
     log_thread_ids = true
     media_compat_file_link = false
     media_startup_check = true
-    only_query_trusted_key_servers = false
     port = [8008, 8448]
     prune_missing_media = true
     query_trusted_key_servers_first = false

docker/Dockerfile.matrix-rust-sdk

@@ -0,0 +1,108 @@
+# syntax = docker/dockerfile:1.11-labs
+
+FROM input AS matrix-rust-sdk-integration
+ARG sys_name
+ARG sys_version
+ARG rust_toolchain
+ARG rust_target
+ARG RUSTUP_HOME
+ARG CARGO_HOME
+ARG CARGO_TARGET
+ARG MRSDK_TARGET_DIR="/usr/src/matrix-rust-sdk"
+ARG mrsdk_profile="release"
+ARG mrsdk_target_share="${MRSDK_TARGET_DIR}/${sys_name}/${sys_version}/${rust_target}/${rust_toolchain}/${mrsdk_profile}/_shared_cache"
+ARG mrsdk_test_args=""
+ARG mrsdk_test_opts=""
+#TODO!!!
+ARG mrsdk_skip_list="--skip test_history_share_on_invite_pin_violation --skip delayed_invite_response_and_sent_message_decryption --skip test_mutual_sas_verification_with_notification_client_ignores_verification_events"
+
+WORKDIR /
+COPY --link --from=input . .
+
+WORKDIR /usr/src
+ADD --link https://github.com/matrix-construct/matrix-rust-sdk.git matrix-rust-sdk
+
+WORKDIR /etc
+COPY <<EOF tuwunel.toml
+    [global]
+    admin_room_notices = false
+    allow_device_name_federation = true
+    allow_guest_registration = true
+    allow_legacy_media = true
+    allow_public_room_directory_over_federation = true
+    allow_public_room_directory_without_auth = true
+    allow_registration = true
+    create_admin_room = false
+    ip_range_denylist = []
+    log = "debug,tuwunel=trace,h2=warn,hyper=warn"
+    log_colors = false
+    log_guest_registrations = false
+    log_span_events = "NONE"
+    log_thread_ids = true
+    media_compat_file_link = false
+    media_startup_check = true
+    query_trusted_key_servers_first = false
+    query_trusted_key_servers_first_on_join = false
+    rocksdb_log_level = "debug"
+    rocksdb_max_log_files = 1
+    rocksdb_paranoid_file_checks = true
+    rocksdb_recovery_mode = 0
+    trusted_servers = []
+    url_preview_domain_contains_allowlist = ["*"]
+    url_preview_domain_explicit_denylist = ["*"]
+    yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true
+EOF
+
+WORKDIR /usr/lib
+COPY --link --from=install /usr/lib .
+
+WORKDIR /usr/bin
+COPY --link --from=install /usr/bin/tuwunel .
+
+WORKDIR /usr/src/matrix-rust-sdk
+ENV targ_dir="${MRSDK_TARGET_DIR}/${mrsdk_profile}"
+ENV targ_targ_dir="${MRSDK_TARGET_DIR}/${rust_target}/${mrsdk_profile}"
+ENV RUST_BACKTRACE="full"
+ENV TUWUNEL_CONFIG="/etc/tuwunel.toml"
+ENV TUWUNEL_DATABASE_PATH="/var/db/tuwunel"
+ENV TUWUNEL_SERVER_NAME="localhost:6167"
+ENV TUWUNEL_PORT="[6167]"
+ENV HOMESERVER_URL="http://localhost:6167"
+RUN \
+--mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=shared \
+--mount=type=cache,dst=${CARGO_HOME}/registry,sharing=shared \
+--mount=type=cache,dst=${CARGO_HOME}/git,sharing=shared \
+--mount=type=cache,dst=${targ_dir}/deps,id=${mrsdk_target_share}/deps,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/build,id=${mrsdk_target_share}/build,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/examples,id=${mrsdk_target_share}/examples,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/incremental,id=${mrsdk_target_share}/incremental,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/.fingerprint,id=${mrsdk_target_share}/fingerprint,sharing=locked \
+--mount=type=cache,dst=${targ_targ_dir}/deps,id=${mrsdk_target_share}/deps,sharing=locked \
+--mount=type=cache,dst=${targ_targ_dir}/build,id=${mrsdk_target_share}/build,sharing=locked \
+--mount=type=cache,dst=${targ_targ_dir}/examples,id=${mrsdk_target_share}/examples,sharing=locked \
+--mount=type=cache,dst=${targ_targ_dir}/incremental,id=${mrsdk_target_share}/incremental,sharing=locked \
+--mount=type=cache,dst=${targ_targ_dir}/.fingerprint,id=${mrsdk_target_share}/fingerprint,sharing=locked \
+<<EOF
+    set -eux
+
+    nohup /usr/bin/tuwunel 1> /var/log/tuwunel.log &
+    PID=$!; trap "sleep 10s; set +ex; kill -QUIT ${PID}; wait ${PID}" EXIT
+    sleep 3s
+
+    rustup run ${rust_toolchain} \
+        cargo test \
+            --locked \
+            --release \
+            --color always \
+            --features default \
+            --target "${rust_target}" \
+            --target-dir "${MRSDK_TARGET_DIR}" \
+            --package matrix-sdk-integration-testing \
+            ${mrsdk_test_args} \
+            -- \
+            --test-threads 1 \
+            --color always \
+            ${mrsdk_skip_list} \
+            ${mrsdk_test_opts} \
+            ;
+EOF

docker/bake.hcl

@@ -241,6 +241,7 @@ group "tests" {
         "unit",
         "smoke",
         "complement",
+        "mrsdk",
     ]
 }
 
@@ -429,6 +430,39 @@ target "complement-config" {
     }
 }
 
+#
+# Matrix Rust SDK tests
+#
+
+group "matrix-rust-sdk" {
+    targets = [
+        "matrix-rust-sdk-integration",
+    ]
+}
+
+target "matrix-rust-sdk-integration" {
+    name = elem("matrix-rust-sdk-integration", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("matrix-rust-sdk-integration", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    output = ["type=docker,compression=zstd,mode=max,compression-level=${zstd_image_compress_level}"]
+    cache_to = ["type=local,compression=zstd,mode=max,compression-level=${cache_compress_level}"]
+    target = "matrix-rust-sdk-integration"
+    dockerfile = "${docker_dir}/Dockerfile.matrix-rust-sdk"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("rust", [rust_toolchain, rust_target, sys_name, sys_version, sys_target])
+    ]
+    contexts = {
+        input = elem("target:rust", [rust_toolchain, rust_target, sys_name, sys_version, sys_target])
+        install = elem("target:install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        mrsdk_test_args=""
+        mrsdk_test_opts=""
+    }
+}
+
 #
 # Smoke tests
 #
@@ -1618,6 +1652,7 @@ kitchen_packages = [
     "jq",
     "libc6-dev",
     "libssl-dev",
+    "libsqlite3-dev",
     "make",
     "openssl",
     "pkg-config",