Use insta and criterion for main integration test and benches respectively.

docker/ci: Separate integration and unit tests and benches jobs.

Add directives to remove db before/after integration tests are performed.

Split start/run/stop phases; add more granular smoketests.

Split main integration tests into units for isolation.

Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk
2025-10-01 02:41:24 +00:00
parent 43f0882d83
commit 8d6bfde5a0
22 changed files with 373 additions and 100 deletions

View File

@@ -87,7 +87,7 @@ jobs:
name: Unit
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["unit"]'
bake_targets: '["unit", "integration"]'
cargo_profiles: '["test"]'
feat_sets: '["all"]'
rust_toolchains: ${{inputs.rust_toolchains}}
@@ -110,7 +110,7 @@ jobs:
name: Bench
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["unit"]'
bake_targets: '["unit", "integration"]'
cargo_profiles: '["bench"]'
feat_sets: '["all"]'
rust_toolchains: '["nightly"]'
@@ -222,7 +222,7 @@ jobs:
excludes: ${{inputs.excludes}}
includes: ${{inputs.includes}}
rust-sdk-integration:
matrix-rust-sdk-integration:
if: >
!failure() && !cancelled()
&& !contains(inputs.head_msg, '[ci no build]')
@@ -232,7 +232,8 @@ jobs:
&& contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
&& contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
name: Rust SDK Integration
name: Matrix SDK Integration
needs: [smoke]
uses: ./.github/workflows/bake.yml
with:
bake_targets: '["matrix-rust-sdk-integration"]'
@@ -251,7 +252,7 @@ jobs:
{
"matrix-rust-sdk-integration": {
"src": "/var/log/tuwunel.log",
"dst": "rust-sdk-integration.tuwunel.log",
"dst": "matrix-rust-sdk-integration.tuwunel.log",
}
}
@@ -293,7 +294,7 @@ jobs:
&& contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
name: Matrix Compliance
needs: [complement]
needs: [complement, smoke]
runs-on: ["${{matrix.machine}}", "${{inputs.complement_runner}}"]
concurrency:
group: complement-cant-walk-and-chew-bubblegum

4
Cargo.lock generated
View File

@@ -4962,13 +4962,17 @@ version = "1.4.2"
dependencies = [
"clap",
"const-str",
"criterion",
"ctor",
"insta",
"log",
"maplit",
"opentelemetry",
"opentelemetry_sdk",
"sentry",
"sentry-tower",
"sentry-tracing",
"similar",
"tokio",
"tokio-metrics",
"tracing",

View File

@@ -703,7 +703,7 @@ inherits = "release-native.build-override"
#]
[profile.bench]
debug = "limited"
debug = false
strip = false
#rustflags = [
# "-Cremark=all",

View File

@@ -87,7 +87,7 @@ RUN \
"--manifest-path=Cargo.toml" \
${recipe_args} \
${color_args} \
${cargo_args}
$cargo_args
# If this image is further reused with other cargo commands, all
# modifications made by cargo chef cook outside of target-dir have to be

View File

@@ -26,14 +26,15 @@ ARG sys_target
WORKDIR /usr/src/tuwunel
COPY --link --from=source /usr/src/tuwunel .
ENV TUWUNEL_DATABASE_PATH="/tmp/buildtest.db"
RUN \
--mount=type=cache,dst=/nix,sharing=shared \
--mount=type=cache,dst=/root/.cache/nix,sharing=shared \
--mount=type=cache,dst=/root/.local/state/nix,sharing=shared \
<<EOF
set -eux
nix-build \
--verbose \
--cores 0 \
--max-jobs $(nproc) \
--log-format raw \
@@ -51,8 +52,8 @@ ARG sys_target
WORKDIR /
COPY --link --from=build-nix . .
WORKDIR /opt/tuwunel
ENV TUWUNEL_DATABASE_PATH="/tmp/smoketest.db"
WORKDIR /tmp/tuwunel
ENV TUWUNEL_DATABASE_PATH="/tmp/tuwunel/smoketest.db"
ENV TUWUNEL_LOG="info"
RUN \
--mount=type=cache,dst=/nix,sharing=shared \
@@ -61,11 +62,8 @@ RUN \
<<EOF
set -eux
bin/tuwunel \
-Otest='["smoke"]' \
-Otest='["smoke", "fresh"]' \
-Oserver_name=\"localhost\" \
-Odatabase_path=\"${TUWUNEL_DATABASE_PATH}\"
rm -rf "${TUWUNEL_DATABASE_PATH}"
EOF

View File

@@ -239,8 +239,15 @@ group "tests" {
"docs",
"unit",
"smoke",
"integration",
"matrix-compliance",
]
}
group "matrix-compliance" {
targets = [
"complement",
"mrsdk",
"matrix-rust-sdk-integration",
]
}
@@ -430,12 +437,13 @@ target "complement-config" {
}
#
# Matrix Rust SDK tests
# Integration tests
#
group "matrix-rust-sdk" {
group "integrations" {
targets = [
"matrix-rust-sdk-integration",
"integration",
]
}
@@ -462,6 +470,29 @@ target "matrix-rust-sdk-integration" {
}
}
target "integration" {
name = elem("integration", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
tags = [
elem_tag("integration", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "cargo"
matrix = cargo_rust_feat_sys
inherits = [
elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
]
contexts = {
input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
}
args = {
TUWUNEL_DATABASE_PATH = "/tmp/integration.test.db"
cargo_cmd = (cargo_profile == "bench"? "bench": "test")
cargo_args = (cargo_profile == "bench"?
"--no-fail-fast --bench=*": "--no-fail-fast --test=*"
)
}
}
#
# Smoke tests
#
@@ -562,6 +593,50 @@ target "tests-smoke" {
}
}
#
# Unit tests
#
target "unit" {
name = elem("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
tags = [
elem_tag("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "cargo"
matrix = cargo_rust_feat_sys
inherits = [
elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
]
contexts = {
input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
}
args = {
cargo_cmd = (cargo_profile == "bench"? "bench": "test")
cargo_args = (cargo_profile == "bench"?
"--no-fail-fast --lib": "--no-fail-fast --lib --bins"
)
}
}
target "docs" {
name = elem("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
tags = [
elem_tag("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "cargo"
matrix = cargo_rust_feat_sys
inherits = [
elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
]
contexts = {
input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
}
args = {
cargo_cmd = "test"
cargo_args = "--doc --no-fail-fast"
}
}
#
# Installation
#
@@ -871,50 +946,6 @@ target "build-nix" {
}
}
#
# Unit tests
#
target "unit" {
name = elem("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
tags = [
elem_tag("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "cargo"
matrix = cargo_rust_feat_sys
inherits = [
elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
]
contexts = {
input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
}
args = {
cargo_cmd = (cargo_profile == "bench"? "bench": "test")
cargo_args = (rust_toolchain == "nightly"?
"--no-fail-fast --all-targets -- --color=always": "--no-fail-fast --bins --tests"
)
}
}
target "docs" {
name = elem("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
tags = [
elem_tag("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
]
target = "cargo"
matrix = cargo_rust_feat_sys
inherits = [
elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
]
contexts = {
input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
}
args = {
cargo_cmd = "test"
cargo_args = "--doc --no-fail-fast"
}
}
#
# Workspace builds
#
@@ -1004,7 +1035,7 @@ target "build-tests" {
}
args = {
cargo_cmd = (cargo_profile == "bench"? "bench": "test")
cargo_args = "--no-run"
cargo_args = (cargo_profile == "bench"? "--no-run --benches": "--no-run --tests")
}
}
@@ -1292,7 +1323,7 @@ target "deps-base" {
CARGO_PROFILE_TEST_DEBUG = "false"
CARGO_PROFILE_TEST_INCREMENTAL = "false"
CARGO_PROFILE_BENCH_DEBUG = "limited"
CARGO_PROFILE_BENCH_DEBUG = "false"
CARGO_PROFILE_BENCH_LTO = "thin"
CARGO_PROFILE_RELEASE_LTO = "thin"
CARGO_PROFILE_RELEASE_DEBUGINFO_DEBUG = "limited"
@@ -1320,7 +1351,7 @@ target "deps-base" {
"-C link-arg=-l:libgcc.a": "",
]):
cargo_profile == "release" && rust_toolchain == "nightly"?
(cargo_profile == "release" || cargo_profile == "bench") && rust_toolchain == "nightly"?
join(" ", [
join(" ", rustflags),
join(" ", nightly_rustflags),
@@ -1342,7 +1373,7 @@ target "deps-base" {
"-C link-arg=-l:libgcc.a": "",
]):
cargo_profile == "release" || cargo_profile == "release-debuginfo"?
cargo_profile == "release" || cargo_profile == "release-debuginfo" || cargo_profile == "bench"?
join(" ", [
join(" ", rustflags),
join(" ", static_rustflags),

View File

@@ -114,10 +114,10 @@ url.workspace = true
nix.workspace = true
[dev-dependencies]
criterion.workspace = true
insta.workspace = true
maplit.workspace = true
similar.workspace = true
criterion.workspace = true
[lints]
workspace = true

View File

@@ -45,10 +45,29 @@ pub struct Args {
#[arg(long)]
pub execute: Vec<String>,
/// Set functional testing modes if available. Ex '--test=smoke'
#[arg(long, hide(true))]
/// Set functional testing modes if available. Ex '--test=smoke'. Empty
/// values are permitted for compatibility with testing and benchmarking
/// frameworks which may simply pass `--test` to the same execution.
#[arg(
long,
hide(true),
num_args = 0..=1,
require_equals(false),
default_missing_value = "",
)]
pub test: Vec<String>,
/// Compatibility option for benchmark frameworks which pass `--bench` to
/// the same execution and must be silently accepted without error.
#[arg(
long,
hide(true),
num_args = 0..=1,
require_equals(false),
default_missing_value = "",
)]
pub bench: Vec<String>,
/// Override the tokio worker_thread count.
#[arg(
long,
@@ -151,12 +170,12 @@ pub struct Args {
impl Args {
#[must_use]
pub fn default_test(name: &str) -> Self {
pub fn default_test(name: &[&str]) -> Self {
let mut args = Self::default();
args.test.push(name.into());
args.test
.extend(name.iter().copied().map(ToOwned::to_owned));
args.option
.push("server_name=\"localhost\"".into());
args
}
}

View File

@@ -17,10 +17,6 @@ crate-type = [
# "dylib",
]
[[bench]]
name = "ser"
harness = false
[features]
bzip2_compression = [
"rust-rocksdb/bzip2",
@@ -76,3 +72,7 @@ criterion.workspace = true
[lints]
workspace = true
[[bench]]
name = "ser"
harness = false

View File

@@ -1,6 +1,6 @@
use criterion::{Criterion, criterion_group, criterion_main};
criterion_group!(benches, ser_str,);
criterion_group!(benches, ser_str);
criterion_main!(benches);

View File

@@ -1,10 +1,15 @@
use std::{
collections::BTreeMap,
fs::remove_dir_all,
path::Path,
sync::{Arc, Mutex},
};
use rocksdb::{Cache, Env, LruCacheOptions};
use tuwunel_core::{Result, Server, debug, utils::math::usize_from_f64};
use tuwunel_core::{
Result, Server, debug,
utils::{math::usize_from_f64, result::LogErr},
};
use crate::{or_else, pool::Pool};
@@ -78,5 +83,55 @@ impl Drop for Context {
debug!("Joining background threads...");
env.join_all_threads();
after_close(self, &self.server.config.database_path)
.expect("Failed to execute after_close handler");
}
}
/// For unit and integration tests the 'fresh' directive deletes found db.
pub(super) fn before_open(ctx: &Arc<Context>, path: &Path) -> Result {
if ctx.server.config.test.contains("fresh") {
match delete_database_for_testing(ctx, path) {
| Err(e) if !e.is_not_found() => return Err(e),
| _ => (),
}
}
Ok(())
}
/// For unit and integration tests the 'cleanup' directive deletes after close
/// to cleanup.
fn after_close(ctx: &Context, path: &Path) -> Result {
if ctx.server.config.test.contains("cleanup") {
delete_database_for_testing(ctx, path)
.log_err()
.ok();
}
Ok(())
}
/// For unit and integration tests; removes the database directory when called.
/// To prevent misuse, cfg!(test) must be true for a unit test or the
/// integration test server is named localhost.
#[tracing::instrument(level = "debug", skip_all)]
fn delete_database_for_testing(ctx: &Context, path: &Path) -> Result {
let config = &ctx.server.config;
let localhost = config
.server_name
.as_str()
.starts_with("localhost");
if !cfg!(test) && !localhost {
return Ok(());
}
debug_assert!(
config.test.contains("cleanup") | config.test.contains("fresh"),
"missing any test directive legitimating this call.",
);
remove_dir_all(path).map_err(Into::into)
}

View File

@@ -10,6 +10,7 @@ use tuwunel_core::{Result, debug, implement, info, warn};
use super::{
Db, Engine,
cf_opts::cf_options,
context,
db_opts::db_options,
descriptor::{self, Descriptor},
repair::repair,
@@ -23,6 +24,7 @@ pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<S
let config = &server.config;
let path = &config.database_path;
context::before_open(&ctx, path)?;
let db_opts = db_options(
config,
&ctx.env.lock().expect("environment locked"),

View File

@@ -13,6 +13,7 @@ version.workspace = true
name = "tuwunel_macros"
path = "mod.rs"
proc-macro = true
bench = false
[dependencies]
syn.workspace = true

View File

@@ -51,11 +51,6 @@ assets = [
name = "tuwunel"
pkgdesc = "High performance Matrix homeserver written in Rust"
[lib]
path = "lib.rs"
bench = false
crate-type = ["rlib"]
[features]
default = [
"brotli_compression",
@@ -239,10 +234,25 @@ tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
tracing.workspace = true
[dev-dependencies]
criterion.workspace = true
insta.workspace = true
maplit.workspace = true
similar.workspace = true
[lints]
workspace = true
[lib]
path = "lib.rs"
bench = false
crate-type = ["rlib"]
[[bin]]
name = "tuwunel"
path = "main.rs"
bench = false
[[bench]]
name = "main"
harness = false

35
src/main/benches/main.rs Normal file
View File

@@ -0,0 +1,35 @@
use criterion::{Criterion, criterion_group, criterion_main};
use tracing::Level;
use tuwunel::Server;
use tuwunel_core::{Args, result::ErrLog, runtime};
criterion_group!(
name = benches;
config = Criterion::default().sample_size(10).nresamples(1);
targets = dummy, smoke
);
criterion_main!(benches);
fn dummy(c: &mut Criterion) { c.bench_function("dummy", |c| c.iter(|| {})); }
fn smoke(c: &mut Criterion) {
let args = Args::default_test(&["fresh", "cleanup"]);
let runtime = runtime::new(Some(&args)).unwrap();
let server = Server::new(Some(&args), Some(runtime.handle())).unwrap();
runtime
.block_on(async {
tuwunel::async_start(&server).await?;
let run = tuwunel::async_run(&server);
c.bench_function("smoke", |c| {
c.iter(|| {});
});
server.server.shutdown().log_err(Level::WARN).ok();
run.await?;
tuwunel::async_stop(&server).await
})
.unwrap();
tuwunel::shutdown(&server.server, runtime).unwrap();
}

View File

@@ -9,10 +9,9 @@ pub mod signals;
use std::sync::Arc;
use tuwunel_core::{
Result, Runtime, debug_info, error, mod_ctor, mod_dtor, runtime::shutdown,
rustc_flags_capture,
};
pub use tuwunel_core::runtime::shutdown;
use tuwunel_core::{Result, Runtime, debug_info, error, mod_ctor, mod_dtor, rustc_flags_capture};
use tuwunel_service::Services;
pub use self::server::Server;
@@ -26,28 +25,55 @@ pub fn exec(server: &Arc<Server>, runtime: Runtime) -> Result {
}
pub fn run(server: &Arc<Server>, runtime: &Runtime) -> Result {
runtime.spawn(signals::enable(server.clone()));
runtime.block_on(run_async(server))
runtime.block_on(async_exec(server))
}
/// Operate the server normally in release-mode static builds. This will start,
/// run and stop the server within the asynchronous runtime.
#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
#[tracing::instrument(
name = "main",
parent = None,
skip_all
)]
pub async fn run_async(server: &Arc<Server>) -> Result {
pub async fn async_exec(server: &Arc<Server>) -> Result {
let signals = server
.server
.runtime()
.spawn(signals::enable(server.clone()));
async_start(server).await?;
async_run(server).await?;
async_stop(server).await?;
signals.await?;
debug_info!("Exit runtime");
Ok(())
}
#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
pub async fn async_start(server: &Arc<Server>) -> Result<Arc<Services>> {
extern crate tuwunel_router as router;
match router::start(&server.server).await {
| Ok(services) => server.services.lock().await.insert(services),
Ok(match router::start(&server.server).await {
| Ok(services) => server
.services
.lock()
.await
.insert(services)
.clone(),
| Err(error) => {
error!("Critical error starting server: {error}");
return Err(error);
},
};
})
}
/// Operate the server normally in release-mode static builds. This will start,
/// run and stop the server within the asynchronous runtime.
#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
pub async fn async_run(server: &Arc<Server>) -> Result {
extern crate tuwunel_router as router;
if let Err(error) = router::run(
server
@@ -63,6 +89,13 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
return Err(error);
}
Ok(())
}
#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
pub async fn async_stop(server: &Arc<Server>) -> Result {
extern crate tuwunel_router as router;
if let Err(error) = router::stop(
server
.services
@@ -77,7 +110,6 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
return Err(error);
}
debug_info!("Exit runtime");
Ok(())
}
@@ -85,7 +117,7 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
/// and hot-reload portions of the server as-needed before returning for an
/// actual shutdown. This is not available in release-mode or static builds.
#[cfg(all(tuwunel_mods, feature = "tuwunel_mods"))]
pub async fn run_async(server: &Arc<Server>) -> Result {
pub async fn async_exec(server: &Arc<Server>) -> Result {
let mut starts = true;
let mut reloads = true;
while reloads {

View File

@@ -1,5 +1,6 @@
#![cfg(test)]
use insta::{assert_debug_snapshot, with_settings};
use tuwunel::Server;
use tuwunel_core::{Args, Result, runtime};
@@ -12,9 +13,16 @@ fn panic_dummy() { panic!("dummy") }
#[test]
fn smoke() -> Result {
let args = Args::default_test("smoke");
let runtime = runtime::new(Some(&args))?;
let server = Server::new(Some(&args), Some(runtime.handle()))?;
with_settings!({
description => "Smoke Test",
snapshot_suffix => "smoke_test",
}, {
let args = Args::default_test(&["smoke", "fresh", "cleanup"]);
let runtime = runtime::new(Some(&args))?;
let server = Server::new(Some(&args), Some(runtime.handle()))?;
let result = tuwunel::exec(&server, runtime);
tuwunel::exec(&server, runtime)
assert_debug_snapshot!(result);
result
})
}

View File

@@ -0,0 +1,24 @@
#![cfg(test)]
use insta::{assert_debug_snapshot, with_settings};
use tuwunel::Server;
use tuwunel_core::{Args, Result, runtime};
#[test]
fn smoke_async() -> Result {
with_settings!({
description => "Smoke Async",
snapshot_suffix => "smoke_async",
}, {
let args = Args::default_test(&["smoke", "fresh", "cleanup"]);
let runtime = runtime::new(Some(&args))?;
let server = Server::new(Some(&args), Some(runtime.handle()))?;
let result = runtime.block_on(async {
tuwunel::async_exec(&server).await
});
runtime::shutdown(&server.server, runtime)?;
assert_debug_snapshot!(result);
result
})
}

View File

@@ -0,0 +1,29 @@
#![cfg(test)]
use insta::{assert_debug_snapshot, with_settings};
use tracing::Level;
use tuwunel::Server;
use tuwunel_core::{Args, Result, runtime, utils::result::ErrLog};
#[test]
fn smoke_shutdown() -> Result {
with_settings!({
description => "Smoke Shutdown",
snapshot_suffix => "smoke_shutdown",
}, {
let args = Args::default_test(&["fresh", "cleanup"]);
let runtime = runtime::new(Some(&args))?;
let server = Server::new(Some(&args), Some(runtime.handle()))?;
let result = runtime.block_on(async {
tuwunel::async_start(&server).await?;
let run = tuwunel::async_run(&server);
server.server.shutdown().log_err(Level::WARN).ok();
run.await?;
tuwunel::async_stop(&server).await
});
runtime::shutdown(&server.server, runtime)?;
assert_debug_snapshot!(result);
result
})
}

View File

@@ -0,0 +1,8 @@
---
source: src/main/tests/smoke.rs
description: Smoke Test
expression: result
---
Ok(
(),
)

View File

@@ -0,0 +1,8 @@
---
source: src/main/tests/smoke_async.rs
description: Smoke Async
expression: result
---
Ok(
(),
)

View File

@@ -0,0 +1,8 @@
---
source: src/main/tests/smoke_shutdown.rs
description: Smoke Shutdown
expression: result
---
Ok(
(),
)