Use insta and criterion for main integration test and benches respectively.

docker/ci: Separate integration and unit tests and benches jobs.

Add directives to remove db before/after integration tests are performed.

Split start/run/stop phases; add more granular smoketests.

Split main integration tests into units for isolation.

Signed-off-by: Jason Volk <jason@zemos.net>

src/core/Cargo.toml

@@ -114,10 +114,10 @@ url.workspace = true
 nix.workspace = true
 
 [dev-dependencies]
+criterion.workspace = true
 insta.workspace = true
 maplit.workspace = true
 similar.workspace = true
-criterion.workspace = true
 
 [lints]
 workspace = true

src/core/args.rs

@@ -45,10 +45,29 @@ pub struct Args {
 	#[arg(long)]
 	pub execute: Vec<String>,
 
-	/// Set functional testing modes if available. Ex '--test=smoke'
-	#[arg(long, hide(true))]
+	/// Set functional testing modes if available. Ex '--test=smoke'. Empty
+	/// values are permitted for compatibility with testing and benchmarking
+	/// frameworks which may simply pass `--test` to the same execution.
+	#[arg(
+		long,
+		hide(true),
+		num_args = 0..=1,
+		require_equals(false),
+		default_missing_value = "",
+	)]
 	pub test: Vec<String>,
 
+	/// Compatibility option for benchmark frameworks which pass `--bench` to
+	/// the same execution and must be silently accepted without error.
+	#[arg(
+		long,
+		hide(true),
+		num_args = 0..=1,
+		require_equals(false),
+		default_missing_value = "",
+	)]
+	pub bench: Vec<String>,
+
 	/// Override the tokio worker_thread count.
 	#[arg(
 		long,
@@ -151,12 +170,12 @@ pub struct Args {
 
 impl Args {
 	#[must_use]
-	pub fn default_test(name: &str) -> Self {
+	pub fn default_test(name: &[&str]) -> Self {
 		let mut args = Self::default();
-		args.test.push(name.into());
+		args.test
+			.extend(name.iter().copied().map(ToOwned::to_owned));
 		args.option
 			.push("server_name=\"localhost\"".into());
-
 		args
 	}
 }

src/database/Cargo.toml

@@ -17,10 +17,6 @@ crate-type = [
 #	"dylib",
 ]
 
-[[bench]]
-name = "ser"
-harness = false
-
 [features]
 bzip2_compression = [
 	"rust-rocksdb/bzip2",
@@ -76,3 +72,7 @@ criterion.workspace = true
 
 [lints]
 workspace = true
+
+[[bench]]
+name = "ser"
+harness = false

src/database/benches/ser.rs

@@ -1,6 +1,6 @@
 use criterion::{Criterion, criterion_group, criterion_main};
 
-criterion_group!(benches, ser_str,);
+criterion_group!(benches, ser_str);
 
 criterion_main!(benches);
 

src/database/engine/context.rs

@@ -1,10 +1,15 @@
 use std::{
 	collections::BTreeMap,
+	fs::remove_dir_all,
+	path::Path,
 	sync::{Arc, Mutex},
 };
 
 use rocksdb::{Cache, Env, LruCacheOptions};
-use tuwunel_core::{Result, Server, debug, utils::math::usize_from_f64};
+use tuwunel_core::{
+	Result, Server, debug,
+	utils::{math::usize_from_f64, result::LogErr},
+};
 
 use crate::{or_else, pool::Pool};
 
@@ -78,5 +83,55 @@ impl Drop for Context {
 
 		debug!("Joining background threads...");
 		env.join_all_threads();
+
+		after_close(self, &self.server.config.database_path)
+			.expect("Failed to execute after_close handler");
 	}
 }
+
+/// For unit and integration tests the 'fresh' directive deletes found db.
+pub(super) fn before_open(ctx: &Arc<Context>, path: &Path) -> Result {
+	if ctx.server.config.test.contains("fresh") {
+		match delete_database_for_testing(ctx, path) {
+			| Err(e) if !e.is_not_found() => return Err(e),
+			| _ => (),
+		}
+	}
+
+	Ok(())
+}
+
+/// For unit and integration tests the 'cleanup' directive deletes after close
+/// to cleanup.
+fn after_close(ctx: &Context, path: &Path) -> Result {
+	if ctx.server.config.test.contains("cleanup") {
+		delete_database_for_testing(ctx, path)
+			.log_err()
+			.ok();
+	}
+
+	Ok(())
+}
+
+/// For unit and integration tests; removes the database directory when called.
+/// To prevent misuse, cfg!(test) must be true for a unit test or the
+/// integration test server is named localhost.
+#[tracing::instrument(level = "debug", skip_all)]
+fn delete_database_for_testing(ctx: &Context, path: &Path) -> Result {
+	let config = &ctx.server.config;
+	let localhost = config
+		.server_name
+		.as_str()
+		.starts_with("localhost");
+
+	if !cfg!(test) && !localhost {
+		return Ok(());
+	}
+
+	debug_assert!(
+		config.test.contains("cleanup") | config.test.contains("fresh"),
+		"missing any test directive legitimating this call.",
+	);
+
+	remove_dir_all(path).map_err(Into::into)
+}

src/database/engine/open.rs

@@ -10,6 +10,7 @@ use tuwunel_core::{Result, debug, implement, info, warn};
 use super::{
 	Db, Engine,
 	cf_opts::cf_options,
+	context,
 	db_opts::db_options,
 	descriptor::{self, Descriptor},
 	repair::repair,
@@ -23,6 +24,7 @@ pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<S
 	let config = &server.config;
 	let path = &config.database_path;
 
+	context::before_open(&ctx, path)?;
 	let db_opts = db_options(
 		config,
 		&ctx.env.lock().expect("environment locked"),

src/macros/Cargo.toml

@@ -13,6 +13,7 @@ version.workspace = true
 name = "tuwunel_macros"
 path = "mod.rs"
 proc-macro = true
+bench = false
 
 [dependencies]
 syn.workspace = true

src/main/Cargo.toml

@@ -51,11 +51,6 @@ assets = [
 name = "tuwunel"
 pkgdesc = "High performance Matrix homeserver written in Rust"
 
-[lib]
-path = "lib.rs"
-bench = false
-crate-type = ["rlib"]
-
 [features]
 default = [
 	"brotli_compression",
@@ -239,10 +234,25 @@ tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
 
+[dev-dependencies]
+criterion.workspace = true
+insta.workspace = true
+maplit.workspace = true
+similar.workspace = true
+
 [lints]
 workspace = true
 
+[lib]
+path = "lib.rs"
+bench = false
+crate-type = ["rlib"]
+
 [[bin]]
 name = "tuwunel"
 path = "main.rs"
 bench = false
+
+[[bench]]
+name = "main"
+harness = false

src/main/benches/main.rs

@@ -0,0 +1,35 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use tracing::Level;
+use tuwunel::Server;
+use tuwunel_core::{Args, result::ErrLog, runtime};
+
+criterion_group!(
+	name = benches;
+	config = Criterion::default().sample_size(10).nresamples(1);
+	targets = dummy, smoke
+);
+
+criterion_main!(benches);
+
+fn dummy(c: &mut Criterion) { c.bench_function("dummy", |c| c.iter(|| {})); }
+
+fn smoke(c: &mut Criterion) {
+	let args = Args::default_test(&["fresh", "cleanup"]);
+	let runtime = runtime::new(Some(&args)).unwrap();
+	let server = Server::new(Some(&args), Some(runtime.handle())).unwrap();
+	runtime
+		.block_on(async {
+			tuwunel::async_start(&server).await?;
+			let run = tuwunel::async_run(&server);
+			c.bench_function("smoke", |c| {
+				c.iter(|| {});
+			});
+
+			server.server.shutdown().log_err(Level::WARN).ok();
+			run.await?;
+			tuwunel::async_stop(&server).await
+		})
+		.unwrap();
+
+	tuwunel::shutdown(&server.server, runtime).unwrap();
+}

src/main/lib.rs

@@ -9,10 +9,9 @@ pub mod signals;
 
 use std::sync::Arc;
 
-use tuwunel_core::{
-	Result, Runtime, debug_info, error, mod_ctor, mod_dtor, runtime::shutdown,
-	rustc_flags_capture,
-};
+pub use tuwunel_core::runtime::shutdown;
+use tuwunel_core::{Result, Runtime, debug_info, error, mod_ctor, mod_dtor, rustc_flags_capture};
+use tuwunel_service::Services;
 
 pub use self::server::Server;
 
@@ -26,28 +25,55 @@ pub fn exec(server: &Arc<Server>, runtime: Runtime) -> Result {
 }
 
 pub fn run(server: &Arc<Server>, runtime: &Runtime) -> Result {
-	runtime.spawn(signals::enable(server.clone()));
-	runtime.block_on(run_async(server))
+	runtime.block_on(async_exec(server))
 }
 
 /// Operate the server normally in release-mode static builds. This will start,
 /// run and stop the server within the asynchronous runtime.
-#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
 #[tracing::instrument(
     name = "main",
     parent = None,
     skip_all
 )]
-pub async fn run_async(server: &Arc<Server>) -> Result {
+pub async fn async_exec(server: &Arc<Server>) -> Result {
+	let signals = server
+		.server
+		.runtime()
+		.spawn(signals::enable(server.clone()));
+
+	async_start(server).await?;
+	async_run(server).await?;
+	async_stop(server).await?;
+	signals.await?;
+
+	debug_info!("Exit runtime");
+	Ok(())
+}
+
+#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
+pub async fn async_start(server: &Arc<Server>) -> Result<Arc<Services>> {
 	extern crate tuwunel_router as router;
 
-	match router::start(&server.server).await {
-		| Ok(services) => server.services.lock().await.insert(services),
+	Ok(match router::start(&server.server).await {
+		| Ok(services) => server
+			.services
+			.lock()
+			.await
+			.insert(services)
+			.clone(),
+
 		| Err(error) => {
 			error!("Critical error starting server: {error}");
 			return Err(error);
 		},
-	};
+	})
+}
+
+/// Operate the server normally in release-mode static builds. This will start,
+/// run and stop the server within the asynchronous runtime.
+#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
+pub async fn async_run(server: &Arc<Server>) -> Result {
+	extern crate tuwunel_router as router;
 
 	if let Err(error) = router::run(
 		server
@@ -63,6 +89,13 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
 		return Err(error);
 	}
 
+	Ok(())
+}
+
+#[cfg(any(not(tuwunel_mods), not(feature = "tuwunel_mods")))]
+pub async fn async_stop(server: &Arc<Server>) -> Result {
+	extern crate tuwunel_router as router;
+
 	if let Err(error) = router::stop(
 		server
 			.services
@@ -77,7 +110,6 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
 		return Err(error);
 	}
 
-	debug_info!("Exit runtime");
 	Ok(())
 }
 
@@ -85,7 +117,7 @@ pub async fn run_async(server: &Arc<Server>) -> Result {
 /// and hot-reload portions of the server as-needed before returning for an
 /// actual shutdown. This is not available in release-mode or static builds.
 #[cfg(all(tuwunel_mods, feature = "tuwunel_mods"))]
-pub async fn run_async(server: &Arc<Server>) -> Result {
+pub async fn async_exec(server: &Arc<Server>) -> Result {
 	let mut starts = true;
 	let mut reloads = true;
 	while reloads {

src/main/tests/smoke.rs

@@ -1,5 +1,6 @@
 #![cfg(test)]
 
+use insta::{assert_debug_snapshot, with_settings};
 use tuwunel::Server;
 use tuwunel_core::{Args, Result, runtime};
 
@@ -12,9 +13,16 @@ fn panic_dummy() { panic!("dummy") }
 
 #[test]
 fn smoke() -> Result {
-	let args = Args::default_test("smoke");
-	let runtime = runtime::new(Some(&args))?;
-	let server = Server::new(Some(&args), Some(runtime.handle()))?;
+	with_settings!({
+		description => "Smoke Test",
+		snapshot_suffix => "smoke_test",
+	}, {
+		let args = Args::default_test(&["smoke", "fresh", "cleanup"]);
+		let runtime = runtime::new(Some(&args))?;
+		let server = Server::new(Some(&args), Some(runtime.handle()))?;
+		let result = tuwunel::exec(&server, runtime);
 
-	tuwunel::exec(&server, runtime)
+		assert_debug_snapshot!(result);
+		result
+	})
 }

src/main/tests/smoke_async.rs

@@ -0,0 +1,24 @@
+#![cfg(test)]
+
+use insta::{assert_debug_snapshot, with_settings};
+use tuwunel::Server;
+use tuwunel_core::{Args, Result, runtime};
+
+#[test]
+fn smoke_async() -> Result {
+	with_settings!({
+		description => "Smoke Async",
+		snapshot_suffix => "smoke_async",
+	}, {
+		let args = Args::default_test(&["smoke", "fresh", "cleanup"]);
+		let runtime = runtime::new(Some(&args))?;
+		let server = Server::new(Some(&args), Some(runtime.handle()))?;
+		let result = runtime.block_on(async {
+			tuwunel::async_exec(&server).await
+		});
+
+		runtime::shutdown(&server.server, runtime)?;
+		assert_debug_snapshot!(result);
+		result
+	})
+}

src/main/tests/smoke_shutdown.rs

@@ -0,0 +1,29 @@
+#![cfg(test)]
+
+use insta::{assert_debug_snapshot, with_settings};
+use tracing::Level;
+use tuwunel::Server;
+use tuwunel_core::{Args, Result, runtime, utils::result::ErrLog};
+
+#[test]
+fn smoke_shutdown() -> Result {
+	with_settings!({
+		description => "Smoke Shutdown",
+		snapshot_suffix => "smoke_shutdown",
+	}, {
+		let args = Args::default_test(&["fresh", "cleanup"]);
+		let runtime = runtime::new(Some(&args))?;
+		let server = Server::new(Some(&args), Some(runtime.handle()))?;
+		let result = runtime.block_on(async {
+			tuwunel::async_start(&server).await?;
+			let run = tuwunel::async_run(&server);
+			server.server.shutdown().log_err(Level::WARN).ok();
+			run.await?;
+			tuwunel::async_stop(&server).await
+		});
+
+		runtime::shutdown(&server.server, runtime)?;
+		assert_debug_snapshot!(result);
+		result
+	})
+}

src/main/tests/snapshots/smoke__smoke@smoke_test.snap

@@ -0,0 +1,8 @@
+---
+source: src/main/tests/smoke.rs
+description: Smoke Test
+expression: result
+---
+Ok(
+    (),
+)

src/main/tests/snapshots/smoke_async__smoke_async@smoke_async.snap

@@ -0,0 +1,8 @@
+---
+source: src/main/tests/smoke_async.rs
+description: Smoke Async
+expression: result
+---
+Ok(
+    (),
+)

src/main/tests/snapshots/smoke_shutdown__smoke_shutdown@smoke_shutdown.snap

@@ -0,0 +1,8 @@
+---
+source: src/main/tests/smoke_shutdown.rs
+description: Smoke Shutdown
+expression: result
+---
+Ok(
+    (),
+)