Switch to Criterion for benchtests.

ci: Add benches to workflow. Signed-off-by: Jason Volk <jason@zemos.net>
2025-09-24 04:18:13 +00:00
parent a47f8f8a82
commit 89be6dc097
17 changed files with 334 additions and 166 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -135,6 +135,7 @@ jobs:
          {"cargo_profile": "test", "feat_set": "logging"},
          {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"},
          {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"},
+          {"cargo_profile": "bench"},
          {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "none"},
          {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "default"},
          {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "logging"},
@@ -219,7 +220,7 @@ jobs:
      && needs.init.outputs.package

    name: Package
-    needs: [init, lint]
+    needs: [init, test]
    uses: ./.github/workflows/package.yml
    with:
      release_url: ${{needs.init.outputs.release_url}}
@@ -238,6 +239,7 @@ jobs:
        [
          {"feat_set": "none"},
          {"cargo_profile": "test"},
+          {"cargo_profile": "bench"},
          {"cargo_profile": "release-native"},
          {"cargo_profile": "release-debuginfo", "feat_set": "default"},
          {"cargo_profile": "release-debuginfo", "feat_set": "logging"},
@@ -293,6 +295,7 @@ jobs:
          {"feat_set": "none"},
          {"feat_set": "logging"},
          {"cargo_profile": "test"},
+          {"cargo_profile": "bench"},
          {"cargo_profile": "release-debuginfo"},
          {"cargo_profile": "release-native"},
          {"cargo_profile": "release", "rust_toolchain": "nightly"},
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -95,6 +95,28 @@ jobs:
      excludes: ${{inputs.excludes}}
      includes: ${{inputs.includes}}

+  bench:
+    if: >
+      contains(fromJSON(inputs.cargo_profiles), fromJSON('["bench"]')[0])
+      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v3-linux-gnu"]')[0])
+
+    name: Bench
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["unit"]'
+      cargo_profiles: '["bench"]'
+      feat_sets: '["all"]'
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: ${{inputs.rust_targets}}
+      sys_targets: '["x86_64-v3-linux-gnu"]'
+      machines: ${{inputs.machines}}
+      excludes: ${{inputs.excludes}}
+      includes: ${{inputs.includes}}
+
  smoke:
    if: >
      !failure() && !cancelled()
@@ -123,6 +145,7 @@ jobs:
          {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"},
          {"cargo_profile": "test", "sys_target": "x86_64-v2-linux-gnu"},
          {"cargo_profile": "test", "sys_target": "x86_64-v3-linux-gnu"},
+          {"cargo_profile": "bench"},
          {"cargo_profile": "release", "rust_toolchain": "nightly"},
          {"cargo_profile": "release", "rust_toolchain": "stable", "feat_set": "none"},
          {"cargo_profile": "release", "bake_target": "smoke-valgrind"},
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -50,6 +50,12 @@ dependencies = [
 "alloc-no-stdlib",
 ]

+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstyle"
 version = "1.0.11"
@@ -575,6 +581,12 @@ dependencies = [
 "toml 0.9.7",
 ]

+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.2.38"
@@ -636,6 +648,33 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "clang-sys"
 version = "1.8.1"
@@ -824,6 +863,39 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "criterion"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "futures",
+ "itertools 0.13.0",
+ "num-traits",
+ "oorandom",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "tokio",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338"
+dependencies = [
+ "cast",
+ "itertools 0.13.0",
+]
+
 [[package]]
 name = "critical-section"
 version = "1.2.0"
@@ -2723,6 +2795,12 @@ dependencies = [
 "portable-atomic",
 ]

+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.6"
@@ -3769,6 +3847,15 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"

+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "sanitize-filename"
 version = "0.6.0"
@@ -4528,6 +4615,16 @@ dependencies = [
 "zerovec",
 ]

+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.10.0"
@@ -4959,6 +5056,7 @@ dependencies = [
 "clap",
 "const-str",
 "core_affinity",
+ "criterion",
 "ctor",
 "cyborgtime",
 "either",
@@ -5009,6 +5107,7 @@ version = "1.4.2"
 dependencies = [
 "async-channel",
 "const-str",
+ "criterion",
 "ctor",
 "futures",
 "log",
@@ -5286,6 +5385,16 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"

+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -5466,6 +5575,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"

+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.0",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -142,6 +142,15 @@ version = "0.8.3"
 [workspace.dependencies.const-str]
 version = "0.7"

+[workspace.dependencies.criterion]
+version = "0.7"
+default-features = false
+features = [
+	"cargo_bench_support",
+	"async_futures",
+	"async_tokio",
+]
+
 [workspace.dependencies.ctor]
 version = "0.5"

--- a/docker/bake.hcl
+++ b/docker/bake.hcl
@@ -184,7 +184,6 @@ dynamic_libs = [

 nightly_rustflags = [
    "--cfg tokio_unstable",
-    "--cfg tuwunel_bench",
    "--allow=unstable-features",
    "-Z crate-attr=feature(test)",
    "-Z enforce-type-length-limit",
@@ -892,7 +891,7 @@ target "unit" {
    args = {
        cargo_cmd = (cargo_profile == "bench"? "bench": "test")
        cargo_args = (rust_toolchain == "nightly"?
-            "--no-fail-fast --all-targets": "--no-fail-fast --bins --tests"
+            "--no-fail-fast --all-targets -- --color=always": "--no-fail-fast --bins --tests"
        )
    }
 }
@@ -1294,7 +1293,7 @@ target "deps-base" {
        CARGO_PROFILE_TEST_DEBUG = "false"
        CARGO_PROFILE_TEST_INCREMENTAL = "false"
        CARGO_PROFILE_BENCH_DEBUG = "limited"
-        CARGO_PROFILE_BENCH_LTO = "false"
+        CARGO_PROFILE_BENCH_LTO = "thin"
        CARGO_PROFILE_RELEASE_LTO = "thin"
        CARGO_PROFILE_RELEASE_DEBUGINFO_DEBUG = "limited"
        CARGO_PROFILE_RELEASE_DEBUGINFO_LTO = "off"
--- a/src/admin/Cargo.toml
+++ b/src/admin/Cargo.toml
@@ -11,6 +11,7 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -11,6 +11,7 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",
--- a/src/core/Cargo.toml
+++ b/src/core/Cargo.toml
@@ -11,11 +11,16 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",
 ]

+[[bench]]
+name = "state_res"
+harness = false
+
 [features]
 brotli_compression = [
 	"reqwest/brotli",
@@ -115,6 +120,7 @@ nix.workspace = true
 insta.workspace = true
 maplit.workspace = true
 similar.workspace = true
+criterion.workspace = true

 [lints]
 workspace = true
--- a/src/core/matrix/state_res/benches.rs
+++ b/src/core/matrix/state_res/benches.rs
@@ -1,14 +1,10 @@
-#![cfg_attr(not(tuwunel_bench), allow(unused_imports, dead_code))]
-
-#[cfg(tuwunel_bench)]
-extern crate test;
-
 use std::{
 	borrow::Borrow,
 	collections::HashMap,
 	sync::atomic::{AtomicU64, Ordering::SeqCst},
 };

+use criterion::{Criterion, async_executor::FuturesExecutor, criterion_group, criterion_main};
 use ruma::{
 	EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, UserId,
 	events::{
@@ -24,25 +20,31 @@ use serde_json::{
 	json,
 	value::{RawValue as RawJsonValue, to_raw_value as to_raw_json_value},
 };
-
-use super::{AuthSet, StateMap, test_utils::not_found};
-use crate::{
-	Result,
-	matrix::{Event, EventHash, PduEvent, event::TypeExt},
+use tuwunel_core::{
+	Result, err,
+	matrix::{
+		Event, EventHash, PduEvent,
+		event::TypeExt,
+		state_res::{AuthSet, StateMap},
+	},
 	utils::stream::IterStream,
 };

+criterion_group!(
+	benches,
+	lexico_topo_sort,
+	resolution_shallow_auth_chain,
+	resolve_deeper_event_set
+);
+
+criterion_main!(benches);
+
 static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0);

-#[cfg(tuwunel_bench)]
-#[cfg_attr(tuwunel_bench, bench)]
-fn lexico_topo_sort(c: &mut test::Bencher) {
+fn lexico_topo_sort(c: &mut Criterion) {
+	c.bench_function("lexico_topo_sort", |c| {
 		use maplit::{hashmap, hashset};

-	let rt = tokio::runtime::Builder::new_current_thread()
-		.build()
-		.unwrap();
-
 		let graph = hashmap! {
 			event_id("l") => hashset![event_id("o")],
 			event_id("m") => hashset![event_id("n"), event_id("o")],
@@ -51,9 +53,8 @@ fn lexico_topo_sort(c: &mut test::Bencher) {
 			event_id("p") => hashset![event_id("o")],
 		};

-	c.iter(move || {
-		rt.block_on(async {
-			_ = super::topological_sort(&graph, &async |_id| {
+		c.to_async(FuturesExecutor).iter(async || {
+			_ = tuwunel_core::matrix::state_res::topological_sort(&graph, &async |_id| {
 				Ok((int!(0).into(), MilliSecondsSinceUnixEpoch(uint!(0))))
 			})
 			.await;
@@ -61,13 +62,8 @@ fn lexico_topo_sort(c: &mut test::Bencher) {
 	});
 }

-#[cfg(tuwunel_bench)]
-#[cfg_attr(tuwunel_bench, bench)]
-fn resolution_shallow_auth_chain(c: &mut test::Bencher) {
-	let rt = tokio::runtime::Builder::new_current_thread()
-		.build()
-		.unwrap();
-
+fn resolution_shallow_auth_chain(c: &mut Criterion) {
+	c.bench_function("resolution_shallow_auth_chain", |c| {
 		let mut store = TestStore(maplit::hashmap! {});

 		// build up the DAG
@@ -86,11 +82,16 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) {
 			.collect::<Vec<_>>();

 		let func = async || {
-		if let Err(e) = super::resolve(
+			if let Err(e) = tuwunel_core::matrix::state_res::resolve(
 				&rules,
 				state_sets.clone().into_iter().stream(),
 				auth_chains.clone().into_iter().stream(),
-			&async |id| ev_map.get(&id).cloned().ok_or_else(not_found),
+				&async |id| {
+					ev_map
+						.get(&id)
+						.cloned()
+						.ok_or(err!(Request(NotFound("Not Found"))))
+				},
 				&async |id| ev_map.contains_key(&id),
 				false,
 			)
@@ -100,20 +101,14 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) {
 			}
 		};

-	c.iter(move || {
-		rt.block_on(async {
+		c.to_async(FuturesExecutor).iter(async || {
 			func().await;
 		});
 	});
 }

-#[cfg(tuwunel_bench)]
-#[cfg_attr(tuwunel_bench, bench)]
-fn resolve_deeper_event_set(c: &mut test::Bencher) {
-	let rt = tokio::runtime::Builder::new_current_thread()
-		.build()
-		.unwrap();
-
+fn resolve_deeper_event_set(c: &mut Criterion) {
+	c.bench_function("resolver_deeper_event_set", |c| {
 		let mut inner = INITIAL_EVENTS();
 		let ban = BAN_STATE_SET();

@@ -170,11 +165,16 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) {
 			.collect::<Vec<_>>();

 		let func = async || {
-		if let Err(e) = super::resolve(
+			if let Err(e) = tuwunel_core::matrix::state_res::resolve(
 				&rules,
 				state_sets.clone().into_iter().stream(),
 				auth_chains.clone().into_iter().stream(),
-			&async |id| inner.get(&id).cloned().ok_or_else(not_found),
+				&async |id| {
+					inner
+						.get(&id)
+						.cloned()
+						.ok_or(err!(Request(NotFound("Not Found"))))
+				},
 				&async |id| inner.contains_key(&id),
 				false,
 			)
@@ -184,8 +184,7 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) {
 			}
 		};

-	c.iter(move || {
-		rt.block_on(async {
+		c.to_async(FuturesExecutor).iter(async || {
 			func().await;
 		});
 	});
@@ -204,7 +203,7 @@ impl<E: Event> TestStore<E> {
 		self.0
 			.get(event_id)
 			.cloned()
-			.ok_or_else(not_found)
+			.ok_or(err!(Request(NotFound("Not Found"))))
 	}

 	/// Returns the events that correspond to the `event_ids` sorted in the same
@@ -458,8 +457,8 @@ where
 		depth: uint!(0),
 		hashes: EventHash::default(),
 		signatures: None,
-		#[cfg(test)]
-		rejected: false,
+		//#[cfg(test)]
+		//rejected: false,
 	}
 }

--- a/src/core/matrix/state_res/mod.rs
+++ b/src/core/matrix/state_res/mod.rs
@@ -58,8 +58,6 @@
 //! [necessary checks on receipt of a PDU]: https://spec.matrix.org/latest/server-server-api/#checks-performed-on-receipt-of-a-pdu
 //! [ruma-events]: https://crates.io/crates/ruma-events

-#[cfg(test)]
-mod benches;
 mod event_auth;
 mod event_format;
 pub mod events;
--- a/src/database/Cargo.toml
+++ b/src/database/Cargo.toml
@@ -11,11 +11,16 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",
 ]

+[[bench]]
+name = "ser"
+harness = false
+
 [features]
 bzip2_compression = [
 	"rust-rocksdb/bzip2",
@@ -66,5 +71,8 @@ tokio.workspace = true
 tracing.workspace = true
 tuwunel-core.workspace = true

+[dev-dependencies]
+criterion.workspace = true
+
 [lints]
 workspace = true
--- a/src/database/benches.rs
+++ b/src/database/benches.rs
@@ -1,17 +0,0 @@
-#[cfg(tuwunel_bench)]
-extern crate test;
-
-#[cfg(tuwunel_bench)]
-#[cfg_attr(tuwunel_bench, bench)]
-fn ser_str(b: &mut test::Bencher) {
-	use tuwunel_core::ruma::{RoomId, UserId};
-
-	use crate::ser::serialize_to_vec;
-
-	let user_id: &UserId = "@user:example.com".try_into().unwrap();
-	let room_id: &RoomId = "!room:example.com".try_into().unwrap();
-	b.iter(|| {
-		let key = (user_id, room_id);
-		let _s = serialize_to_vec(key).expect("failed to serialize user_id");
-	});
-}
--- a/src/database/benches/ser.rs
+++ b/src/database/benches/ser.rs
@@ -0,0 +1,19 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+
+criterion_group!(benches, ser_str,);
+
+criterion_main!(benches);
+
+fn ser_str(b: &mut Criterion) {
+	b.bench_function("ser_str", |c| {
+		use tuwunel_core::ruma::{RoomId, UserId};
+		use tuwunel_database::serialize_to_vec;
+
+		let user_id: &UserId = "@user:example.com".try_into().unwrap();
+		let room_id: &RoomId = "!room:example.com".try_into().unwrap();
+		c.iter(|| {
+			let key = (user_id, room_id);
+			let _s = serialize_to_vec(key).expect("failed to serialize user_id");
+		});
+	});
+}
--- a/src/database/mod.rs
+++ b/src/database/mod.rs
@@ -6,8 +6,6 @@ tuwunel_core::mod_ctor! {}
 tuwunel_core::mod_dtor! {}
 tuwunel_core::rustc_flags_capture! {}

-#[cfg(test)]
-mod benches;
 mod cork;
 mod de;
 mod deserialized;
--- a/src/main/Cargo.toml
+++ b/src/main/Cargo.toml
@@ -239,3 +239,4 @@ workspace = true
 [[bin]]
 name = "tuwunel"
 path = "main.rs"
+bench = false
--- a/src/router/Cargo.toml
+++ b/src/router/Cargo.toml
@@ -11,6 +11,7 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",
--- a/src/service/Cargo.toml
+++ b/src/service/Cargo.toml
@@ -11,6 +11,7 @@ version.workspace = true

 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",