From 578cf9d2d74267d576e9c9226032f664648d2b45 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Jun 2025 21:23:55 +0000 Subject: [PATCH] Fix missing lz4 and bz2; tweak feature projection. (closes #78) Signed-off-by: Jason Volk --- Cargo.toml | 4 ++-- docker/bake.hcl | 39 ++++++++++++++++++++++++++++++++------- src/admin/Cargo.toml | 10 ++++++++++ src/api/Cargo.toml | 6 ++++++ src/database/Cargo.toml | 6 ++++++ src/main/Cargo.toml | 36 +++++++++++++++++++++++++++++++++--- src/router/Cargo.toml | 11 ++++++++++- src/service/Cargo.toml | 6 ++++++ 8 files changed, 105 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 276d928b..33f6532a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -362,11 +362,11 @@ git = "https://github.com/matrix-construct/rust-rocksdb" rev = "225a42519276e502205bdc845cebdb22d70ee245" default-features = false features = [ + "bzip2", + "lz4", "multi-threaded-cf", "mt_static", - "lz4", "zstd", - "bzip2", ] [workspace.dependencies.sanitize-filename] diff --git a/docker/bake.hcl b/docker/bake.hcl index 9b5bb264..10e96ae3 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -33,7 +33,7 @@ variable "git_ref_name" { cargo_feat_sets = { none = "" default = "brotli_compression,element_hacks,gzip_compression,io_uring,jemalloc,jemalloc_conf,media_thumbnail,release_max_log_level,systemd,url_preview,zstd_compression" - all = "blurhashing,brotli_compression,tuwunel_mods,console,default,direct_tls,element_hacks,gzip_compression,hardened_malloc,io_uring,jemalloc,jemalloc_conf,jemalloc_prof,jemalloc_stats,ldap,media_thumbnail,perf_measurements,release_max_log_level,sentry_telemetry,systemd,tokio_console,url_preview,zstd_compression" + all = "blurhashing,brotli_compression,bzip2_compression,tuwunel_mods,console,default,direct_tls,element_hacks,gzip_compression,hardened_malloc,io_uring,jemalloc,jemalloc_conf,jemalloc_prof,jemalloc_stats,ldap,lz4_compression,media_thumbnail,perf_measurements,release_max_log_level,sentry_telemetry,systemd,tokio_console,url_preview,zstd_compression" } variable "cargo_features_always" { default = "direct_tls" @@ -1204,6 +1204,10 @@ target "deps-base" { join(" ", static_nightly_rustflags), join(" ", native_rustflags), "-C link-arg=-L/usr/lib/gcc/${sys_target}/14", #FIXME + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? + "-C link-arg=-l:libbz2.a": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? + "-C link-arg=-l:liblz4.a": "", contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "-C link-arg=-l:libzstd.a": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? @@ -1220,6 +1224,10 @@ target "deps-base" { join(" ", static_rustflags), join(" ", static_nightly_rustflags), "-C link-arg=-L/usr/lib/gcc/${sys_target}/14", #FIXME + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? + "-C link-arg=-l:libbz2.a": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? + "-C link-arg=-l:liblz4.a": "", contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "-C link-arg=-l:libzstd.a": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? @@ -1234,6 +1242,10 @@ target "deps-base" { join(" ", rustflags), join(" ", static_rustflags), "-C link-arg=-L/usr/lib/gcc/${sys_target}/14", #FIXME + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? + "-C link-arg=-l:libbz2.a": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? + "-C link-arg=-l:liblz4.a": "", contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "-C link-arg=-l:libzstd.a": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? @@ -1248,6 +1260,10 @@ target "deps-base" { join(" ", rustflags), join(" ", static_rustflags), "-C link-arg=-L/usr/lib/gcc/${sys_target}/14", #FIXME + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? + "-C link-arg=-l:libbz2.a": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? + "-C link-arg=-l:liblz4.a": "", contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "-C link-arg=-l:libzstd.a": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? @@ -1262,6 +1278,10 @@ target "deps-base" { join(" ", rustflags), join(" ", nightly_rustflags), join(" ", dynamic_rustflags), + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? + "-C link-arg=-lbz2": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? + "-C link-arg=-llz4": "", contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "-C link-arg=-lzstd": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? @@ -1311,6 +1331,8 @@ target "rocksdb-build" { input = elem("target:kitchen", [feat_set, sys_name, sys_version, sys_target]) } args = { + rocksdb_bz2 = contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? 1: 0 + rocksdb_lz4 = contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? 1: 0 rocksdb_zstd = contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? 1: 0 rocksdb_jemalloc = contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? 1: 0 rocksdb_iouring = contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? 1: 0 @@ -1544,6 +1566,7 @@ kitchen_packages = [ "cmake", "curl", "git", + "gzip", "libc6-dev", "libssl-dev", "make", @@ -1567,10 +1590,12 @@ target "kitchen" { } args = { packages = join(" ", [ - contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? "liburing-dev": "", - contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "libzstd-dev": "", - contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? "libjemalloc-dev": "", + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? "libbz2-dev": "", contains(split(",", cargo_feat_sets[feat_set]), "hardened_malloc")? "g++": "", + contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? "liburing-dev": "", + contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? "libjemalloc-dev": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? "liblz4-dev": "", + contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "libzstd-dev": "", ]) } } @@ -1670,18 +1695,18 @@ target "runtime" { } args = { packages = join(" ", [ + contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")? "bzip2": "", contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? "liburing2": "", - contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "libzstd1": "", contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? "libjemalloc2": "", + contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")? "liblz4-1": "", + contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? "libzstd1": "", ]) } } base_pkgs = [ "adduser", - "bzip2", "ca-certificates", - "gzip", ] target "base" { diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 0a28a02b..04178604 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -22,6 +22,11 @@ brotli_compression = [ "tuwunel-core/brotli_compression", "tuwunel-service/brotli_compression", ] +bzip2_compression = [ + "tuwunel-api/bzip2_compression", + "tuwunel-database/bzip2_compression", + "tuwunel-service/bzip2_compression", +] gzip_compression = [ "tuwunel-api/gzip_compression", "tuwunel-core/gzip_compression", @@ -56,6 +61,11 @@ jemalloc_stats = [ "tuwunel-database/jemalloc_stats", "tuwunel-service/jemalloc_stats", ] +lz4_compression = [ + "tuwunel-api/lz4_compression", + "tuwunel-database/lz4_compression", + "tuwunel-service/lz4_compression", +] release_max_log_level = [ "tuwunel-api/release_max_log_level", "tuwunel-core/release_max_log_level", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index ca8bfcce..a469bcc8 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -22,6 +22,9 @@ brotli_compression = [ "tuwunel-service/brotli_compression", "reqwest/brotli", ] +bzip2_compression = [ + "tuwunel-service/bzip2_compression", +] element_hacks = [ "tuwunel-service/element_hacks", ] @@ -52,6 +55,9 @@ jemalloc_stats = [ ldap = [ "tuwunel-service/ldap", ] +lz4_compression = [ + "tuwunel-service/lz4_compression", +] release_max_log_level = [ "tuwunel-core/release_max_log_level", "tuwunel-service/release_max_log_level", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 303997c2..8591b90a 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -17,6 +17,9 @@ crate-type = [ ] [features] +bzip2_compression = [ + "rust-rocksdb/bzip2", +] io_uring = [ "rust-rocksdb/io-uring", ] @@ -33,6 +36,9 @@ jemalloc_prof = [ jemalloc_stats = [ "tuwunel-core/jemalloc_stats", ] +lz4_compression = [ + "rust-rocksdb/lz4", +] release_max_log_level = [ "tuwunel-core/release_max_log_level", "log/max_level_trace", diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 7b88d821..3c3f7084 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -70,11 +70,19 @@ blurhashing = [ "tuwunel-service/blurhashing", ] brotli_compression = [ + "tuwunel-admin/brotli_compression", "tuwunel-api/brotli_compression", "tuwunel-core/brotli_compression", "tuwunel-router/brotli_compression", "tuwunel-service/brotli_compression", ] +bzip2_compression = [ + "tuwunel-admin/bzip2_compression", + "tuwunel-api/bzip2_compression", + "tuwunel-database/bzip2_compression", + "tuwunel-router/bzip2_compression", + "tuwunel-service/bzip2_compression", +] console = [ "tuwunel-service/console", ] @@ -86,6 +94,7 @@ element_hacks = [ "tuwunel-service/element_hacks", ] gzip_compression = [ + "tuwunel-admin/gzip_compression", "tuwunel-api/gzip_compression", "tuwunel-core/gzip_compression", "tuwunel-router/gzip_compression", @@ -95,11 +104,27 @@ hardened_malloc = [ "tuwunel-core/hardened_malloc", ] io_uring = [ + "tuwunel-admin/io_uring", + "tuwunel-api/io_uring", "tuwunel-database/io_uring", + "tuwunel-router/io_uring", + "tuwunel-service/io_uring", ] jemalloc = [ + "tuwunel-admin/jemalloc", + "tuwunel-api/jemalloc", "tuwunel-core/jemalloc", "tuwunel-database/jemalloc", + "tuwunel-router/jemalloc", + "tuwunel-service/jemalloc", +] +jemalloc_conf = [ + "tuwunel-admin/jemalloc_conf", + "tuwunel-api/jemalloc_conf", + "tuwunel-core/jemalloc_conf", + "tuwunel-database/jemalloc_conf", + "tuwunel-router/jemalloc_conf", + "tuwunel-service/jemalloc_conf", ] jemalloc_prof = [ "tuwunel-core/jemalloc_prof", @@ -107,13 +132,17 @@ jemalloc_prof = [ jemalloc_stats = [ "tuwunel-core/jemalloc_stats", ] -jemalloc_conf = [ - "tuwunel-core/jemalloc_conf", -] ldap = [ "tuwunel-api/ldap", "tuwunel-service/ldap", ] +lz4_compression = [ + "tuwunel-admin/lz4_compression", + "tuwunel-api/lz4_compression", + "tuwunel-database/lz4_compression", + "tuwunel-router/lz4_compression", + "tuwunel-service/lz4_compression", +] media_thumbnail = [ "tuwunel-service/media_thumbnail", ] @@ -159,6 +188,7 @@ url_preview = [ "tuwunel-service/url_preview", ] zstd_compression = [ + "tuwunel-admin/zstd_compression", "tuwunel-api/zstd_compression", "tuwunel-core/zstd_compression", "tuwunel-database/zstd_compression", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index be4e83c5..b4613ff9 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -24,6 +24,11 @@ brotli_compression = [ "tuwunel-service/brotli_compression", "tower-http/compression-br", ] +bzip2_compression = [ + "tuwunel-admin/bzip2_compression", + "tuwunel-api/bzip2_compression", + "tuwunel-service/bzip2_compression", +] direct_tls = [ "axum-server/tls-rustls", "dep:rustls", @@ -40,7 +45,6 @@ io_uring = [ "tuwunel-admin/io_uring", "tuwunel-api/io_uring", "tuwunel-service/io_uring", - "tuwunel-api/io_uring", ] jemalloc = [ "tuwunel-admin/jemalloc", @@ -66,6 +70,11 @@ jemalloc_stats = [ "tuwunel-core/jemalloc_stats", "tuwunel-service/jemalloc_stats", ] +lz4_compression = [ + "tuwunel-admin/lz4_compression", + "tuwunel-api/lz4_compression", + "tuwunel-service/lz4_compression", +] release_max_log_level = [ "tuwunel-admin/release_max_log_level", "tuwunel-api/release_max_log_level", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index c27839fc..884f9095 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -25,6 +25,9 @@ brotli_compression = [ "tuwunel-core/brotli_compression", "reqwest/brotli", ] +bzip2_compression = [ + "tuwunel-database/bzip2_compression", +] console = [ "dep:rustyline-async", "dep:termimad", @@ -56,6 +59,9 @@ jemalloc_stats = [ ldap = [ "dep:ldap3", ] +lz4_compression = [ + "tuwunel-database/lz4_compression", +] media_thumbnail = [ "dep:image", ]