Sienna Meridian Satterwhite
c9cddc80d9
Extract a SearchBackend trait from the existing RocksDB search code and add an OpenSearch implementation supporting cross-room search, relevance ranking, fuzzy matching, English stemming, and optional hybrid neural+BM25 semantic search using sentence-transformers. Fix macOS build by gating RLIMIT_NPROC and getrusage to supported platforms.
112 lines
3.3 KiB
Rust
112 lines
3.3 KiB
Rust
#[cfg(unix)]
|
|
use nix::sys::resource::{Resource, getrlimit};
|
|
|
|
use crate::{Result, apply, debug, utils::math::usize_from_u64_truncated};
|
|
|
|
#[cfg(unix)]
|
|
/// This is needed for opening lots of file descriptors, which tends to
|
|
/// happen more often when using RocksDB and making lots of federation
|
|
/// connections at startup. The soft limit is usually 1024, and the hard
|
|
/// limit is usually 512000; I've personally seen it hit >2000.
|
|
///
|
|
/// * <https://www.freedesktop.org/software/systemd/man/systemd.exec.html#id-1.12.2.1.17.6>
|
|
/// * <https://github.com/systemd/systemd/commit/0abf94923b4a95a7d89bc526efc84e7ca2b71741>
|
|
pub fn maximize_fd_limit() -> Result {
|
|
use nix::sys::resource::setrlimit;
|
|
|
|
let (soft_limit, hard_limit) = max_file_descriptors()?;
|
|
if soft_limit < hard_limit {
|
|
let new_limit = hard_limit.try_into()?;
|
|
setrlimit(Resource::RLIMIT_NOFILE, new_limit, new_limit)?;
|
|
assert_eq!((hard_limit, hard_limit), max_file_descriptors()?, "getrlimit != setrlimit");
|
|
debug!(to = hard_limit, from = soft_limit, "Raised RLIMIT_NOFILE");
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(not(unix))]
|
|
pub fn maximize_fd_limit() -> Result { Ok(()) }
|
|
|
|
#[cfg(any(
|
|
linux_android,
|
|
netbsdlike,
|
|
target_os = "aix",
|
|
target_os = "freebsd",
|
|
))]
|
|
/// Some distributions ship with very low defaults for thread counts; similar to
|
|
/// low default file descriptor limits. But unlike fd's, thread limit is rarely
|
|
/// reached, though on large systems (32+ cores) shipping with defaults of
|
|
/// ~1024 as have been observed are problematic.
|
|
pub fn maximize_thread_limit() -> Result {
|
|
use nix::sys::resource::setrlimit;
|
|
|
|
let (soft_limit, hard_limit) = max_threads()?;
|
|
if soft_limit < hard_limit {
|
|
let new_limit = hard_limit.try_into()?;
|
|
setrlimit(Resource::RLIMIT_NPROC, new_limit, new_limit)?;
|
|
assert_eq!((hard_limit, hard_limit), max_threads()?, "getrlimit != setrlimit");
|
|
debug!(to = hard_limit, from = soft_limit, "Raised RLIMIT_NPROC");
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(not(any(
|
|
linux_android,
|
|
netbsdlike,
|
|
target_os = "aix",
|
|
target_os = "freebsd",
|
|
)))]
|
|
pub fn maximize_thread_limit() -> Result { Ok(()) }
|
|
|
|
#[cfg(unix)]
|
|
pub fn max_file_descriptors() -> Result<(usize, usize)> {
|
|
getrlimit(Resource::RLIMIT_NOFILE)
|
|
.map(apply!(2, usize_from_u64_truncated))
|
|
.map_err(Into::into)
|
|
}
|
|
|
|
#[cfg(not(unix))]
|
|
pub fn max_file_descriptors() -> Result<(usize, usize)> { Ok((usize::MAX, usize::MAX)) }
|
|
|
|
#[cfg(unix)]
|
|
pub fn max_stack_size() -> Result<(usize, usize)> {
|
|
getrlimit(Resource::RLIMIT_STACK)
|
|
.map(apply!(2, usize_from_u64_truncated))
|
|
.map_err(Into::into)
|
|
}
|
|
|
|
#[cfg(not(unix))]
|
|
pub fn max_stack_size() -> Result<(usize, usize)> { Ok((usize::MAX, usize::MAX)) }
|
|
|
|
#[cfg(any(linux_android, netbsdlike, target_os = "freebsd",))]
|
|
pub fn max_memory_locked() -> Result<(usize, usize)> {
|
|
getrlimit(Resource::RLIMIT_MEMLOCK)
|
|
.map(apply!(2, usize_from_u64_truncated))
|
|
.map_err(Into::into)
|
|
}
|
|
|
|
#[cfg(not(any(linux_android, netbsdlike, target_os = "freebsd",)))]
|
|
pub fn max_memory_locked() -> Result<(usize, usize)> { Ok((usize::MIN, usize::MIN)) }
|
|
|
|
#[cfg(any(
|
|
linux_android,
|
|
netbsdlike,
|
|
target_os = "aix",
|
|
target_os = "freebsd",
|
|
))]
|
|
pub fn max_threads() -> Result<(usize, usize)> {
|
|
getrlimit(Resource::RLIMIT_NPROC)
|
|
.map(apply!(2, usize_from_u64_truncated))
|
|
.map_err(Into::into)
|
|
}
|
|
|
|
#[cfg(not(any(
|
|
linux_android,
|
|
netbsdlike,
|
|
target_os = "aix",
|
|
target_os = "freebsd",
|
|
)))]
|
|
pub fn max_threads() -> Result<(usize, usize)> { Ok((usize::MAX, usize::MAX)) }
|