Add OpenSearch search backend with hybrid neural+BM25 support
Extract a SearchBackend trait from the existing RocksDB search code and add an OpenSearch implementation supporting cross-room search, relevance ranking, fuzzy matching, English stemming, and optional hybrid neural+BM25 semantic search using sentence-transformers. Fix macOS build by gating RLIMIT_NPROC and getrusage to supported platforms.
This commit is contained in:
13
.github/workflows/main.yml
vendored
13
.github/workflows/main.yml
vendored
@@ -332,6 +332,19 @@ jobs:
|
||||
{"sys_target": "x86_64-v4-linux-gnu", "feat_set": "default"},
|
||||
]
|
||||
|
||||
opensearch-tests:
|
||||
if: >
|
||||
!failure() && !cancelled()
|
||||
&& fromJSON(needs.init.outputs.enable_test)
|
||||
&& !fromJSON(needs.init.outputs.is_release)
|
||||
&& !contains(needs.init.outputs.pipeline, '[ci no test]')
|
||||
|
||||
name: OpenSearch
|
||||
needs: [init, lint]
|
||||
uses: ./.github/workflows/opensearch-tests.yml
|
||||
with:
|
||||
checkout: ${{needs.init.outputs.checkout}}
|
||||
|
||||
package:
|
||||
if: >
|
||||
!failure() && !cancelled()
|
||||
|
||||
114
.github/workflows/opensearch-tests.yml
vendored
Normal file
114
.github/workflows/opensearch-tests.yml
vendored
Normal file
@@ -0,0 +1,114 @@
|
||||
name: OpenSearch Integration Tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
checkout:
|
||||
type: string
|
||||
default: 'HEAD'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/service/rooms/search/**'
|
||||
- 'src/core/config/mod.rs'
|
||||
- '.github/workflows/opensearch-tests.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'src/service/rooms/search/**'
|
||||
- 'src/core/config/mod.rs'
|
||||
- '.github/workflows/opensearch-tests.yml'
|
||||
|
||||
jobs:
|
||||
opensearch-tests:
|
||||
name: OpenSearch Integration
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:3.5.0
|
||||
ports:
|
||||
- 9200:9200
|
||||
env:
|
||||
discovery.type: single-node
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_INITIAL_ADMIN_PASSWORD: "SuperSecret123!"
|
||||
OPENSEARCH_JAVA_OPTS: "-Xms1g -Xmx2g"
|
||||
options: >-
|
||||
--health-cmd "curl -f http://localhost:9200/_cluster/health || exit 1"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
--health-start-period 60s
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.checkout || github.sha }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@nightly
|
||||
|
||||
- name: Cache cargo registry and build
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: opensearch-tests-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
opensearch-tests-${{ runner.os }}-
|
||||
|
||||
- name: Wait for OpenSearch
|
||||
run: |
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9200/_cluster/health; then
|
||||
echo ""
|
||||
echo "OpenSearch is ready"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for OpenSearch... ($i/60)"
|
||||
sleep 2
|
||||
done
|
||||
echo "OpenSearch did not start in time"
|
||||
exit 1
|
||||
|
||||
- name: Configure OpenSearch ML plugin
|
||||
run: |
|
||||
curl -sf -X PUT http://localhost:9200/_cluster/settings \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"persistent": {
|
||||
"plugins.ml_commons.only_run_on_ml_node": "false",
|
||||
"plugins.ml_commons.native_memory_threshold": "99",
|
||||
"plugins.ml_commons.allow_registering_model_via_url": "true"
|
||||
}
|
||||
}'
|
||||
|
||||
- name: Run OpenSearch integration tests (BM25)
|
||||
env:
|
||||
OPENSEARCH_URL: http://localhost:9200
|
||||
run: >
|
||||
cargo test
|
||||
-p tuwunel_service
|
||||
--lib
|
||||
rooms::search::opensearch::tests
|
||||
--
|
||||
--ignored
|
||||
--test-threads=1
|
||||
--skip test_neural
|
||||
--skip test_hybrid
|
||||
|
||||
- name: Run OpenSearch neural/hybrid tests
|
||||
env:
|
||||
OPENSEARCH_URL: http://localhost:9200
|
||||
timeout-minutes: 15
|
||||
run: >
|
||||
cargo test
|
||||
-p tuwunel_service
|
||||
--lib
|
||||
rooms::search::opensearch::tests
|
||||
--
|
||||
--ignored
|
||||
--test-threads=1
|
||||
test_neural test_hybrid
|
||||
Reference in New Issue
Block a user