studio/docs
Archived
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨(backend) improve search indexer service configuration

Browse Source 
New SEARCH_INDEXER_CLASS setting to define the indexer service class.
Raise ImpoperlyConfigured errors instead of RuntimeError in index service.

Signed-off-by: Fabre Florian <ffabre@hybird.org>
This commit is contained in:
Fabre Florian
2025-09-11 14:05:56 +02:00
committed by Quentin BEY
parent d721b97f68
commit 24460ffc3a
11 changed files with 193 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/backend/core/tests/commands/test_index.py
View File

@@ -2,6 +2,7 @@
Unit test for `index` command.
"""
from operator import itemgetter
from unittest import mock
from django.core.management import call_command
@@ -34,19 +35,18 @@ def test_index():
str(no_title_doc.path): {"users": [user.sub]},
}
def sortkey(d):
return d["id"]
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
call_command("index")
push_call_args = [call.args[0] for call in mock_push.call_args_list]
assert len(push_call_args) == 1 # called once but with a batch of docs
assert sorted(push_call_args[0], key=sortkey) == sorted(
# called once but with a batch of docs
mock_push.assert_called_once()
assert sorted(push_call_args[0], key=itemgetter("id")) == sorted(
[
indexer.serialize_document(doc, accesses),
indexer.serialize_document(no_title_doc, accesses),
],
key=sortkey,
key=itemgetter("id"),
)

7
src/backend/core/tests/documents/test_api_documents_search.py
View File

@@ -57,7 +57,7 @@ def test_api_documents_search_endpoint_is_none(settings):
response = APIClient().get("/api/v1.0/documents/search/", data={"q": "alpha"})
assert response.status_code == 401
assert response.json() == {"detail": "The service is not configured properly."}
assert response.json() == {"detail": "The service is not properly configured."}
@responses.activate
@@ -75,6 +75,11 @@ def test_api_documents_search_invalid_params(settings):
assert response.status_code == 400
assert response.json() == {"q": ["This field is required."]}
response = APIClient().get("/api/v1.0/documents/search/", data={"q": " "})
assert response.status_code == 400
assert response.json() == {"q": ["This field may not be blank."]}
@responses.activate
def test_api_documents_search_format(settings):

10
src/backend/core/tests/test_models_documents.py
View File

@@ -3,6 +3,7 @@ Unit tests for the Document model
"""
# pylint: disable=too-many-lines
from operator import itemgetter
import random
import smtplib
import time
@@ -1638,7 +1639,7 @@ def test_models_documents_post_save_indexer(mock_push, settings):
factories.UserDocumentAccessFactory(document=doc2, user=user)
factories.UserDocumentAccessFactory(document=doc3, user=user)
time.sleep(0.1) # waits for the end of the tasks
time.sleep(0.2) # waits for the end of the tasks
accesses = {
str(doc1.path): {"users": [user.sub]},
@@ -1650,16 +1651,13 @@ def test_models_documents_post_save_indexer(mock_push, settings):
indexer = FindDocumentIndexer()
def sortkey(d):
return d["id"]
assert sorted(data, key=sortkey) == sorted(
assert sorted(data, key=itemgetter('id')) == sorted(
[
indexer.serialize_document(doc1, accesses),
indexer.serialize_document(doc2, accesses),
indexer.serialize_document(doc3, accesses),
],
key=sortkey,
key=itemgetter('id'),
)
# The debounce counters should be reset

144
src/backend/core/tests/test_services_search_indexers.py
View File

@@ -4,77 +4,154 @@ from functools import partial
from unittest.mock import patch
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ImproperlyConfigured
from django.utils.module_loading import import_string
import pytest
from core import factories, models, utils
from core.services.search_indexers import (
BaseDocumentIndexer,
FindDocumentIndexer,
get_document_indexer_class,
get_visited_document_ids_of,
)
pytestmark = pytest.mark.django_db
def test_push_raises_error_if_search_indexer_url_is_none(settings):
class FakeDocumentIndexer(BaseDocumentIndexer):
"""Fake indexer for test purpose"""
def serialize_document(self, document, accesses):
return {}
def push(self, data):
pass
def search_query(self, data, token):
return {}
def format_response(self, data: dict):
return {}
@pytest.fixture(name="fake_indexer_settings")
def fake_indexer_settings_fixture(settings):
"""Fixture to switch the indexer to the FakeDocumentIndexer."""
_original_backend = str(settings.SEARCH_INDEXER_CLASS)
settings.SEARCH_INDEXER_CLASS = (
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
get_document_indexer_class.cache_clear()
yield settings
settings.SEARCH_INDEXER_CLASS = _original_backend
# clear cache to prevent issues with other tests
get_document_indexer_class.cache_clear()
def test_services_search_indexer_class_is_empty(fake_indexer_settings):
"""
Should raise ImproperlyConfigured if SEARCH_INDEXER_CLASS is None or empty.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = None
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert "SEARCH_INDEXER_CLASS must be set in Django settings." in str(exc_info.value)
fake_indexer_settings.SEARCH_INDEXER_CLASS = ""
# clear cache again
get_document_indexer_class.cache_clear()
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert "SEARCH_INDEXER_CLASS must be set in Django settings." in str(exc_info.value)
def test_services_search_indexer_class_invalid(fake_indexer_settings):
"""
Should raise RuntimeError if SEARCH_INDEXER_CLASS cannot be imported.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = "unknown.Unknown"
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert (
"SEARCH_INDEXER_CLASS setting is not valid : No module named 'unknown'"
in str(exc_info.value)
)
def test_services_search_indexer_class(fake_indexer_settings):
"""
Import indexer class defined in setting SEARCH_INDEXER_CLASS.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = (
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
assert get_document_indexer_class() == import_string(
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
def test_services_search_indexer_url_is_none(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is None or empty.
"""
settings.SEARCH_INDEXER_URL = None
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert "SEARCH_INDEXER_URL must be set in Django settings before indexing." in str(
exc_info.value
)
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
def test_push_raises_error_if_search_indexer_url_is_empty(settings):
def test_services_search_indexer_url_is_empty(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is empty string.
"""
settings.SEARCH_INDEXER_URL = ""
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert "SEARCH_INDEXER_URL must be set in Django settings before indexing." in str(
exc_info.value
)
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
def test_push_raises_error_if_search_indexer_secret_is_none(settings):
def test_services_search_indexer_secret_is_none(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is None or empty.
"""
settings.SEARCH_INDEXER_SECRET = None
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_SECRET must be set in Django settings before indexing."
in str(exc_info.value)
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
exc_info.value
)
def test_push_raises_error_if_search_indexer_secret_is_empty(settings):
def test_services_search_indexer_secret_is_empty(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is empty string.
"""
settings.SEARCH_INDEXER_SECRET = ""
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_SECRET must be set in Django settings before indexing."
in str(exc_info.value)
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
exc_info.value
)
@@ -333,13 +410,10 @@ def test_search_query_raises_error_if_search_endpoint_is_none(settings):
Indexer should raise RuntimeError if SEARCH_INDEXER_QUERY_URL is None or empty.
"""
settings.SEARCH_INDEXER_QUERY_URL = None
indexer = FindDocumentIndexer()
user = factories.UserFactory()
with pytest.raises(RuntimeError) as exc_info:
indexer.search("alpha", user=user, token="mytoken")
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_QUERY_URL must be set in Django settings before search."
in str(exc_info.value)
assert "SEARCH_INDEXER_QUERY_URL must be set in Django settings." in str(
exc_info.value
)