studio/docs
Archived
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨(backend) improve search indexer service configuration

Browse Source 
New SEARCH_INDEXER_CLASS setting to define the indexer service class.
Raise ImpoperlyConfigured errors instead of RuntimeError in index service.

Signed-off-by: Fabre Florian <ffabre@hybird.org>
This commit is contained in:
Fabre Florian
2025-09-11 14:05:56 +02:00
committed by Quentin BEY
parent d721b97f68
commit 24460ffc3a
11 changed files with 193 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/backend/core/api/serializers.py
View File

@@ -1018,12 +1018,4 @@ class ThreadSerializer(serializers.ModelSerializer):
class FindDocumentSerializer(serializers.Serializer):
"""Serializer for Find search requests"""
q = serializers.CharField(required=True)
def validate_q(self, value):
"""Ensure the text field is not empty."""
if len(value.strip()) == 0:
raise serializers.ValidationError("Text field cannot be empty.")
return value
q = serializers.CharField(required=True, allow_blank=False, trim_whitespace=True)

10
src/backend/core/api/viewsets.py
View File

@@ -13,7 +13,7 @@ from django.conf import settings
from django.contrib.postgres.aggregates import ArrayAgg
from django.contrib.postgres.search import TrigramSimilarity
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.core.files.storage import default_storage
from django.core.validators import URLValidator
from django.db import connection, transaction
@@ -52,7 +52,7 @@ from core.services.converter_services import (
from core.services.converter_services import (
YdocConverter,
)
from core.services.search_indexers import FindDocumentIndexer
from core.services.search_indexers import get_document_indexer_class
from core.tasks.mail import send_ask_for_access_mail
from core.utils import extract_attachments, filter_descendants
@@ -1095,15 +1095,15 @@ class DocumentViewSet(
serializer.is_valid(raise_exception=True)
try:
indexer = FindDocumentIndexer()
indexer = get_document_indexer_class()()
queryset = indexer.search(
text=serializer.validated_data.get("q", ""),
user=request.user,
token=access_token,
)
except RuntimeError:
except ImproperlyConfigured:
return drf.response.Response(
{"detail": "The service is not configured properly."},
{"detail": "The service is not properly configured."},
status=status.HTTP_401_UNAUTHORIZED,
)

4
src/backend/core/models.py
View File

@@ -956,7 +956,7 @@ class Document(MP_Node, BaseModel):
@receiver(signals.post_save, sender=Document)
def document_post_save(sender, instance, **kwargs):
def document_post_save(sender, instance, **kwargs): # pylint: disable=unused-argument
"""
Asynchronous call to the document indexer at the end of the transaction.
Note : Within the transaction we can have an empty content and a serialization
@@ -1196,7 +1196,7 @@ class DocumentAccess(BaseAccess):
@receiver(signals.post_save, sender=DocumentAccess)
def document_access_post_save(sender, instance, created, **kwargs):
def document_access_post_save(sender, instance, created, **kwargs): # pylint: disable=unused-argument
"""
Asynchronous call to the document indexer at the end of the transaction.
"""

70
src/backend/core/services/search_indexers.py
View File

@@ -3,9 +3,12 @@
import logging
from abc import ABC, abstractmethod
from collections import defaultdict
from functools import cache
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ImproperlyConfigured
from django.utils.module_loading import import_string
import requests
@@ -14,18 +17,33 @@ from core import models, utils
logger = logging.getLogger(__name__)
@cache
def get_document_indexer_class() -> "BaseDocumentIndexer":
"""Return the indexer backend class based on the settings."""
classpath = settings.SEARCH_INDEXER_CLASS
if not classpath:
raise ImproperlyConfigured(
"SEARCH_INDEXER_CLASS must be set in Django settings."
)
try:
return import_string(settings.SEARCH_INDEXER_CLASS)
except ImportError as err:
raise ImproperlyConfigured(
f"SEARCH_INDEXER_CLASS setting is not valid : {err}"
) from err
def get_batch_accesses_by_users_and_teams(paths):
"""
Get accesses related to a list of document paths,
grouped by users and teams, including all ancestor paths.
"""
# print("paths: ", paths)
ancestor_map = utils.get_ancestor_to_descendants_map(
paths, steplen=models.Document.steplen
)
ancestor_paths = list(ancestor_map.keys())
# print("ancestor map: ", ancestor_map)
# print("ancestor paths: ", ancestor_paths)
access_qs = models.DocumentAccess.objects.filter(
document__path__in=ancestor_paths
@@ -80,6 +98,24 @@ class BaseDocumentIndexer(ABC):
Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
"""
self.batch_size = batch_size or settings.SEARCH_INDEXER_BATCH_SIZE
self.indexer_url = settings.SEARCH_INDEXER_URL
self.indexer_secret = settings.SEARCH_INDEXER_SECRET
self.search_url = settings.SEARCH_INDEXER_QUERY_URL
if not self.indexer_url:
raise ImproperlyConfigured(
"SEARCH_INDEXER_URL must be set in Django settings."
)
if not self.indexer_secret:
raise ImproperlyConfigured(
"SEARCH_INDEXER_SECRET must be set in Django settings."
)
if not self.search_url:
raise ImproperlyConfigured(
"SEARCH_INDEXER_QUERY_URL must be set in Django settings."
)
def index(self):
"""
@@ -143,7 +179,7 @@ class BaseDocumentIndexer(ABC):
@abstractmethod
def search_query(self, data, token) -> dict:
"""
Retreive documents from the Find app API.
Retrieve documents from the Find app API.
Must be implemented by subclasses.
"""
@@ -204,16 +240,9 @@ class FindDocumentIndexer(BaseDocumentIndexer):
Returns:
dict: A JSON-serializable dictionary.
"""
url = getattr(settings, "SEARCH_INDEXER_QUERY_URL", None)
if not url:
raise RuntimeError(
"SEARCH_INDEXER_QUERY_URL must be set in Django settings before search."
)
try:
response = requests.post(
url,
self.search_url,
json=data,
headers={"Authorization": f"Bearer {token}"},
timeout=10,
@@ -222,7 +251,6 @@ class FindDocumentIndexer(BaseDocumentIndexer):
return response.json()
except requests.exceptions.HTTPError as e:
logger.error("HTTPError: %s", e)
logger.error("Response content: %s", response.text) # type: ignore
raise
def format_response(self, data: dict):
@@ -238,27 +266,15 @@ class FindDocumentIndexer(BaseDocumentIndexer):
Args:
data (list): List of document dictionaries.
"""
url = getattr(settings, "SEARCH_INDEXER_URL", None)
if not url:
raise RuntimeError(
"SEARCH_INDEXER_URL must be set in Django settings before indexing."
)
secret = getattr(settings, "SEARCH_INDEXER_SECRET", None)
if not secret:
raise RuntimeError(
"SEARCH_INDEXER_SECRET must be set in Django settings before indexing."
)
try:
response = requests.post(
url,
self.indexer_url,
json=data,
headers={"Authorization": f"Bearer {secret}"},
headers={"Authorization": f"Bearer {self.indexer_secret}"},
timeout=10,
)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
logger.error("HTTPError: %s", e)
logger.error("Response content: %s", response.text)
raise

13
src/backend/core/tasks/find.py
View File

@@ -1,5 +1,6 @@
"""Trigger document indexation using celery task."""
from functools import partial
from logging import getLogger
from django.conf import settings
@@ -8,8 +9,8 @@ from django.db import transaction
from core import models
from core.services.search_indexers import (
FindDocumentIndexer,
get_batch_accesses_by_users_and_teams,
get_document_indexer_class,
)
from impress.celery_app import app
@@ -52,7 +53,7 @@ def document_indexer_task(document_id):
return
doc = models.Document.objects.get(pk=document_id)
indexer = FindDocumentIndexer()
indexer = get_document_indexer_class()()
accesses = get_batch_accesses_by_users_and_teams((doc.path,))
data = indexer.serialize_document(document=doc, accesses=accesses)
@@ -75,11 +76,9 @@ def trigger_document_indexer(document, on_commit=False):
pass
if on_commit:
def _aux():
trigger_document_indexer(document, on_commit=False)
transaction.on_commit(_aux)
transaction.on_commit(
partial(trigger_document_indexer, document, on_commit=False)
)
else:
key = document_indexer_debounce_key(document.pk)
countdown = getattr(settings, "SEARCH_INDEXER_COUNTDOWN", 1)

12
src/backend/core/tests/commands/test_index.py
View File

@@ -2,6 +2,7 @@
Unit test for `index` command.
"""
from operator import itemgetter
from unittest import mock
from django.core.management import call_command
@@ -34,19 +35,18 @@ def test_index():
str(no_title_doc.path): {"users": [user.sub]},
}
def sortkey(d):
return d["id"]
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
call_command("index")
push_call_args = [call.args[0] for call in mock_push.call_args_list]
assert len(push_call_args) == 1 # called once but with a batch of docs
assert sorted(push_call_args[0], key=sortkey) == sorted(
# called once but with a batch of docs
mock_push.assert_called_once()
assert sorted(push_call_args[0], key=itemgetter("id")) == sorted(
[
indexer.serialize_document(doc, accesses),
indexer.serialize_document(no_title_doc, accesses),
],
key=sortkey,
key=itemgetter("id"),
)

7
src/backend/core/tests/documents/test_api_documents_search.py
View File

@@ -57,7 +57,7 @@ def test_api_documents_search_endpoint_is_none(settings):
response = APIClient().get("/api/v1.0/documents/search/", data={"q": "alpha"})
assert response.status_code == 401
assert response.json() == {"detail": "The service is not configured properly."}
assert response.json() == {"detail": "The service is not properly configured."}
@responses.activate
@@ -75,6 +75,11 @@ def test_api_documents_search_invalid_params(settings):
assert response.status_code == 400
assert response.json() == {"q": ["This field is required."]}
response = APIClient().get("/api/v1.0/documents/search/", data={"q": " "})
assert response.status_code == 400
assert response.json() == {"q": ["This field may not be blank."]}
@responses.activate
def test_api_documents_search_format(settings):

10
src/backend/core/tests/test_models_documents.py
View File

@@ -3,6 +3,7 @@ Unit tests for the Document model
"""
# pylint: disable=too-many-lines
from operator import itemgetter
import random
import smtplib
import time
@@ -1638,7 +1639,7 @@ def test_models_documents_post_save_indexer(mock_push, settings):
factories.UserDocumentAccessFactory(document=doc2, user=user)
factories.UserDocumentAccessFactory(document=doc3, user=user)
time.sleep(0.1) # waits for the end of the tasks
time.sleep(0.2) # waits for the end of the tasks
accesses = {
str(doc1.path): {"users": [user.sub]},
@@ -1650,16 +1651,13 @@ def test_models_documents_post_save_indexer(mock_push, settings):
indexer = FindDocumentIndexer()
def sortkey(d):
return d["id"]
assert sorted(data, key=sortkey) == sorted(
assert sorted(data, key=itemgetter('id')) == sorted(
[
indexer.serialize_document(doc1, accesses),
indexer.serialize_document(doc2, accesses),
indexer.serialize_document(doc3, accesses),
],
key=sortkey,
key=itemgetter('id'),
)
# The debounce counters should be reset

144
src/backend/core/tests/test_services_search_indexers.py
View File

@@ -4,77 +4,154 @@ from functools import partial
from unittest.mock import patch
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ImproperlyConfigured
from django.utils.module_loading import import_string
import pytest
from core import factories, models, utils
from core.services.search_indexers import (
BaseDocumentIndexer,
FindDocumentIndexer,
get_document_indexer_class,
get_visited_document_ids_of,
)
pytestmark = pytest.mark.django_db
def test_push_raises_error_if_search_indexer_url_is_none(settings):
class FakeDocumentIndexer(BaseDocumentIndexer):
"""Fake indexer for test purpose"""
def serialize_document(self, document, accesses):
return {}
def push(self, data):
pass
def search_query(self, data, token):
return {}
def format_response(self, data: dict):
return {}
@pytest.fixture(name="fake_indexer_settings")
def fake_indexer_settings_fixture(settings):
"""Fixture to switch the indexer to the FakeDocumentIndexer."""
_original_backend = str(settings.SEARCH_INDEXER_CLASS)
settings.SEARCH_INDEXER_CLASS = (
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
get_document_indexer_class.cache_clear()
yield settings
settings.SEARCH_INDEXER_CLASS = _original_backend
# clear cache to prevent issues with other tests
get_document_indexer_class.cache_clear()
def test_services_search_indexer_class_is_empty(fake_indexer_settings):
"""
Should raise ImproperlyConfigured if SEARCH_INDEXER_CLASS is None or empty.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = None
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert "SEARCH_INDEXER_CLASS must be set in Django settings." in str(exc_info.value)
fake_indexer_settings.SEARCH_INDEXER_CLASS = ""
# clear cache again
get_document_indexer_class.cache_clear()
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert "SEARCH_INDEXER_CLASS must be set in Django settings." in str(exc_info.value)
def test_services_search_indexer_class_invalid(fake_indexer_settings):
"""
Should raise RuntimeError if SEARCH_INDEXER_CLASS cannot be imported.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = "unknown.Unknown"
with pytest.raises(ImproperlyConfigured) as exc_info:
get_document_indexer_class()
assert (
"SEARCH_INDEXER_CLASS setting is not valid : No module named 'unknown'"
in str(exc_info.value)
)
def test_services_search_indexer_class(fake_indexer_settings):
"""
Import indexer class defined in setting SEARCH_INDEXER_CLASS.
"""
fake_indexer_settings.SEARCH_INDEXER_CLASS = (
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
assert get_document_indexer_class() == import_string(
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
)
def test_services_search_indexer_url_is_none(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is None or empty.
"""
settings.SEARCH_INDEXER_URL = None
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert "SEARCH_INDEXER_URL must be set in Django settings before indexing." in str(
exc_info.value
)
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
def test_push_raises_error_if_search_indexer_url_is_empty(settings):
def test_services_search_indexer_url_is_empty(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is empty string.
"""
settings.SEARCH_INDEXER_URL = ""
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert "SEARCH_INDEXER_URL must be set in Django settings before indexing." in str(
exc_info.value
)
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
def test_push_raises_error_if_search_indexer_secret_is_none(settings):
def test_services_search_indexer_secret_is_none(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is None or empty.
"""
settings.SEARCH_INDEXER_SECRET = None
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_SECRET must be set in Django settings before indexing."
in str(exc_info.value)
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
exc_info.value
)
def test_push_raises_error_if_search_indexer_secret_is_empty(settings):
def test_services_search_indexer_secret_is_empty(settings):
"""
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is empty string.
"""
settings.SEARCH_INDEXER_SECRET = ""
indexer = FindDocumentIndexer()
with pytest.raises(RuntimeError) as exc_info:
indexer.push([])
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_SECRET must be set in Django settings before indexing."
in str(exc_info.value)
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
exc_info.value
)
@@ -333,13 +410,10 @@ def test_search_query_raises_error_if_search_endpoint_is_none(settings):
Indexer should raise RuntimeError if SEARCH_INDEXER_QUERY_URL is None or empty.
"""
settings.SEARCH_INDEXER_QUERY_URL = None
indexer = FindDocumentIndexer()
user = factories.UserFactory()
with pytest.raises(RuntimeError) as exc_info:
indexer.search("alpha", user=user, token="mytoken")
with pytest.raises(ImproperlyConfigured) as exc_info:
FindDocumentIndexer()
assert (
"SEARCH_INDEXER_QUERY_URL must be set in Django settings before search."
in str(exc_info.value)
assert "SEARCH_INDEXER_QUERY_URL must be set in Django settings." in str(
exc_info.value
)

10
src/backend/impress/settings.py
View File

@@ -100,6 +100,11 @@ class Base(Configuration):
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
# Search
SEARCH_INDEXER_CLASS = values.Value(
default="core.services.search_indexers.FindDocumentIndexer",
environ_name="SEARCH_INDEXER_CLASS",
environ_prefix=None,
)
SEARCH_INDEXER_BATCH_SIZE = values.IntegerValue(
default=100_000, environ_name="SEARCH_INDEXER_BATCH_SIZE", environ_prefix=None
)
@@ -977,6 +982,11 @@ class Test(Base):
# Tests are raising warnings because the /data/static directory does not exist
STATIC_ROOT = None
# Setup indexer configuration to make test working on the CI.
SEARCH_INDEXER_SECRET = "ThisIsAKeyForTest" # noqa
SEARCH_INDEXER_URL = "http://localhost:8081/api/v1.0/documents/index/"
SEARCH_INDEXER_QUERY_URL = "http://localhost:8081/api/v1.0/documents/search/"
CELERY_TASK_ALWAYS_EAGER = values.BooleanValue(True)
def __init__(self):