✨(backend) add async triggers to enable document indexation with find

On document content or permission changes, start a celery job that will call the
indexation API of the app "Find".

Signed-off-by: Fabre Florian <ffabre@hybird.org>

src/backend/core/tests/test_models_documents.py

@@ -5,6 +5,7 @@ Unit tests for the Document model
 
 import random
 import smtplib
+import time
 from logging import Logger
 from unittest import mock
 
@@ -13,12 +14,15 @@ from django.core import mail
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.core.files.storage import default_storage
+from django.db import transaction
 from django.test.utils import override_settings
 from django.utils import timezone
 
 import pytest
 
 from core import factories, models
+from core.services.search_indexers import FindDocumentIndexer
+from core.tasks.find import document_indexer_debounce_key
 
 pytestmark = pytest.mark.django_db
 
@@ -1617,3 +1621,125 @@ def test_models_documents_compute_ancestors_links_paths_mapping_structure(
                 {"link_reach": sibling.link_reach, "link_role": sibling.link_role},
             ],
         }
+
+
+@mock.patch.object(FindDocumentIndexer, "push")
+@pytest.mark.django_db(transaction=True)
+def test_models_documents_post_save_indexer(mock_push, settings):
+    """Test indexation task on document creation"""
+    settings.SEARCH_INDEXER_COUNTDOWN = 0
+
+    user = factories.UserFactory()
+
+    with transaction.atomic():
+        doc1, doc2, doc3 = factories.DocumentFactory.create_batch(3)
+
+        factories.UserDocumentAccessFactory(document=doc1, user=user)
+        factories.UserDocumentAccessFactory(document=doc2, user=user)
+        factories.UserDocumentAccessFactory(document=doc3, user=user)
+
+    time.sleep(0.1)  # waits for the end of the tasks
+
+    accesses = {
+        str(doc1.path): {"users": [user.sub]},
+        str(doc2.path): {"users": [user.sub]},
+        str(doc3.path): {"users": [user.sub]},
+    }
+
+    data = [call.args[0] for call in mock_push.call_args_list]
+
+    indexer = FindDocumentIndexer()
+
+    def sortkey(d):
+        return d["id"]
+
+    assert sorted(data, key=sortkey) == sorted(
+        [
+            indexer.serialize_document(doc1, accesses),
+            indexer.serialize_document(doc2, accesses),
+            indexer.serialize_document(doc3, accesses),
+        ],
+        key=sortkey,
+    )
+
+    # The debounce counters should be reset
+    assert cache.get(document_indexer_debounce_key(doc1.pk)) == 0
+    assert cache.get(document_indexer_debounce_key(doc2.pk)) == 0
+    assert cache.get(document_indexer_debounce_key(doc3.pk)) == 0
+
+
+@pytest.mark.django_db(transaction=True)
+def test_models_documents_post_save_indexer_debounce(settings):
+    """Test indexation task skipping on document update"""
+    settings.SEARCH_INDEXER_COUNTDOWN = 0
+
+    indexer = FindDocumentIndexer()
+    user = factories.UserFactory()
+
+    with mock.patch.object(FindDocumentIndexer, "push"):
+        with transaction.atomic():
+            doc = factories.DocumentFactory()
+            factories.UserDocumentAccessFactory(document=doc, user=user)
+
+    accesses = {
+        str(doc.path): {"users": [user.sub]},
+    }
+
+    time.sleep(0.1)  # waits for the end of the tasks
+
+    with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
+        # Simulate 1 waiting task
+        cache.set(document_indexer_debounce_key(doc.pk), 1)
+
+        # save doc to trigger the indexer, but nothing should be done since
+        # the counter is over 0
+        with transaction.atomic():
+            doc.save()
+
+        time.sleep(0.1)
+
+        assert [call.args[0] for call in mock_push.call_args_list] == []
+
+    with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
+        # No waiting task
+        cache.set(document_indexer_debounce_key(doc.pk), 0)
+
+        with transaction.atomic():
+            doc = models.Document.objects.get(pk=doc.pk)
+            doc.save()
+
+        time.sleep(0.1)
+
+        assert [call.args[0] for call in mock_push.call_args_list] == [
+            indexer.serialize_document(doc, accesses),
+        ]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_models_documents_access_post_save_indexer(settings):
+    """Test indexation task on DocumentAccess update"""
+    settings.SEARCH_INDEXER_COUNTDOWN = 0
+
+    indexer = FindDocumentIndexer()
+    user = factories.UserFactory()
+
+    with mock.patch.object(FindDocumentIndexer, "push"):
+        with transaction.atomic():
+            doc = factories.DocumentFactory()
+            doc_access = factories.UserDocumentAccessFactory(document=doc, user=user)
+
+    accesses = {
+        str(doc.path): {"users": [user.sub]},
+    }
+
+    indexer = FindDocumentIndexer()
+
+    with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
+        with transaction.atomic():
+            doc_access.save()
+
+        time.sleep(0.1)
+
+        assert [call.args[0] for call in mock_push.call_args_list] == [
+            indexer.serialize_document(doc, accesses),
+        ]

src/backend/core/tests/test_services_search_indexers.py

@@ -107,7 +107,7 @@ def test_services_search_indexers_serialize_document_returns_expected_json():
         "created_at": document.created_at.isoformat(),
         "updated_at": document.updated_at.isoformat(),
         "reach": document.link_reach,
-        'size': 13,
+        "size": 13,
         "is_active": True,
     }
 
@@ -126,6 +126,17 @@ def test_services_search_indexers_serialize_document_deleted():
     assert result["is_active"] is False
 
 
+def test_services_search_indexers_serialize_document_empty():
+    """Empty documents returns empty content in the serialized json."""
+    document = factories.DocumentFactory(content="", title=None)
+
+    indexer = FindDocumentIndexer()
+    result = indexer.serialize_document(document, {})
+
+    assert result["content"] == ""
+    assert result["title"] == ""
+
+
 @patch.object(FindDocumentIndexer, "push")
 def test_services_search_indexers_batches_pass_only_batch_accesses(mock_push, settings):
     """
@@ -168,7 +179,9 @@ def test_services_search_indexers_batches_pass_only_batch_accesses(mock_push, se
 def test_services_search_indexers_ancestors_link_reach(mock_push):
     """Document accesses and reach should take into account ancestors link reaches."""
     great_grand_parent = factories.DocumentFactory(link_reach="restricted")
-    grand_parent = factories.DocumentFactory(parent=great_grand_parent, link_reach="authenticated")
+    grand_parent = factories.DocumentFactory(
+        parent=great_grand_parent, link_reach="authenticated"
+    )
     parent = factories.DocumentFactory(parent=grand_parent, link_reach="public")
     document = factories.DocumentFactory(parent=parent, link_reach="restricted")
 
@@ -199,7 +212,11 @@ def test_services_search_indexers_ancestors_users(mock_push):
     assert len(results) == 3
     assert results[str(grand_parent.id)]["users"] == [str(user_gp.sub)]
     assert set(results[str(parent.id)]["users"]) == {str(user_gp.sub), str(user_p.sub)}
-    assert set(results[str(document.id)]["users"]) == {str(user_gp.sub), str(user_p.sub), str(user_d.sub)}
+    assert set(results[str(document.id)]["users"]) == {
+        str(user_gp.sub),
+        str(user_p.sub),
+        str(user_d.sub),
+    }
 
 
 @patch.object(FindDocumentIndexer, "push")

src/backend/core/tests/test_utils.py

@@ -79,24 +79,24 @@ def test_utils_extract_attachments():
 
 def test_utils_get_ancestor_to_descendants_map_single_path():
     """Test ancestor mapping of a single path."""
-    paths = ['000100020005']
+    paths = ["000100020005"]
     result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
 
     assert result == {
-        '0001': {'000100020005'},
-        '00010002': {'000100020005'},
-        '000100020005': {'000100020005'},
+        "0001": {"000100020005"},
+        "00010002": {"000100020005"},
+        "000100020005": {"000100020005"},
     }
 
 
 def test_utils_get_ancestor_to_descendants_map_multiple_paths():
     """Test ancestor mapping of multiple paths with shared prefixes."""
-    paths = ['000100020005', '00010003']
+    paths = ["000100020005", "00010003"]
     result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
 
     assert result == {
-        '0001': {'000100020005', '00010003'},
-        '00010002': {'000100020005'},
-        '000100020005': {'000100020005'},
-        '00010003': {'00010003'},
+        "0001": {"000100020005", "00010003"},
+        "00010002": {"000100020005"},
+        "000100020005": {"000100020005"},
+        "00010003": {"00010003"},
     }