✨(backend) throttle indexation tasks instead of debounce (simplier)

Replace indexer_debounce_lock|release functions by indexer_throttle_acquire()
Instead of mutex-like mechanism, simply set a flag in cache for an amount of
time that prevents any other task creation.

Signed-off-by: Fabre Florian <ffabre@hybird.org>

src/backend/core/tests/documents/test_api_documents_search.py

@@ -93,6 +93,7 @@ def test_api_documents_search_endpoint_is_none(indexer_settings):
         "path": document.path,
         "title": document.title,
         "updated_at": document.updated_at.isoformat().replace("+00:00", "Z"),
+        "deleted_at": None,
         "user_role": access.role,
     }
 
@@ -184,6 +185,7 @@ def test_api_documents_search_format(indexer_settings):
         "path": document.path,
         "title": document.title,
         "updated_at": document.updated_at.isoformat().replace("+00:00", "Z"),
+        "deleted_at": None,
         "user_role": access.role,
     }
 

src/backend/core/tests/test_models_documents.py

@@ -22,6 +22,7 @@ import pytest
 
 from core import factories, models
 from core.services.search_indexers import SearchIndexer
+from core.tasks.search import document_indexer_task
 
 pytestmark = pytest.mark.django_db
 
@@ -1645,10 +1646,10 @@ def test_models_documents_post_save_indexer(mock_push, indexer_settings):
         key=itemgetter("id"),
     )
 
-    # The debounce counters should be reset
-    assert cache.get(f"doc-indexer-debounce-{doc1.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc2.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc3.pk}") == 0
+    # The throttle counters should be reset
+    assert cache.get(f"doc-indexer-throttle-{doc1.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc2.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc3.pk}") is None
 
 
 @mock.patch.object(SearchIndexer, "push")
@@ -1658,10 +1659,31 @@ def test_models_documents_post_save_indexer_not_configured(mock_push, indexer_se
     indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
     indexer_settings.SEARCH_INDEXER_CLASS = None
 
-    with transaction.atomic():
-        factories.DocumentFactory()
+    user = factories.UserFactory()
 
-    assert mock_push.call_args_list == []
+    with transaction.atomic():
+        doc = factories.DocumentFactory()
+        factories.UserDocumentAccessFactory(document=doc, user=user)
+
+    assert mock_push.assert_not_called
+
+
+@mock.patch.object(SearchIndexer, "push")
+@pytest.mark.django_db(transaction=True)
+def test_models_documents_post_save_indexer_wrongly_configured(
+    mock_push, indexer_settings
+):
+    """Task should not start an indexation when disabled"""
+    indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
+    indexer_settings.SEARCH_INDEXER_URL = None
+
+    user = factories.UserFactory()
+
+    with transaction.atomic():
+        doc = factories.DocumentFactory()
+        factories.UserDocumentAccessFactory(document=doc, user=user)
+
+    assert mock_push.assert_not_called
 
 
 @mock.patch.object(SearchIndexer, "push")
@@ -1698,10 +1720,10 @@ def test_models_documents_post_save_indexer_with_accesses(mock_push, indexer_set
         key=itemgetter("id"),
     )
 
-    # The debounce counters should be reset
-    assert cache.get(f"doc-indexer-debounce-{doc1.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc2.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc3.pk}") == 0
+    # The throttle counters should be reset
+    assert cache.get(f"doc-indexer-throttle-{doc1.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc2.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc3.pk}") is None
 
 
 @mock.patch.object(SearchIndexer, "push")
@@ -1760,10 +1782,34 @@ def test_models_documents_post_save_indexer_deleted(mock_push, indexer_settings)
         key=itemgetter("id"),
     )
 
-    # The debounce counters should be reset
-    assert cache.get(f"doc-indexer-debounce-{doc.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc_deleted.pk}") == 0
-    assert cache.get(f"doc-indexer-debounce-{doc_ancestor_deleted.pk}") == 0
+    # The throttle counters should be reset
+    assert cache.get(f"doc-indexer-throttle-{doc.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc_deleted.pk}") is None
+    assert cache.get(f"doc-indexer-throttle-{doc_ancestor_deleted.pk}") is None
+
+
+@pytest.mark.django_db(transaction=True)
+def test_models_documents_indexer_hard_deleted(indexer_settings):
+    """Indexation task on hard deleted document"""
+    indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
+
+    user = factories.UserFactory()
+
+    with transaction.atomic():
+        doc = factories.DocumentFactory(
+            link_reach=models.LinkReachChoices.AUTHENTICATED
+        )
+        factories.UserDocumentAccessFactory(document=doc, user=user)
+
+    doc_id = doc.pk
+    doc.delete()
+
+    # Call task on deleted document.
+    document_indexer_task.apply(args=[doc_id])
+
+    with mock.patch.object(SearchIndexer, "push") as mock_push:
+        # Hard delete document are not re-indexed.
+        assert mock_push.assert_not_called
 
 
 @mock.patch.object(SearchIndexer, "push")
@@ -1836,7 +1882,7 @@ def test_models_documents_post_save_indexer_restored(mock_push, indexer_settings
 
 
 @pytest.mark.django_db(transaction=True)
-def test_models_documents_post_save_indexer_debounce(indexer_settings):
+def test_models_documents_post_save_indexer_throttle(indexer_settings):
     """Test indexation task skipping on document update"""
     indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
 
@@ -1853,11 +1899,11 @@ def test_models_documents_post_save_indexer_debounce(indexer_settings):
     }
 
     with mock.patch.object(SearchIndexer, "push") as mock_push:
-        # Simulate 1 waiting task
-        cache.set(f"doc-indexer-debounce-{doc.pk}", 1)
+        # Simulate 1 running task
+        cache.set(f"doc-indexer-throttle-{doc.pk}", 1)
 
         # save doc to trigger the indexer, but nothing should be done since
-        # the counter is over 0
+        # the flag is up
         with transaction.atomic():
             doc.save()
 
@@ -1865,7 +1911,7 @@ def test_models_documents_post_save_indexer_debounce(indexer_settings):
 
     with mock.patch.object(SearchIndexer, "push") as mock_push:
         # No waiting task
-        cache.set(f"doc-indexer-debounce-{doc.pk}", 0)
+        cache.delete(f"doc-indexer-throttle-{doc.pk}")
 
         with transaction.atomic():
             doc = models.Document.objects.get(pk=doc.pk)