✨(backend) use batches in indexing task
Reduce the number of Find API calls by grouping all the latest changes for indexation : send all the documents updated or deleted since the triggering of the task. Signed-off-by: Fabre Florian <ffabre@hybird.org>
This commit is contained in:
committed by
Quentin BEY
parent
65d572ccd6
commit
780bcb360a
@@ -130,16 +130,17 @@ class BaseDocumentIndexer(ABC):
|
||||
"SEARCH_INDEXER_QUERY_URL must be set in Django settings."
|
||||
)
|
||||
|
||||
def index(self):
|
||||
def index(self, queryset=None):
|
||||
"""
|
||||
Fetch documents in batches, serialize them, and push to the search backend.
|
||||
"""
|
||||
last_id = 0
|
||||
count = 0
|
||||
queryset = queryset or models.Document.objects.all()
|
||||
|
||||
while True:
|
||||
documents_batch = list(
|
||||
models.Document.objects.filter(
|
||||
queryset.filter(
|
||||
id__gt=last_id,
|
||||
).order_by("id")[: self.batch_size]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user