✨(backend) add django-treebeard to allow tree structure on documents

We choose to use Django-treebeard for its quality, performance and
stability. Adding tree structure to documents is as simple as
inheriting from the MP_Node class.

CHANGELOG.md

@@ -11,7 +11,8 @@ and this project adheres to
 
 ## Added
 
-- github actions to manage Crowdin workflow
+- ✨(backend) allow organizing documents in a tree structure #516
+- ✨(backend) add github actions to manage Crowdin workflow #559 & #563
 - 📈Integrate Posthog #540
 - 🏷️(backend) add content-type to uploaded files #552
 - ✨(frontend) export pdf docx front side #537

src/backend/core/admin.py

@@ -4,12 +4,16 @@ from django.contrib import admin
 from django.contrib.auth import admin as auth_admin
 from django.utils.translation import gettext_lazy as _
 
+from treebeard.admin import TreeAdmin
+from treebeard.forms import movenodeform_factory
+
 from . import models
 
 
 class TemplateAccessInline(admin.TabularInline):
     """Inline admin class for template accesses."""
 
+    autocomplete_fields = ["user"]
     model = models.TemplateAccess
     extra = 0
 
@@ -111,14 +115,47 @@ class TemplateAdmin(admin.ModelAdmin):
 class DocumentAccessInline(admin.TabularInline):
     """Inline admin class for template accesses."""
 
+    autocomplete_fields = ["user"]
     model = models.DocumentAccess
     extra = 0
 
 
 @admin.register(models.Document)
-class DocumentAdmin(admin.ModelAdmin):
+class DocumentAdmin(TreeAdmin):
     """Document admin interface declaration."""
 
+    fieldsets = (
+        (
+            None,
+            {
+                "fields": (
+                    "id",
+                    "title",
+                )
+            },
+        ),
+        (
+            _("Permissions"),
+            {
+                "fields": (
+                    "creator",
+                    "link_reach",
+                    "link_role",
+                )
+            },
+        ),
+        (
+            _("Tree structure"),
+            {
+                "fields": (
+                    "path",
+                    "depth",
+                    "numchild",
+                )
+            },
+        ),
+    )
+    form = movenodeform_factory(models.Document)
     inlines = (DocumentAccessInline,)
     list_display = (
         "id",
@@ -128,6 +165,14 @@ class DocumentAdmin(admin.ModelAdmin):
         "created_at",
         "updated_at",
     )
+    readonly_fields = (
+        "creator",
+        "depth",
+        "id",
+        "numchild",
+        "path",
+    )
+    search_fields = ("id", "title")
 
 
 @admin.register(models.Invitation)

src/backend/core/api/serializers.py

@@ -287,7 +287,7 @@ class ServerCreateDocumentSerializer(serializers.Serializer):
                 {"content": ["Could not convert content"]}
             ) from err
 
-        document = models.Document.objects.create(
+        document = models.Document.add_root(
             title=validated_data["title"],
             content=document_content,
             creator=user,

src/backend/core/api/viewsets.py

@@ -431,7 +431,11 @@ class DocumentViewSet(
 
     def perform_create(self, serializer):
         """Set the current user as creator and owner of the newly created object."""
-        obj = serializer.save(creator=self.request.user)
+        obj = models.Document.add_root(
+            creator=self.request.user,
+            **serializer.validated_data,
+        )
+        serializer.instance = obj
         models.DocumentAccess.objects.create(
             document=obj,
             user=self.request.user,

src/backend/core/factories.py

@@ -46,6 +46,23 @@ class UserFactory(factory.django.DjangoModelFactory):
             UserTemplateAccessFactory(user=self, role="owner")
 
 
+class ParentNodeFactory(factory.declarations.ParameteredAttribute):
+    """Custom factory attribute for setting the parent node."""
+
+    def generate(self, step, params):
+        """
+        Generate a parent node for the factory.
+
+        This method is invoked during the factory's build process to determine the parent
+        node of the current object being created. If `params` is provided, it uses the factory's
+        metadata to recursively create or fetch the parent node. Otherwise, it returns `None`.
+        """
+        if not params:
+            return None
+        subfactory = step.builder.factory_meta.factory
+        return step.recurse(subfactory, params)
+
+
 class DocumentFactory(factory.django.DjangoModelFactory):
     """A factory to create documents"""
 
@@ -54,6 +71,8 @@ class DocumentFactory(factory.django.DjangoModelFactory):
         django_get_or_create = ("title",)
         skip_postgeneration_save = True
 
+    parent = ParentNodeFactory()
+
     title = factory.Sequence(lambda n: f"document{n}")
     content = factory.Sequence(lambda n: f"content{n}")
     creator = factory.SubFactory(UserFactory)
@@ -64,6 +83,21 @@ class DocumentFactory(factory.django.DjangoModelFactory):
         [r[0] for r in models.LinkRoleChoices.choices]
     )
 
+    @classmethod
+    def _create(cls, model_class, *args, **kwargs):
+        """
+        Custom creation logic for the factory: creates a document as a child node if
+        a parent is provided; otherwise, creates it as a root node.
+        """
+        parent = kwargs.pop("parent", None)
+
+        if parent:
+            # Add as a child node
+            return parent.add_child(instance=model_class(**kwargs))
+
+        # Add as a root node
+        return model_class.add_root(instance=model_class(**kwargs))
+
     @factory.post_generation
     def users(self, create, extracted, **kwargs):
         """Add users to document from a given list of users with or without roles."""

src/backend/core/migrations/0014_add_tree_structure_to_documents.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.1.2 on 2024-12-07 09:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_activate_fuzzystrmatch_extension'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='depth',
+            field=models.PositiveIntegerField(default=0),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='numchild',
+            field=models.PositiveIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='path',
+            # Allow null values pending the next datamigration to populate the field
+            field=models.CharField(db_collation='C', max_length=252, null=True, unique=True),
+            preserve_default=False,
+        ),
+    ]

src/backend/core/migrations/0015_set_path_on_existing_documents.py

@@ -0,0 +1,52 @@
+# Generated by Django 5.1.2 on 2024-12-07 10:33
+
+from django.db import migrations, models
+
+from treebeard.numconv import NumConv
+
+ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+STEPLEN = 7
+
+def set_path_on_existing_documents(apps, schema_editor):
+    """
+    Updates the `path` and `depth` fields for all existing Document records 
+    to ensure valid materialized paths.
+
+    This function assigns a unique `path` to each Document as a root node
+
+    Note: After running this migration, we quickly modify the schema to make
+    the `path` field required as it should.
+    """
+    Document = apps.get_model("core", "Document")
+
+    # Iterate over all existing documents and make them root nodes
+    documents = Document.objects.order_by("created_at").values_list("id", flat=True)
+    numconv = NumConv(len(ALPHABET), ALPHABET)
+
+    updates = []
+    for i, pk in enumerate(documents):
+        key = numconv.int2str(i)
+        path = "{0}{1}".format(
+            ALPHABET[0] * (STEPLEN - len(key)),
+            key
+        )
+        updates.append(Document(pk=pk, path=path, depth=1))
+
+    # Bulk update using the prepared updates list
+    Document.objects.bulk_update(updates, ['depth', 'path'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_add_tree_structure_to_documents'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_path_on_existing_documents, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='document',
+            name='path',
+            field=models.CharField(db_collation='C', max_length=252, unique=True),
+        ),
+    ]

src/backend/core/models.py

@@ -26,6 +26,7 @@ from django.utils.translation import gettext_lazy as _
 
 from botocore.exceptions import ClientError
 from timezone_field import TimeZoneField
+from treebeard.mp_tree import MP_Node
 
 logger = getLogger(__name__)
 
@@ -366,7 +367,7 @@ class BaseAccess(BaseModel):
         }
 
 
-class Document(BaseModel):
+class Document(MP_Node, BaseModel):
     """Pad document carrying the content."""
 
     title = models.CharField(_("title"), max_length=255, null=True, blank=True)
@@ -388,9 +389,16 @@ class Document(BaseModel):
 
     _content = None
 
+    # Tree structure
+    alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+    steplen = 7  # nb siblings max: 3,521,614,606,208
+    node_order_by = []  # Manual ordering
+
+    path = models.CharField(max_length=7 * 36, unique=True, db_collation="C")
+
     class Meta:
         db_table = "impress_document"
-        ordering = ("title",)
+        ordering = ("path",)
         verbose_name = _("Document")
         verbose_name_plural = _("Documents")
 

src/backend/core/tests/test_models_documents.py

@@ -34,20 +34,18 @@ def test_models_documents_id_unique():
 
 def test_models_documents_creator_required():
     """No field should be required on the Document model."""
-    models.Document.objects.create()
+    models.Document.add_root()
 
 
 def test_models_documents_title_null():
     """The "title" field can be null."""
-    document = models.Document.objects.create(
-        title=None, creator=factories.UserFactory()
-    )
+    document = models.Document.add_root(title=None, creator=factories.UserFactory())
     assert document.title is None
 
 
 def test_models_documents_title_empty():
     """The "title" field can be empty."""
-    document = models.Document.objects.create(title="", creator=factories.UserFactory())
+    document = models.Document.add_root(title="", creator=factories.UserFactory())
     assert document.title == ""
 
 
@@ -67,6 +65,22 @@ def test_models_documents_file_key():
     assert document.file_key == "9531a5f1-42b1-496c-b3f4-1c09ed139b3c/file"
 
 
+def test_models_documents_tree_alphabet():
+    """Test the creation of documents with treebeard methods."""
+    models.Document.load_bulk(
+        [
+            {
+                "data": {
+                    "title": f"document-{i}",
+                }
+            }
+            for i in range(len(models.Document.alphabet) * 2)
+        ]
+    )
+
+    assert models.Document.objects.count() == 124
+
+
 # get_abilities
 
 

src/backend/demo/management/commands/create_demo.py

@@ -135,9 +135,14 @@ def create_demo(stdout):
     users_ids = list(models.User.objects.values_list("id", flat=True))
 
     with Timeit(stdout, "Creating documents"):
-        for _ in range(defaults.NB_OBJECTS["docs"]):
+        for i in range(defaults.NB_OBJECTS["docs"]):
+            # pylint: disable=protected-access
+            key = models.Document._int2str(i)  # noqa: SLF001
+            padding = models.Document.alphabet[0] * (models.Document.steplen - len(key))
             queue.push(
                 models.Document(
+                    depth=1,
+                    path=f"{padding}{key}",
                     creator_id=random.choice(users_ids),
                     title=fake.sentence(nb_words=4),
                     link_reach=models.LinkReachChoices.AUTHENTICATED

src/backend/impress/settings.py

@@ -299,6 +299,7 @@ class Base(Configuration):
         "dockerflow.django",
         "rest_framework",
         "parler",
+        "treebeard",
         "easy_thumbnails",
         # Django
         "django.contrib.admin",

src/backend/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
     "django-storages[s3]==1.14.4",
     "django-timezone-field>=5.1",
     "django==5.1.5",
+    "django-treebeard==4.7.1",
     "djangorestframework==3.15.2",
     "drf_spectacular==0.28.0",
     "dockerflow==2024.4.2",