✨(mailboxes) add mail provisioning api integration

We want people to create new mailboxes in La Régie.
This commit adds integration with intermediary dimail-api,
which will in turn send our email creation request to Open-Xchange.

src/backend/mailbox_manager/api/serializers.py

@@ -11,6 +11,8 @@ class MailboxSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Mailbox
         fields = ["id", "first_name", "last_name", "local_part", "secondary_email"]
+        # everything is actually read-only as we do not allow update for now
+        read_only_fields = ["id"]
 
 
 class MailDomainSerializer(serializers.ModelSerializer):

src/backend/mailbox_manager/api/viewsets.py

@@ -74,7 +74,18 @@ class MailBoxViewSet(
     mixins.ListModelMixin,
     viewsets.GenericViewSet,
 ):
-    """MailBox ViewSet"""
+    """MailBox ViewSet
+
+    GET /api/<version>/mail-domains/<domain-slug>/mailboxes/
+        Return a list of mailboxes on the domain
+
+    POST /api/<version>/mail-domains/<domain-slug>/mailboxes/ with expected data:
+        - first_name: str
+        - last_name: str
+        - local_part: str
+        - secondary_email: str
+        Sends request to email provisioning API and returns newly created mailbox
+    """
 
     permission_classes = [permissions.MailBoxPermission]
     serializer_class = serializers.MailboxSerializer

src/backend/mailbox_manager/models.py

@@ -3,15 +3,15 @@ Declare and configure the models for the People additional application : mailbox
 """
 
 from django.conf import settings
-from django.core import validators
+from django.core import exceptions, validators
-from django.core.exceptions import ValidationError
+from django.db import models, transaction
-from django.db import models
 from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 
 from core.models import BaseModel
 
 from mailbox_manager.enums import MailDomainRoleChoices, MailDomainStatusChoices
+from mailbox_manager.utils.dimail import DimailAPIClient
 
 
 class MailDomain(BaseModel):
@@ -138,8 +138,30 @@ class Mailbox(BaseModel):
     def __str__(self):
         return f"{self.local_part!s}@{self.domain.name:s}"
 
-    def save(self, *args, **kwargs):
+    def clean(self):
-        self.full_clean()
+        """Mailboxes can be created only on enabled domains, with a set secret."""
         if self.domain.status != MailDomainStatusChoices.ENABLED:
-            raise ValidationError("You can create mailbox only for a domain enabled")
+            raise exceptions.ValidationError(
-        super().save(*args, **kwargs)
+                "You can create mailbox only for a domain enabled"
+            )
+
+        if not self.domain.secret:
+            raise exceptions.ValidationError(
+                "Please configure your domain's secret before creating any mailbox."
+            )
+
+    def save(self, *args, **kwargs):
+        """
+        Override save function to fire a request on mailbox creation.
+        Modification is forbidden for now.
+        """
+        self.full_clean()
+
+        if self._state.adding:
+            with transaction.atomic():
+                client = DimailAPIClient()
+                client.send_mailbox_request(self)
+                return super().save(*args, **kwargs)
+
+        # Update is not implemented for now
+        raise NotImplementedError()

src/backend/mailbox_manager/tests/api/mailboxes/test_api_mailboxes_create.py

@@ -2,7 +2,11 @@
 Unit tests for the mailbox API
 """
 
+import json
+import re
+
 import pytest
+import responses
 from rest_framework import status
 from rest_framework.test import APIClient
 
@@ -130,12 +134,12 @@ def test_api_mailboxes__create_with_accent_success(role):
     mailbox_values = serializers.MailboxSerializer(
         factories.MailboxFactory.build(first_name="Aimé")
     ).data
+
     response = client.post(
         f"/api/v1.0/mail-domains/{mail_domain.slug}/mailboxes/",
         mailbox_values,
         format="json",
     )
-
     assert response.status_code == status.HTTP_201_CREATED
     mailbox = models.Mailbox.objects.get()
 
@@ -187,3 +191,135 @@ def test_api_mailboxes__create_administrator_missing_fields():
     assert response.status_code == status.HTTP_400_BAD_REQUEST
     assert not models.Mailbox.objects.exists()
     assert response.json() == {"secondary_email": ["This field is required."]}
+
+
+### SYNC TO PROVISIONING API
+
+
+def test_api_mailboxes__unrelated_user_provisioning_api_not_called():
+    """
+    Provisioning API should not be called if an user tries
+    to create a mailbox on a domain they have no access to.
+    """
+    domain = factories.MailDomainEnabledFactory()
+
+    client = APIClient()
+    client.force_login(core_factories.UserFactory())  # user with no access
+    body_values = serializers.MailboxSerializer(
+        factories.MailboxFactory.build(domain=domain)
+    ).data
+    with responses.RequestsMock():
+        # We add no simulated response in RequestsMock
+        # because we expected no "outside" calls to be made
+        response = client.post(
+            f"/api/v1.0/mail-domains/{domain.slug}/mailboxes/",
+            body_values,
+            format="json",
+        )
+        # No exception raised by RequestsMock means no call was sent
+        # our API blocked the request before sending it
+        assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+def test_api_mailboxes__domain_viewer_provisioning_api_not_called():
+    """
+    Provisioning API should not be called if a domain viewer tries
+    to create a mailbox on a domain they are not owner/admin of.
+    """
+    access = factories.MailDomainAccessFactory(
+        domain=factories.MailDomainEnabledFactory(),
+        user=core_factories.UserFactory(),
+        role=enums.MailDomainRoleChoices.VIEWER,
+    )
+
+    client = APIClient()
+    client.force_login(access.user)
+    body_values = serializers.MailboxSerializer(factories.MailboxFactory.build()).data
+    with responses.RequestsMock():
+        # We add no simulated response in RequestsMock
+        # because we expected no "outside" calls to be made
+        response = client.post(
+            f"/api/v1.0/mail-domains/{access.domain.slug}/mailboxes/",
+            body_values,
+            format="json",
+        )
+        # No exception raised by RequestsMock means no call was sent
+        # our API blocked the request before sending it
+        assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+@pytest.mark.parametrize(
+    "role",
+    [enums.MailDomainRoleChoices.ADMIN, enums.MailDomainRoleChoices.OWNER],
+)
+def test_api_mailboxes__domain_owner_or_admin_successful_creation_and_provisioning(
+    role,
+):
+    """
+    Domain owner/admin should be able to create mailboxes.
+    Provisioning API should be called when owner/admin makes a call.
+    Expected response contains new email and password.
+    """
+    # creating all needed objects
+    access = factories.MailDomainAccessFactory(role=role)
+
+    client = APIClient()
+    client.force_login(access.user)
+    mailbox_data = serializers.MailboxSerializer(
+        factories.MailboxFactory.build(domain=access.domain)
+    ).data
+
+    with responses.RequestsMock() as rsps:
+        # Ensure successful response using "responses":
+        rsps.add(
+            rsps.GET,
+            re.compile(r".*/token/"),
+            body='{"access_token": "domain_owner_token"}',
+            status=status.HTTP_200_OK,
+            content_type="application/json",
+        )
+        rsp = rsps.add(
+            rsps.POST,
+            re.compile(rf".*/domains/{access.domain.name}/mailboxes/"),
+            body=str(
+                {
+                    "email": f"{mailbox_data['local_part']}@{access.domain.name}",
+                    "password": "newpass",
+                    "uuid": "uuid",
+                }
+            ),
+            status=status.HTTP_201_CREATED,
+            content_type="application/json",
+        )
+
+        response = client.post(
+            f"/api/v1.0/mail-domains/{access.domain.slug}/mailboxes/",
+            mailbox_data,
+            format="json",
+        )
+
+        # Checks payload sent to email-provisioning API
+        payload = json.loads(rsps.calls[1].request.body)
+        assert payload == {
+            "displayName": f"{mailbox_data['first_name']} {mailbox_data['last_name']}",
+            "email": f"{mailbox_data['local_part']}@{access.domain.name}",
+            "givenName": mailbox_data["first_name"],
+            "surName": mailbox_data["last_name"],
+        }
+
+        # Checks response
+        assert response.status_code == status.HTTP_201_CREATED
+        assert rsp.call_count == 1
+
+    mailbox = models.Mailbox.objects.get()
+    assert response.json() == {
+        "id": str(mailbox.id),
+        "first_name": str(mailbox_data["first_name"]),
+        "last_name": str(mailbox_data["last_name"]),
+        "local_part": str(mailbox_data["local_part"]),
+        "secondary_email": str(mailbox_data["secondary_email"]),
+    }
+    assert mailbox.first_name == mailbox_data["first_name"]
+    assert mailbox.last_name == mailbox_data["last_name"]
+    assert mailbox.local_part == mailbox_data["local_part"]
+    assert mailbox.secondary_email == mailbox_data["secondary_email"]

src/backend/mailbox_manager/tests/test_models_mailboxes.py

@@ -2,14 +2,35 @@
 Unit tests for the mailbox model
 """
 
+import json
+import logging
+import re
+from logging import Logger
+from unittest import mock
+
 from django.core.exceptions import ValidationError
 
 import pytest
+import requests
+import responses
+from rest_framework import status
+from urllib3.util import Retry
 
-from mailbox_manager import enums, factories
+from mailbox_manager import enums, factories, models
+from mailbox_manager.api import serializers
 
 pytestmark = pytest.mark.django_db
 
+logger = logging.getLogger(__name__)
+
+adapter = requests.adapters.HTTPAdapter(
+    max_retries=Retry(
+        total=4,
+        backoff_factor=0.1,
+        status_forcelist=[500, 502],
+        allowed_methods=["PATCH"],
+    )
+)
 
 # LOCAL PART FIELD
 
@@ -33,11 +54,13 @@ def test_models_mailboxes__local_part_matches_expected_format():
     """
     factories.MailboxFactory(local_part="Marie-Jose.Perec+JO_2024")
 
+    # other special characters (such as "@" or "!") should raise a validation error
     with pytest.raises(ValidationError, match="Enter a valid value"):
         factories.MailboxFactory(local_part="mariejo@unnecessarydomain.com")
 
-    with pytest.raises(ValidationError, match="Enter a valid value"):
+    for character in ["!", "$", "%"]:
-        factories.MailboxFactory(local_part="!")
+        with pytest.raises(ValidationError, match="Enter a valid value"):
+            factories.MailboxFactory(local_part=f"marie{character}jo")
 
 
 def test_models_mailboxes__local_part_unique_per_domain():
@@ -59,6 +82,9 @@ def test_models_mailboxes__local_part_unique_per_domain():
 
 # DOMAIN FIELD
 
+session = requests.Session()
+session.mount("http://", adapter)
+
 
 def test_models_mailboxes__domain_must_be_a_maildomain_instance():
     """The "domain" field should be an instance of MailDomain."""
@@ -72,7 +98,7 @@ def test_models_mailboxes__domain_must_be_a_maildomain_instance():
 
 def test_models_mailboxes__domain_cannot_be_null():
     """The "domain" field should not be null."""
-    with pytest.raises(ValidationError, match="This field cannot be null"):
+    with pytest.raises(models.MailDomain.DoesNotExist, match="Mailbox has no domain."):
         factories.MailboxFactory(domain=None)
 
 
@@ -126,3 +152,125 @@ def test_models_mailboxes__cannot_be_created_for_pending_maildomain():
         # MailDomainFactory initializes a mail domain with default values,
         # so mail domain status is pending!
         factories.MailboxFactory(domain=factories.MailDomainFactory())
+
+
+### SYNC TO DIMAIL-API
+
+
+def test_models_mailboxes__no_secret():
+    """If no secret is declared on the domain, the function should raise an error."""
+    domain = factories.MailDomainEnabledFactory(secret=None)
+
+    with pytest.raises(
+        ValidationError,
+        match="Please configure your domain's secret before creating any mailbox.",
+    ):
+        factories.MailboxFactory(domain=domain)
+
+
+def test_models_mailboxes__wrong_secret():
+    """If domain secret is inaccurate, the function should raise an error."""
+
+    domain = factories.MailDomainEnabledFactory()
+
+    with responses.RequestsMock() as rsps:
+        # Ensure successful response by scim provider using "responses":
+        rsps.add(
+            rsps.GET,
+            re.compile(r".*/token/"),
+            body='{"detail": "Permission denied"}',
+            status=status.HTTP_401_UNAUTHORIZED,
+            content_type="application/json",
+        )
+        rsps.add(
+            rsps.POST,
+            re.compile(rf".*/domains/{domain.name}/mailboxes/"),
+            body='{"detail": "Permission denied"}',
+            status=status.HTTP_401_UNAUTHORIZED,
+            content_type="application/json",
+        )
+
+        mailbox = factories.MailboxFactory(domain=domain)
+
+        # Payload sent to mailbox provider
+        payload = json.loads(rsps.calls[1].request.body)
+        assert payload == {
+            "displayName": f"{mailbox.first_name} {mailbox.last_name}",
+            "email": f"{mailbox.local_part}@{domain.name}",
+            "givenName": mailbox.first_name,
+            "surName": mailbox.last_name,
+        }
+
+
+@mock.patch.object(Logger, "error")
+@mock.patch.object(Logger, "info")
+def test_models_mailboxes__create_mailbox_success(mock_info, mock_error):
+    """Creating a mailbox sends the expected information and get expected response before saving."""
+    domain = factories.MailDomainEnabledFactory()
+
+    # generate mailbox data before mailbox, to mock responses
+    mailbox_data = serializers.MailboxSerializer(
+        factories.MailboxFactory.build(domain=domain)
+    ).data
+
+    with responses.RequestsMock() as rsps:
+        # Ensure successful response using "responses":
+        rsps.add(
+            rsps.GET,
+            re.compile(r".*/token/"),
+            body='{"access_token": "domain_owner_token"}',
+            status=status.HTTP_200_OK,
+            content_type="application/json",
+        )
+        rsps.add(
+            rsps.POST,
+            re.compile(rf".*/domains/{domain.name}/mailboxes/"),
+            body=str(
+                {
+                    "email": f"{mailbox_data['local_part']}@{domain.name}",
+                    "password": "newpass",
+                    "uuid": "uuid",
+                }
+            ),
+            status=status.HTTP_201_CREATED,
+            content_type="application/json",
+        )
+
+        mailbox = factories.MailboxFactory(
+            local_part=mailbox_data["local_part"], domain=domain
+        )
+
+        # Check headers
+        headers = rsps.calls[1].request.headers
+        # assert "Authorization" not in headers
+        assert headers["Content-Type"] == "application/json"
+
+        # Payload sent to mailbox provider
+        payload = json.loads(rsps.calls[1].request.body)
+        assert payload == {
+            "displayName": f"{mailbox.first_name} {mailbox.last_name}",
+            "email": f"{mailbox.local_part}@{domain.name}",
+            "givenName": mailbox.first_name,
+            "surName": mailbox.last_name,
+        }
+
+    # Logger
+    assert not mock_error.called
+    assert mock_info.call_count == 1
+    assert mock_info.call_args_list[0][0] == (
+        "Mailbox successfully created on domain %s",
+        domain.name,
+    )
+    assert mock_info.call_args_list[0][1] == (
+        {
+            "extra": {
+                "response": str(
+                    {
+                        "email": f"{mailbox.local_part}@{domain.name}",
+                        "password": "newpass",
+                        "uuid": "uuid",
+                    }
+                )
+            }
+        }
+    )

src/backend/mailbox_manager/utils/dimail.py

@@ -0,0 +1,88 @@
+"""A minimalist client to synchronize with mailbox provisioning API."""
+
+from logging import getLogger
+
+from django.conf import settings
+from django.core import exceptions
+
+import requests
+from rest_framework import status
+from urllib3.util import Retry
+
+logger = getLogger(__name__)
+
+adapter = requests.adapters.HTTPAdapter(
+    max_retries=Retry(
+        total=4,
+        backoff_factor=0.1,
+        status_forcelist=[500, 502],
+        allowed_methods=["PATCH"],
+    )
+)
+
+session = requests.Session()
+session.mount("http://", adapter)
+session.mount("https://", adapter)
+
+
+class DimailAPIClient:
+    """A dimail-API client."""
+
+    def get_headers(self, domain):
+        """Build header dict from domain object."""
+        # self.secret is the encoded basic auth, to request a new token from dimail-api
+        headers = {"Content-Type": "application/json"}
+
+        response = requests.get(
+            f"{settings.MAIL_PROVISIONING_API_URL}/token/",
+            headers={"Authorization": f"Basic {domain.secret}"},
+            timeout=status.HTTP_200_OK,
+        )
+
+        if response.json() == "{'detail': 'Permission denied'}":
+            raise exceptions.PermissionDenied(
+                "This secret does not allow for a new token."
+            )
+
+        if "access_token" in response.json():
+            headers["Authorization"] = f"Bearer {response.json()['access_token']}"
+
+        return headers
+
+    def send_mailbox_request(self, mailbox):
+        """Send a CREATE mailbox request to mail provisioning API."""
+
+        payload = {
+            "email": f"{mailbox.local_part}@{mailbox.domain}",
+            "givenName": mailbox.first_name,
+            "surName": mailbox.last_name,
+            "displayName": f"{mailbox.first_name} {mailbox.last_name}",
+        }
+
+        try:
+            response = session.post(
+                f"{settings.MAIL_PROVISIONING_API_URL}/domains/{mailbox.domain}/mailboxes/",
+                json=payload,
+                headers=self.get_headers(mailbox.domain),
+                verify=True,
+                timeout=10,
+            )
+        except requests.exceptions.ConnectionError as e:
+            logger.error(
+                "Connection error while trying to reach %s.",
+                settings.MAIL_PROVISIONING_API_URL,
+                exc_info=e,
+            )
+
+        if response.status_code == status.HTTP_201_CREATED:
+            extra = {"response": response.content.decode("utf-8")}
+            # This a temporary broken solution. Password will soon be sent
+            # from OX servers but their prod is not ready.
+            # In the meantime, we log mailbox info (including password !)
+            logger.info(
+                "Mailbox successfully created on domain %s",
+                mailbox.domain.name,
+                extra=extra,
+            )
+
+        return response

src/backend/people/settings.py

@@ -364,6 +364,13 @@ class Base(Configuration):
         environ_prefix=None,
     )
 
+    # mailboxes provisioning API
+    MAIL_PROVISIONING_API_URL = values.Value(
+        default="https://main.dev.ox.numerique.gouv.fr",
+        environ_name="MAIL_PROVISIONING_API_URL",
+        environ_prefix=None,
+    )
+
     # pylint: disable=invalid-name
     @property
     def ENVIRONMENT(self):

src/helm/env.d/dev/values.desk.yaml.gotmpl

@@ -50,6 +50,7 @@ backend:
     POSTGRES_USER: dinum
     POSTGRES_PASSWORD: pass
     REDIS_URL: redis://default:pass@redis-master:6379/1
+    MAIL_PROVISIONING_API_URL: "http://host.docker.internal:8000"
   command:
     - "gunicorn"
     - "-c"

src/helm/env.d/preprod/values.desk.yaml.gotmpl

@@ -84,6 +84,7 @@ backend:
       secretKeyRef:
         name: redis.redis.libre.sh
         key: url
+    MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr"
 
   createsuperuser:
     command:

src/helm/env.d/production/values.desk.yaml.gotmpl

@@ -84,6 +84,7 @@ backend:
       secretKeyRef:
         name: redis.redis.libre.sh
         key: url
+    MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr"
 
   createsuperuser:
     command:

src/helm/env.d/staging/values.desk.yaml.gotmpl

@@ -84,6 +84,8 @@ backend:
       secretKeyRef:
         name: redis.redis.libre.sh
         key: url
+    MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr"
+
 
   createsuperuser:
     command: