From f55cb3a8133f357f00e99b7729785d0b9fe40d13 Mon Sep 17 00:00:00 2001 From: Marie PUPO JEAMMET Date: Mon, 5 Aug 2024 12:20:44 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(mailboxes)=20add=20mail=20provisionin?= =?UTF-8?q?g=20api=20integration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want people to create new mailboxes in La Régie. This commit adds integration with intermediary dimail-api, which will in turn send our email creation request to Open-Xchange. --- .../mailbox_manager/api/serializers.py | 2 + src/backend/mailbox_manager/api/viewsets.py | 13 +- src/backend/mailbox_manager/models.py | 36 +++- .../mailboxes/test_api_mailboxes_create.py | 138 +++++++++++++++- .../tests/test_models_mailboxes.py | 156 +++++++++++++++++- src/backend/mailbox_manager/utils/__init__.py | 0 src/backend/mailbox_manager/utils/dimail.py | 88 ++++++++++ src/backend/people/settings.py | 7 + src/helm/env.d/dev/values.desk.yaml.gotmpl | 1 + .../env.d/preprod/values.desk.yaml.gotmpl | 1 + .../env.d/production/values.desk.yaml.gotmpl | 1 + .../env.d/staging/values.desk.yaml.gotmpl | 2 + 12 files changed, 432 insertions(+), 13 deletions(-) create mode 100644 src/backend/mailbox_manager/utils/__init__.py create mode 100644 src/backend/mailbox_manager/utils/dimail.py diff --git a/src/backend/mailbox_manager/api/serializers.py b/src/backend/mailbox_manager/api/serializers.py index a5230c2..85795a5 100644 --- a/src/backend/mailbox_manager/api/serializers.py +++ b/src/backend/mailbox_manager/api/serializers.py @@ -11,6 +11,8 @@ class MailboxSerializer(serializers.ModelSerializer): class Meta: model = models.Mailbox fields = ["id", "first_name", "last_name", "local_part", "secondary_email"] + # everything is actually read-only as we do not allow update for now + read_only_fields = ["id"] class MailDomainSerializer(serializers.ModelSerializer): diff --git a/src/backend/mailbox_manager/api/viewsets.py b/src/backend/mailbox_manager/api/viewsets.py index 716ed8b..15be4df 100644 --- a/src/backend/mailbox_manager/api/viewsets.py +++ b/src/backend/mailbox_manager/api/viewsets.py @@ -74,7 +74,18 @@ class MailBoxViewSet( mixins.ListModelMixin, viewsets.GenericViewSet, ): - """MailBox ViewSet""" + """MailBox ViewSet + + GET /api//mail-domains//mailboxes/ + Return a list of mailboxes on the domain + + POST /api//mail-domains//mailboxes/ with expected data: + - first_name: str + - last_name: str + - local_part: str + - secondary_email: str + Sends request to email provisioning API and returns newly created mailbox + """ permission_classes = [permissions.MailBoxPermission] serializer_class = serializers.MailboxSerializer diff --git a/src/backend/mailbox_manager/models.py b/src/backend/mailbox_manager/models.py index c7fa6e3..9db60c1 100644 --- a/src/backend/mailbox_manager/models.py +++ b/src/backend/mailbox_manager/models.py @@ -3,15 +3,15 @@ Declare and configure the models for the People additional application : mailbox """ from django.conf import settings -from django.core import validators -from django.core.exceptions import ValidationError -from django.db import models +from django.core import exceptions, validators +from django.db import models, transaction from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from core.models import BaseModel from mailbox_manager.enums import MailDomainRoleChoices, MailDomainStatusChoices +from mailbox_manager.utils.dimail import DimailAPIClient class MailDomain(BaseModel): @@ -138,8 +138,30 @@ class Mailbox(BaseModel): def __str__(self): return f"{self.local_part!s}@{self.domain.name:s}" - def save(self, *args, **kwargs): - self.full_clean() + def clean(self): + """Mailboxes can be created only on enabled domains, with a set secret.""" if self.domain.status != MailDomainStatusChoices.ENABLED: - raise ValidationError("You can create mailbox only for a domain enabled") - super().save(*args, **kwargs) + raise exceptions.ValidationError( + "You can create mailbox only for a domain enabled" + ) + + if not self.domain.secret: + raise exceptions.ValidationError( + "Please configure your domain's secret before creating any mailbox." + ) + + def save(self, *args, **kwargs): + """ + Override save function to fire a request on mailbox creation. + Modification is forbidden for now. + """ + self.full_clean() + + if self._state.adding: + with transaction.atomic(): + client = DimailAPIClient() + client.send_mailbox_request(self) + return super().save(*args, **kwargs) + + # Update is not implemented for now + raise NotImplementedError() diff --git a/src/backend/mailbox_manager/tests/api/mailboxes/test_api_mailboxes_create.py b/src/backend/mailbox_manager/tests/api/mailboxes/test_api_mailboxes_create.py index c447f81..4adf91b 100644 --- a/src/backend/mailbox_manager/tests/api/mailboxes/test_api_mailboxes_create.py +++ b/src/backend/mailbox_manager/tests/api/mailboxes/test_api_mailboxes_create.py @@ -2,7 +2,11 @@ Unit tests for the mailbox API """ +import json +import re + import pytest +import responses from rest_framework import status from rest_framework.test import APIClient @@ -130,12 +134,12 @@ def test_api_mailboxes__create_with_accent_success(role): mailbox_values = serializers.MailboxSerializer( factories.MailboxFactory.build(first_name="Aimé") ).data + response = client.post( f"/api/v1.0/mail-domains/{mail_domain.slug}/mailboxes/", mailbox_values, format="json", ) - assert response.status_code == status.HTTP_201_CREATED mailbox = models.Mailbox.objects.get() @@ -187,3 +191,135 @@ def test_api_mailboxes__create_administrator_missing_fields(): assert response.status_code == status.HTTP_400_BAD_REQUEST assert not models.Mailbox.objects.exists() assert response.json() == {"secondary_email": ["This field is required."]} + + +### SYNC TO PROVISIONING API + + +def test_api_mailboxes__unrelated_user_provisioning_api_not_called(): + """ + Provisioning API should not be called if an user tries + to create a mailbox on a domain they have no access to. + """ + domain = factories.MailDomainEnabledFactory() + + client = APIClient() + client.force_login(core_factories.UserFactory()) # user with no access + body_values = serializers.MailboxSerializer( + factories.MailboxFactory.build(domain=domain) + ).data + with responses.RequestsMock(): + # We add no simulated response in RequestsMock + # because we expected no "outside" calls to be made + response = client.post( + f"/api/v1.0/mail-domains/{domain.slug}/mailboxes/", + body_values, + format="json", + ) + # No exception raised by RequestsMock means no call was sent + # our API blocked the request before sending it + assert response.status_code == status.HTTP_403_FORBIDDEN + + +def test_api_mailboxes__domain_viewer_provisioning_api_not_called(): + """ + Provisioning API should not be called if a domain viewer tries + to create a mailbox on a domain they are not owner/admin of. + """ + access = factories.MailDomainAccessFactory( + domain=factories.MailDomainEnabledFactory(), + user=core_factories.UserFactory(), + role=enums.MailDomainRoleChoices.VIEWER, + ) + + client = APIClient() + client.force_login(access.user) + body_values = serializers.MailboxSerializer(factories.MailboxFactory.build()).data + with responses.RequestsMock(): + # We add no simulated response in RequestsMock + # because we expected no "outside" calls to be made + response = client.post( + f"/api/v1.0/mail-domains/{access.domain.slug}/mailboxes/", + body_values, + format="json", + ) + # No exception raised by RequestsMock means no call was sent + # our API blocked the request before sending it + assert response.status_code == status.HTTP_403_FORBIDDEN + + +@pytest.mark.parametrize( + "role", + [enums.MailDomainRoleChoices.ADMIN, enums.MailDomainRoleChoices.OWNER], +) +def test_api_mailboxes__domain_owner_or_admin_successful_creation_and_provisioning( + role, +): + """ + Domain owner/admin should be able to create mailboxes. + Provisioning API should be called when owner/admin makes a call. + Expected response contains new email and password. + """ + # creating all needed objects + access = factories.MailDomainAccessFactory(role=role) + + client = APIClient() + client.force_login(access.user) + mailbox_data = serializers.MailboxSerializer( + factories.MailboxFactory.build(domain=access.domain) + ).data + + with responses.RequestsMock() as rsps: + # Ensure successful response using "responses": + rsps.add( + rsps.GET, + re.compile(r".*/token/"), + body='{"access_token": "domain_owner_token"}', + status=status.HTTP_200_OK, + content_type="application/json", + ) + rsp = rsps.add( + rsps.POST, + re.compile(rf".*/domains/{access.domain.name}/mailboxes/"), + body=str( + { + "email": f"{mailbox_data['local_part']}@{access.domain.name}", + "password": "newpass", + "uuid": "uuid", + } + ), + status=status.HTTP_201_CREATED, + content_type="application/json", + ) + + response = client.post( + f"/api/v1.0/mail-domains/{access.domain.slug}/mailboxes/", + mailbox_data, + format="json", + ) + + # Checks payload sent to email-provisioning API + payload = json.loads(rsps.calls[1].request.body) + assert payload == { + "displayName": f"{mailbox_data['first_name']} {mailbox_data['last_name']}", + "email": f"{mailbox_data['local_part']}@{access.domain.name}", + "givenName": mailbox_data["first_name"], + "surName": mailbox_data["last_name"], + } + + # Checks response + assert response.status_code == status.HTTP_201_CREATED + assert rsp.call_count == 1 + + mailbox = models.Mailbox.objects.get() + assert response.json() == { + "id": str(mailbox.id), + "first_name": str(mailbox_data["first_name"]), + "last_name": str(mailbox_data["last_name"]), + "local_part": str(mailbox_data["local_part"]), + "secondary_email": str(mailbox_data["secondary_email"]), + } + assert mailbox.first_name == mailbox_data["first_name"] + assert mailbox.last_name == mailbox_data["last_name"] + assert mailbox.local_part == mailbox_data["local_part"] + assert mailbox.secondary_email == mailbox_data["secondary_email"] diff --git a/src/backend/mailbox_manager/tests/test_models_mailboxes.py b/src/backend/mailbox_manager/tests/test_models_mailboxes.py index a2656c2..9ed3b49 100644 --- a/src/backend/mailbox_manager/tests/test_models_mailboxes.py +++ b/src/backend/mailbox_manager/tests/test_models_mailboxes.py @@ -2,14 +2,35 @@ Unit tests for the mailbox model """ +import json +import logging +import re +from logging import Logger +from unittest import mock + from django.core.exceptions import ValidationError import pytest +import requests +import responses +from rest_framework import status +from urllib3.util import Retry -from mailbox_manager import enums, factories +from mailbox_manager import enums, factories, models +from mailbox_manager.api import serializers pytestmark = pytest.mark.django_db +logger = logging.getLogger(__name__) + +adapter = requests.adapters.HTTPAdapter( + max_retries=Retry( + total=4, + backoff_factor=0.1, + status_forcelist=[500, 502], + allowed_methods=["PATCH"], + ) +) # LOCAL PART FIELD @@ -33,11 +54,13 @@ def test_models_mailboxes__local_part_matches_expected_format(): """ factories.MailboxFactory(local_part="Marie-Jose.Perec+JO_2024") + # other special characters (such as "@" or "!") should raise a validation error with pytest.raises(ValidationError, match="Enter a valid value"): factories.MailboxFactory(local_part="mariejo@unnecessarydomain.com") - with pytest.raises(ValidationError, match="Enter a valid value"): - factories.MailboxFactory(local_part="!") + for character in ["!", "$", "%"]: + with pytest.raises(ValidationError, match="Enter a valid value"): + factories.MailboxFactory(local_part=f"marie{character}jo") def test_models_mailboxes__local_part_unique_per_domain(): @@ -59,6 +82,9 @@ def test_models_mailboxes__local_part_unique_per_domain(): # DOMAIN FIELD +session = requests.Session() +session.mount("http://", adapter) + def test_models_mailboxes__domain_must_be_a_maildomain_instance(): """The "domain" field should be an instance of MailDomain.""" @@ -72,7 +98,7 @@ def test_models_mailboxes__domain_must_be_a_maildomain_instance(): def test_models_mailboxes__domain_cannot_be_null(): """The "domain" field should not be null.""" - with pytest.raises(ValidationError, match="This field cannot be null"): + with pytest.raises(models.MailDomain.DoesNotExist, match="Mailbox has no domain."): factories.MailboxFactory(domain=None) @@ -126,3 +152,125 @@ def test_models_mailboxes__cannot_be_created_for_pending_maildomain(): # MailDomainFactory initializes a mail domain with default values, # so mail domain status is pending! factories.MailboxFactory(domain=factories.MailDomainFactory()) + + +### SYNC TO DIMAIL-API + + +def test_models_mailboxes__no_secret(): + """If no secret is declared on the domain, the function should raise an error.""" + domain = factories.MailDomainEnabledFactory(secret=None) + + with pytest.raises( + ValidationError, + match="Please configure your domain's secret before creating any mailbox.", + ): + factories.MailboxFactory(domain=domain) + + +def test_models_mailboxes__wrong_secret(): + """If domain secret is inaccurate, the function should raise an error.""" + + domain = factories.MailDomainEnabledFactory() + + with responses.RequestsMock() as rsps: + # Ensure successful response by scim provider using "responses": + rsps.add( + rsps.GET, + re.compile(r".*/token/"), + body='{"detail": "Permission denied"}', + status=status.HTTP_401_UNAUTHORIZED, + content_type="application/json", + ) + rsps.add( + rsps.POST, + re.compile(rf".*/domains/{domain.name}/mailboxes/"), + body='{"detail": "Permission denied"}', + status=status.HTTP_401_UNAUTHORIZED, + content_type="application/json", + ) + + mailbox = factories.MailboxFactory(domain=domain) + + # Payload sent to mailbox provider + payload = json.loads(rsps.calls[1].request.body) + assert payload == { + "displayName": f"{mailbox.first_name} {mailbox.last_name}", + "email": f"{mailbox.local_part}@{domain.name}", + "givenName": mailbox.first_name, + "surName": mailbox.last_name, + } + + +@mock.patch.object(Logger, "error") +@mock.patch.object(Logger, "info") +def test_models_mailboxes__create_mailbox_success(mock_info, mock_error): + """Creating a mailbox sends the expected information and get expected response before saving.""" + domain = factories.MailDomainEnabledFactory() + + # generate mailbox data before mailbox, to mock responses + mailbox_data = serializers.MailboxSerializer( + factories.MailboxFactory.build(domain=domain) + ).data + + with responses.RequestsMock() as rsps: + # Ensure successful response using "responses": + rsps.add( + rsps.GET, + re.compile(r".*/token/"), + body='{"access_token": "domain_owner_token"}', + status=status.HTTP_200_OK, + content_type="application/json", + ) + rsps.add( + rsps.POST, + re.compile(rf".*/domains/{domain.name}/mailboxes/"), + body=str( + { + "email": f"{mailbox_data['local_part']}@{domain.name}", + "password": "newpass", + "uuid": "uuid", + } + ), + status=status.HTTP_201_CREATED, + content_type="application/json", + ) + + mailbox = factories.MailboxFactory( + local_part=mailbox_data["local_part"], domain=domain + ) + + # Check headers + headers = rsps.calls[1].request.headers + # assert "Authorization" not in headers + assert headers["Content-Type"] == "application/json" + + # Payload sent to mailbox provider + payload = json.loads(rsps.calls[1].request.body) + assert payload == { + "displayName": f"{mailbox.first_name} {mailbox.last_name}", + "email": f"{mailbox.local_part}@{domain.name}", + "givenName": mailbox.first_name, + "surName": mailbox.last_name, + } + + # Logger + assert not mock_error.called + assert mock_info.call_count == 1 + assert mock_info.call_args_list[0][0] == ( + "Mailbox successfully created on domain %s", + domain.name, + ) + assert mock_info.call_args_list[0][1] == ( + { + "extra": { + "response": str( + { + "email": f"{mailbox.local_part}@{domain.name}", + "password": "newpass", + "uuid": "uuid", + } + ) + } + } + ) diff --git a/src/backend/mailbox_manager/utils/__init__.py b/src/backend/mailbox_manager/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/mailbox_manager/utils/dimail.py b/src/backend/mailbox_manager/utils/dimail.py new file mode 100644 index 0000000..44bde44 --- /dev/null +++ b/src/backend/mailbox_manager/utils/dimail.py @@ -0,0 +1,88 @@ +"""A minimalist client to synchronize with mailbox provisioning API.""" + +from logging import getLogger + +from django.conf import settings +from django.core import exceptions + +import requests +from rest_framework import status +from urllib3.util import Retry + +logger = getLogger(__name__) + +adapter = requests.adapters.HTTPAdapter( + max_retries=Retry( + total=4, + backoff_factor=0.1, + status_forcelist=[500, 502], + allowed_methods=["PATCH"], + ) +) + +session = requests.Session() +session.mount("http://", adapter) +session.mount("https://", adapter) + + +class DimailAPIClient: + """A dimail-API client.""" + + def get_headers(self, domain): + """Build header dict from domain object.""" + # self.secret is the encoded basic auth, to request a new token from dimail-api + headers = {"Content-Type": "application/json"} + + response = requests.get( + f"{settings.MAIL_PROVISIONING_API_URL}/token/", + headers={"Authorization": f"Basic {domain.secret}"}, + timeout=status.HTTP_200_OK, + ) + + if response.json() == "{'detail': 'Permission denied'}": + raise exceptions.PermissionDenied( + "This secret does not allow for a new token." + ) + + if "access_token" in response.json(): + headers["Authorization"] = f"Bearer {response.json()['access_token']}" + + return headers + + def send_mailbox_request(self, mailbox): + """Send a CREATE mailbox request to mail provisioning API.""" + + payload = { + "email": f"{mailbox.local_part}@{mailbox.domain}", + "givenName": mailbox.first_name, + "surName": mailbox.last_name, + "displayName": f"{mailbox.first_name} {mailbox.last_name}", + } + + try: + response = session.post( + f"{settings.MAIL_PROVISIONING_API_URL}/domains/{mailbox.domain}/mailboxes/", + json=payload, + headers=self.get_headers(mailbox.domain), + verify=True, + timeout=10, + ) + except requests.exceptions.ConnectionError as e: + logger.error( + "Connection error while trying to reach %s.", + settings.MAIL_PROVISIONING_API_URL, + exc_info=e, + ) + + if response.status_code == status.HTTP_201_CREATED: + extra = {"response": response.content.decode("utf-8")} + # This a temporary broken solution. Password will soon be sent + # from OX servers but their prod is not ready. + # In the meantime, we log mailbox info (including password !) + logger.info( + "Mailbox successfully created on domain %s", + mailbox.domain.name, + extra=extra, + ) + + return response diff --git a/src/backend/people/settings.py b/src/backend/people/settings.py index 09d5169..1cb44e5 100755 --- a/src/backend/people/settings.py +++ b/src/backend/people/settings.py @@ -364,6 +364,13 @@ class Base(Configuration): environ_prefix=None, ) + # mailboxes provisioning API + MAIL_PROVISIONING_API_URL = values.Value( + default="https://main.dev.ox.numerique.gouv.fr", + environ_name="MAIL_PROVISIONING_API_URL", + environ_prefix=None, + ) + # pylint: disable=invalid-name @property def ENVIRONMENT(self): diff --git a/src/helm/env.d/dev/values.desk.yaml.gotmpl b/src/helm/env.d/dev/values.desk.yaml.gotmpl index 37d06ec..6b4587c 100644 --- a/src/helm/env.d/dev/values.desk.yaml.gotmpl +++ b/src/helm/env.d/dev/values.desk.yaml.gotmpl @@ -50,6 +50,7 @@ backend: POSTGRES_USER: dinum POSTGRES_PASSWORD: pass REDIS_URL: redis://default:pass@redis-master:6379/1 + MAIL_PROVISIONING_API_URL: "http://host.docker.internal:8000" command: - "gunicorn" - "-c" diff --git a/src/helm/env.d/preprod/values.desk.yaml.gotmpl b/src/helm/env.d/preprod/values.desk.yaml.gotmpl index 1769489..1441af9 100644 --- a/src/helm/env.d/preprod/values.desk.yaml.gotmpl +++ b/src/helm/env.d/preprod/values.desk.yaml.gotmpl @@ -84,6 +84,7 @@ backend: secretKeyRef: name: redis.redis.libre.sh key: url + MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr" createsuperuser: command: diff --git a/src/helm/env.d/production/values.desk.yaml.gotmpl b/src/helm/env.d/production/values.desk.yaml.gotmpl index ec292d3..d244ee5 100644 --- a/src/helm/env.d/production/values.desk.yaml.gotmpl +++ b/src/helm/env.d/production/values.desk.yaml.gotmpl @@ -84,6 +84,7 @@ backend: secretKeyRef: name: redis.redis.libre.sh key: url + MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr" createsuperuser: command: diff --git a/src/helm/env.d/staging/values.desk.yaml.gotmpl b/src/helm/env.d/staging/values.desk.yaml.gotmpl index 5a1c4e0..f683205 100644 --- a/src/helm/env.d/staging/values.desk.yaml.gotmpl +++ b/src/helm/env.d/staging/values.desk.yaml.gotmpl @@ -84,6 +84,8 @@ backend: secretKeyRef: name: redis.redis.libre.sh key: url + MAIL_PROVISIONING_API_URL: "https://main.dev.ox.numerique.gouv.fr" + createsuperuser: command: