From 8a8a1460e58ce0adfc45b82e05a0b68f314ecffc Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 02:31:50 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(api)=20add=20API=20route=20to=20fetch?= =?UTF-8?q?=20document=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows API users to process document content, enabling the use of Docs as a headless CMS for instance, or any kind of document processing. Fixes #1206. --- CHANGELOG.md | 4 + docker/auth/realm.json | 2 +- src/backend/core/api/serializers.py | 6 +- src/backend/core/api/viewsets.py | 64 ++++++ src/backend/core/models.py | 1 + .../core/services/converter_services.py | 53 ----- .../core/services/yprovider_services.py | 80 +++++++ .../documents/test_api_documents_content.py | 161 ++++++++++++++ .../test_api_documents_create_for_owner.py | 6 +- .../documents/test_api_documents_retrieve.py | 5 + .../documents/test_api_documents_trashbin.py | 1 + .../core/tests/test_models_documents.py | 8 + ...py => test_services_yprovider_services.py} | 82 +++++-- .../y-provider/__tests__/content.test.ts | 207 ++++++++++++++++++ src/frontend/servers/y-provider/package.json | 2 +- .../y-provider/src/handlers/contentHandler.ts | 71 ++++++ .../servers/y-provider/src/handlers/index.ts | 1 + src/frontend/servers/y-provider/src/routes.ts | 1 + .../y-provider/src/servers/appServer.ts | 6 + 19 files changed, 687 insertions(+), 74 deletions(-) delete mode 100644 src/backend/core/services/converter_services.py create mode 100644 src/backend/core/services/yprovider_services.py create mode 100644 src/backend/core/tests/documents/test_api_documents_content.py rename src/backend/core/tests/{test_services_converter_services.py => test_services_yprovider_services.py} (54%) create mode 100644 src/frontend/servers/y-provider/__tests__/content.test.ts create mode 100644 src/frontend/servers/y-provider/src/handlers/contentHandler.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a2d78c..b8525204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to ## [Unreleased] +### Added + +- ✨(api) add API route to fetch document content #1206 + ### Changed - 🔒️(backend) configure throttle on every viewsets #1343 diff --git a/docker/auth/realm.json b/docker/auth/realm.json index db5f1be0..06ab6e55 100644 --- a/docker/auth/realm.json +++ b/docker/auth/realm.json @@ -26,7 +26,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "external", + "sslRequired": "none", "registrationAllowed": true, "registrationEmailAsUsername": false, "rememberMe": true, diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 9b843908..df06518d 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -15,9 +15,9 @@ from rest_framework import serializers from core import choices, enums, models, utils, validators from core.services.ai_services import AI_ACTIONS -from core.services.converter_services import ( +from core.services.yprovider_services import ( ConversionError, - YdocConverter, + YProviderAPI, ) @@ -451,7 +451,7 @@ class ServerCreateDocumentSerializer(serializers.Serializer): language = user.language or language try: - document_content = YdocConverter().convert(validated_data["content"]) + document_content = YProviderAPI().convert(validated_data["content"]) except ConversionError as err: raise serializers.ValidationError( {"content": ["Could not convert content"]} diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 19841185..58bf61c0 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -37,6 +37,15 @@ from rest_framework.permissions import AllowAny from core import authentication, choices, enums, models from core.services.ai_services import AIService from core.services.collaboration_services import CollaborationService +from core.services.yprovider_services import ( + ServiceUnavailableError as YProviderServiceUnavailableError, +) +from core.services.yprovider_services import ( + ValidationError as YProviderValidationError, +) +from core.services.yprovider_services import ( + YProviderAPI, +) from core.tasks.mail import send_ask_for_access_mail from core.utils import extract_attachments, filter_descendants @@ -1494,6 +1503,61 @@ class DocumentViewSet( status=status.HTTP_400_BAD_REQUEST, ) + @drf.decorators.action( + detail=True, + methods=["get"], + url_path="content", + name="Get document content in different formats", + ) + def content(self, request, pk=None): + """ + Retrieve document content in different formats (JSON, Markdown, HTML). + + Query parameters: + - content_format: The desired output format (json, markdown, html) + + Returns: + JSON response with content in the specified format. + """ + + document = self.get_object() + + content_format = request.query_params.get("content_format", "json").lower() + if content_format not in {"json", "markdown", "html"}: + raise drf.exceptions.ValidationError( + "Invalid format. Must be one of: json, markdown, html" + ) + + # Get the base64 content from the document + content = None + base64_content = document.content + if base64_content is not None: + # Convert using the y-provider service + try: + yprovider = YProviderAPI() + result = yprovider.content(base64_content, content_format) + content = result["content"] + except YProviderValidationError as e: + return drf_response.Response( + {"error": str(e)}, status=status.HTTP_400_BAD_REQUEST + ) + except YProviderServiceUnavailableError as e: + logger.error("Error getting content for document %s: %s", pk, e) + return drf_response.Response( + {"error": "Failed to get document content"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return drf_response.Response( + { + "id": str(document.id), + "title": document.title, + "content": content, + "created_at": document.created_at, + "updated_at": document.updated_at, + } + ) + class DocumentAccessViewSet( ResourceAccessViewsetMixin, diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 6ccca08d..9b8e13fe 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -783,6 +783,7 @@ class Document(MP_Node, BaseModel): "children_list": can_get, "children_create": can_create_children, "collaboration_auth": can_get, + "content": can_get, "cors_proxy": can_get, "descendants": can_get, "destroy": can_destroy, diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py deleted file mode 100644 index d6a6dbf4..00000000 --- a/src/backend/core/services/converter_services.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Converter services.""" - -from base64 import b64encode - -from django.conf import settings - -import requests - - -class ConversionError(Exception): - """Base exception for conversion-related errors.""" - - -class ValidationError(ConversionError): - """Raised when the input validation fails.""" - - -class ServiceUnavailableError(ConversionError): - """Raised when the conversion service is unavailable.""" - - -class YdocConverter: - """Service class for conversion-related operations.""" - - @property - def auth_header(self): - """Build microservice authentication header.""" - # Note: Yprovider microservice accepts only raw token, which is not recommended - return f"Bearer {settings.Y_PROVIDER_API_KEY}" - - def convert(self, text): - """Convert a Markdown text into our internal format using an external microservice.""" - - if not text: - raise ValidationError("Input text cannot be empty") - - try: - response = requests.post( - f"{settings.Y_PROVIDER_API_BASE_URL}{settings.CONVERSION_API_ENDPOINT}/", - data=text, - headers={ - "Authorization": self.auth_header, - "Content-Type": "text/markdown", - }, - timeout=settings.CONVERSION_API_TIMEOUT, - verify=settings.CONVERSION_API_SECURE, - ) - response.raise_for_status() - return b64encode(response.content).decode("utf-8") - except requests.RequestException as err: - raise ServiceUnavailableError( - "Failed to connect to conversion service", - ) from err diff --git a/src/backend/core/services/yprovider_services.py b/src/backend/core/services/yprovider_services.py new file mode 100644 index 00000000..09b80f8c --- /dev/null +++ b/src/backend/core/services/yprovider_services.py @@ -0,0 +1,80 @@ +"""Y-Provider API services.""" + +import json +from base64 import b64encode + +from django.conf import settings + +import requests + + +class ConversionError(Exception): + """Base exception for conversion-related errors.""" + + +class ValidationError(ConversionError): + """Raised when the input validation fails.""" + + +class ServiceUnavailableError(ConversionError): + """Raised when the conversion service is unavailable.""" + + +class YProviderAPI: + """Service class for Y-Provider API operations.""" + + @property + def auth_header(self): + """Build microservice authentication header.""" + # Note: Yprovider microservice accepts only raw token, which is not recommended + return f"Bearer {settings.Y_PROVIDER_API_KEY}" + + def _request(self, url, data, content_type): + """Make a request to the Y-Provider API.""" + response = requests.post( + url, + data=data, + headers={ + "Authorization": self.auth_header, + "Content-Type": content_type, + }, + timeout=settings.CONVERSION_API_TIMEOUT, + verify=settings.CONVERSION_API_SECURE, + ) + response.raise_for_status() + return response + + def convert(self, text): + """Convert a Markdown text into our internal format using an external microservice.""" + + if not text: + raise ValidationError("Input text cannot be empty") + + try: + response = self._request( + f"{settings.Y_PROVIDER_API_BASE_URL}{settings.CONVERSION_API_ENDPOINT}/", + text, + "text/markdown", + ) + return b64encode(response.content).decode("utf-8") + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to backend service", + ) from err + + def content(self, base64_content, format_type): + """Convert base64 Yjs content to different formats using the y-provider service.""" + + if not base64_content: + raise ValidationError("Input content cannot be empty") + + data = json.dumps({"content": base64_content, "format": format_type}) + try: + response = self._request( + f"{settings.Y_PROVIDER_API_BASE_URL}content/", data, "application/json" + ) + return response.json() + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to backend service", + ) from err diff --git a/src/backend/core/tests/documents/test_api_documents_content.py b/src/backend/core/tests/documents/test_api_documents_content.py new file mode 100644 index 00000000..62b215c9 --- /dev/null +++ b/src/backend/core/tests/documents/test_api_documents_content.py @@ -0,0 +1,161 @@ +""" +Tests for Documents API endpoint in impress's core app: content +""" + +from unittest.mock import patch + +import pytest +import requests +from rest_framework import status +from rest_framework.test import APIClient + +from core import factories + +pytestmark = pytest.mark.django_db + + +@pytest.mark.parametrize( + "reach, role", + [ + ("public", "reader"), + ("public", "editor"), + ], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_public(mock_content, reach, role): + """Anonymous users should be allowed to access content of public documents.""" + document = factories.DocumentFactory(link_reach=reach, link_role=role) + mock_content.return_value = {"content": {"some": "data"}} + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == {"some": "data"} + mock_content.assert_called_once_with(document.content, "json") + + +@pytest.mark.parametrize( + "reach, doc_role, user_role", + [ + ("restricted", "reader", "reader"), + ("restricted", "reader", "editor"), + ("restricted", "reader", "administrator"), + ("restricted", "reader", "owner"), + ("restricted", "editor", "reader"), + ("restricted", "editor", "editor"), + ("restricted", "editor", "administrator"), + ("restricted", "editor", "owner"), + ("authenticated", "reader", None), + ("authenticated", "editor", None), + ], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_not_public(mock_content, reach, doc_role, user_role): + """Authenticated users need access to get non-public document content.""" + user = factories.UserFactory() + document = factories.DocumentFactory(link_reach=reach, link_role=doc_role) + mock_content.return_value = {"content": {"some": "data"}} + + # First anonymous request should fail + client = APIClient() + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + mock_content.assert_not_called() + + # Login and try again + client.force_login(user) + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + # If restricted, we still should not have access + if user_role is not None: + assert response.status_code == status.HTTP_403_FORBIDDEN + mock_content.assert_not_called() + + # Create an access as a reader. This should unlock the access. + factories.UserDocumentAccessFactory( + document=document, user=user, role=user_role + ) + + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == {"some": "data"} + mock_content.assert_called_once_with(document.content, "json") + + +@pytest.mark.parametrize( + "content_format", + ["markdown", "html", "json"], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_format(mock_content, content_format): + """Test that the content endpoint returns a specific format.""" + document = factories.DocumentFactory(link_reach="public") + mock_content.return_value = {"content": "whatever"} + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/content/?content_format={content_format}" + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == "whatever" + mock_content.assert_called_once_with(document.content, content_format) + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_invalid_format(mock_request): + """Test that the content endpoint rejects invalid formats.""" + document = factories.DocumentFactory(link_reach="public") + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/content/?content_format=invalid" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + mock_request.assert_not_called() + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_yservice_error(mock_request): + """Test that service errors are handled properly.""" + document = factories.DocumentFactory(link_reach="public") + mock_request.side_effect = requests.RequestException() + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + mock_request.assert_called_once() + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_nonexistent_document(mock_request): + """Test that accessing a nonexistent document returns 404.""" + client = APIClient() + response = client.get( + "/api/v1.0/documents/00000000-0000-0000-0000-000000000000/content/" + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + mock_request.assert_not_called() + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_empty_document(mock_request): + """Test that accessing an empty document returns empty content.""" + document = factories.DocumentFactory(link_reach="public", content="") + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] is None + mock_request.assert_not_called() diff --git a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py index 346fe407..f011559b 100644 --- a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py +++ b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py @@ -16,16 +16,16 @@ from rest_framework.test import APIClient from core import factories from core.api.serializers import ServerCreateDocumentSerializer from core.models import Document, Invitation, User -from core.services.converter_services import ConversionError, YdocConverter +from core.services.yprovider_services import ConversionError, YProviderAPI pytestmark = pytest.mark.django_db @pytest.fixture def mock_convert_md(): - """Mock YdocConverter.convert to return a converted content.""" + """Mock YProviderAPI.convert to return a converted content.""" with patch.object( - YdocConverter, + YProviderAPI, "convert", return_value="Converted document content", ) as mock: diff --git a/src/backend/core/tests/documents/test_api_documents_retrieve.py b/src/backend/core/tests/documents/test_api_documents_retrieve.py index 5b8b1403..d1f8e1f0 100644 --- a/src/backend/core/tests/documents/test_api_documents_retrieve.py +++ b/src/backend/core/tests/documents/test_api_documents_retrieve.py @@ -37,6 +37,7 @@ def test_api_documents_retrieve_anonymous_public_standalone(): "children_list": True, "collaboration_auth": True, "cors_proxy": True, + "content": True, "descendants": True, "destroy": False, "duplicate": False, @@ -113,6 +114,7 @@ def test_api_documents_retrieve_anonymous_public_parent(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": False, # Anonymous user can't favorite a document even with read access @@ -218,6 +220,7 @@ def test_api_documents_retrieve_authenticated_unrelated_public_or_authenticated( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -300,6 +303,7 @@ def test_api_documents_retrieve_authenticated_public_or_authenticated_parent(rea "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -494,6 +498,7 @@ def test_api_documents_retrieve_authenticated_related_parent(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": access.role in ["administrator", "owner"], "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/documents/test_api_documents_trashbin.py b/src/backend/core/tests/documents/test_api_documents_trashbin.py index 25bc1cda..28ea6a01 100644 --- a/src/backend/core/tests/documents/test_api_documents_trashbin.py +++ b/src/backend/core/tests/documents/test_api_documents_trashbin.py @@ -81,6 +81,7 @@ def test_api_documents_trashbin_format(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": True, "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/test_models_documents.py b/src/backend/core/tests/test_models_documents.py index 58687092..cc760aff 100644 --- a/src/backend/core/tests/test_models_documents.py +++ b/src/backend/core/tests/test_models_documents.py @@ -161,6 +161,7 @@ def test_models_documents_get_abilities_forbidden( "collaboration_auth": False, "descendants": False, "cors_proxy": False, + "content": False, "destroy": False, "duplicate": False, "favorite": False, @@ -224,6 +225,7 @@ def test_models_documents_get_abilities_reader( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": is_authenticated, "favorite": is_authenticated, @@ -289,6 +291,7 @@ def test_models_documents_get_abilities_editor( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": is_authenticated, "favorite": is_authenticated, @@ -343,6 +346,7 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": True, "duplicate": True, "favorite": True, @@ -394,6 +398,7 @@ def test_models_documents_get_abilities_administrator(django_assert_num_queries) "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -448,6 +453,7 @@ def test_models_documents_get_abilities_editor_user(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -509,6 +515,7 @@ def test_models_documents_get_abilities_reader_user( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -568,6 +575,7 @@ def test_models_documents_get_abilities_preset_role(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_yprovider_services.py similarity index 54% rename from src/backend/core/tests/test_services_converter_services.py rename to src/backend/core/tests/test_services_yprovider_services.py index 01773f85..603c2207 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_yprovider_services.py @@ -1,28 +1,29 @@ -"""Test converter services.""" +"""Test y-provider services.""" +import json from base64 import b64decode from unittest.mock import MagicMock, patch import pytest import requests -from core.services.converter_services import ( +from core.services.yprovider_services import ( ServiceUnavailableError, ValidationError, - YdocConverter, + YProviderAPI, ) def test_auth_header(settings): """Test authentication header generation.""" settings.Y_PROVIDER_API_KEY = "test-key" - converter = YdocConverter() + converter = YProviderAPI() assert converter.auth_header == "Bearer test-key" def test_convert_empty_text(): """Should raise ValidationError when text is empty.""" - converter = YdocConverter() + converter = YProviderAPI() with pytest.raises(ValidationError, match="Input text cannot be empty"): converter.convert("") @@ -30,13 +31,13 @@ def test_convert_empty_text(): @patch("requests.post") def test_convert_service_unavailable(mock_post): """Should raise ServiceUnavailableError when service is unavailable.""" - converter = YdocConverter() + converter = YProviderAPI() mock_post.side_effect = requests.RequestException("Connection error") with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") @@ -44,7 +45,7 @@ def test_convert_service_unavailable(mock_post): @patch("requests.post") def test_convert_http_error(mock_post): """Should raise ServiceUnavailableError when HTTP error occurs.""" - converter = YdocConverter() + converter = YProviderAPI() mock_response = MagicMock() mock_response.raise_for_status.side_effect = requests.HTTPError("HTTP Error") @@ -52,7 +53,7 @@ def test_convert_http_error(mock_post): with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") @@ -67,7 +68,7 @@ def test_convert_full_integration(mock_post, settings): settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_CONTENT_FIELD = "content" - converter = YdocConverter() + converter = YProviderAPI() expected_content = b"converted content" mock_response = MagicMock() @@ -93,20 +94,75 @@ def test_convert_full_integration(mock_post, settings): @patch("requests.post") def test_convert_timeout(mock_post): """Should raise ServiceUnavailableError when request times out.""" - converter = YdocConverter() + converter = YProviderAPI() mock_post.side_effect = requests.Timeout("Request timed out") with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") def test_convert_none_input(): """Should raise ValidationError when input is None.""" - converter = YdocConverter() + converter = YProviderAPI() with pytest.raises(ValidationError, match="Input text cannot be empty"): converter.convert(None) + + +def test_content_empty_content(): + """Should raise ValidationError when content is empty.""" + converter = YProviderAPI() + with pytest.raises(ValidationError, match="Input content cannot be empty"): + converter.content("", "markdown") + + +@patch("requests.post") +def test_content_service_unavailable(mock_post): + """Should raise ServiceUnavailableError when service is unavailable.""" + converter = YProviderAPI() + + mock_post.side_effect = requests.RequestException("Connection error") + + with pytest.raises( + ServiceUnavailableError, + match="Failed to connect to backend service", + ): + converter.content("test_content", "markdown") + + +@patch("requests.post") +def test_content_success(mock_post, settings): + """Test successful content fetch.""" + settings.Y_PROVIDER_API_BASE_URL = "http://test.com/api/" + settings.Y_PROVIDER_API_KEY = "test-key" + settings.CONVERSION_API_TIMEOUT = 5 + settings.CONVERSION_API_SECURE = False + + converter = YProviderAPI() + + expected_response = { + "content": "# Test Document\n\nThis is test content.", + "format": "markdown", + } + mock_response = MagicMock() + mock_response.json.return_value = expected_response + mock_response.raise_for_status.return_value = None + mock_post.return_value = mock_response + + result = converter.content("test_content", "markdown") + + assert result == expected_response + mock_post.assert_called_once_with( + "http://test.com/api/content/", + data=json.dumps({"content": "test_content", "format": "markdown"}), + headers={ + "Authorization": "Bearer test-key", + "Content-Type": "application/json", + }, + timeout=5, + verify=False, + ) diff --git a/src/frontend/servers/y-provider/__tests__/content.test.ts b/src/frontend/servers/y-provider/__tests__/content.test.ts new file mode 100644 index 00000000..555b453a --- /dev/null +++ b/src/frontend/servers/y-provider/__tests__/content.test.ts @@ -0,0 +1,207 @@ +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import request from 'supertest'; +import { describe, expect, test, vi } from 'vitest'; +import * as Y from 'yjs'; + +vi.mock('../src/env', async (importOriginal) => { + return { + ...(await importOriginal()), + COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', + Y_PROVIDER_API_KEY: 'yprovider-api-key', + }; +}); + +import { initApp } from '@/servers'; + +import { + Y_PROVIDER_API_KEY as apiKey, + COLLABORATION_SERVER_ORIGIN as origin, +} from '../src/env'; + +console.error = vi.fn(); + +describe('Content API Tests', () => { + test('POST /api/content with incorrect API key responds with 401', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', 'wrong-api-key') + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', // base64 for "test" + format: 'json', + }); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + test('POST /api/content with incorrect Bearer token responds with 401', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', 'Bearer test-secret-api-key') + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', // base64 for "test" + format: 'json', + }); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + test('POST /api/content with missing content parameter', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + format: 'json', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid request: missing content', + }); + }); + + test('POST /api/content with empty content', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: '', + format: 'json', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid request: missing content', + }); + }); + + test('POST /api/content with missing format parameter', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid format. Must be one of: json, markdown, html', + }); + }); + + test('POST /api/content with invalid format', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', + format: 'invalid', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid format. Must be one of: json, markdown, html', + }); + }); + + test.each([ + { authHeader: `Bearer ${apiKey}`, format: 'json' }, + { authHeader: `Bearer ${apiKey}`, format: 'markdown' }, + { authHeader: `Bearer ${apiKey}`, format: 'html' }, + ])( + 'POST /api/content with correct content and format $format with Authorization: $authHeader', + async ({ authHeader, format }) => { + const app = initApp(); + + // Create a simple Yjs document for testing using BlockNote + const editor = ServerBlockNoteEditor.create(); + const markdownContent = '# Test Document\n\nThis is test content.'; + const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const yjsUpdate = Y.encodeStateAsUpdate(yDocument); + const base64Content = Buffer.from(yjsUpdate).toString('base64'); + + const response = await request(app) + .post('/api/content') + .set('Origin', origin) + .set('Authorization', authHeader) + .set('content-type', 'application/json') + .send({ + content: base64Content, + format: format, + }); + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('content'); + expect(response.body).toHaveProperty('format', format); + + // Verify the content based on format + if (format === 'json') { + const parsedContent = response.body.content; + expect(Array.isArray(parsedContent)).toBe(true); + expect(parsedContent.length).toBe(2); + expect(parsedContent[0].type).toBe('heading'); + expect(parsedContent[1].type).toBe('paragraph'); + expect(parsedContent[0].content[0].type).toBe('text'); + expect(parsedContent[0].content[0].text).toBe('Test Document'); + expect(parsedContent[1].content[0].type).toBe('text'); + expect(parsedContent[1].content[0].text).toBe('This is test content.'); + } else if (format === 'markdown') { + expect(typeof response.body.content).toBe('string'); + expect(response.body.content.trim()).toBe(markdownContent); + } else if (format === 'html') { + expect(typeof response.body.content).toBe('string'); + expect(response.body.content).toBe( + '

Test Document

This is test content.

', + ); + } + }, + ); + + test('POST /api/content with invalid base64 content returns 500', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'invalid-base64-content!@#', + format: 'json', + }); + + expect(response.status).toBe(500); + expect(response.body).toStrictEqual({ + error: 'An error occurred during conversion', + }); + }); +}); diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index 759578db..eb86b204 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -10,7 +10,7 @@ "dev": "cross-env COLLABORATION_LOGGING=true && nodemon --config nodemon.json", "start": "node ./dist/start-server.js", "lint": "eslint . --ext .ts", - "test": "vitest --run" + "test": "vitest --run --disable-console-intercept" }, "engines": { "node": ">=22" diff --git a/src/frontend/servers/y-provider/src/handlers/contentHandler.ts b/src/frontend/servers/y-provider/src/handlers/contentHandler.ts new file mode 100644 index 00000000..671ec89d --- /dev/null +++ b/src/frontend/servers/y-provider/src/handlers/contentHandler.ts @@ -0,0 +1,71 @@ +import { PartialBlock } from '@blocknote/core'; +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import { Request, Response } from 'express'; +import * as Y from 'yjs'; + +import { logger } from '@/utils'; + +interface ErrorResponse { + error: string; +} + +interface ContentRequest { + content: string; + format: string; +} + +const editor = ServerBlockNoteEditor.create(); + +export const contentHandler = async ( + req: Request, + res: Response, +) => { + const { content, format } = req.body; + + if (!content) { + res.status(400).json({ error: 'Invalid request: missing content' }); + return; + } + + if (!format || !['json', 'markdown', 'html'].includes(format)) { + res + .status(400) + .json({ error: 'Invalid format. Must be one of: json, markdown, html' }); + return; + } + + try { + // Decode base64 content to Uint8Array + const uint8Array = new Uint8Array(Buffer.from(content, 'base64')); + + // Create Yjs document and apply the update + const yDocument = new Y.Doc(); + Y.applyUpdate(yDocument, uint8Array); + + // Convert to blocks + const blocks = editor.yDocToBlocks(yDocument, 'document-store'); + + let result: string | object | null; + + if (!blocks || blocks.length === 0) { + result = null; + } else if (format === 'json') { + result = blocks; + } else if (format === 'markdown') { + result = await editor.blocksToMarkdownLossy(blocks as PartialBlock[]); + } else if (format === 'html') { + result = await editor.blocksToHTMLLossy(blocks as PartialBlock[]); + } else { + res.status(400).json({ error: 'Unsupported format' }); + return; + } + + res.status(200).json({ + content: result, + format: format, + }); + } catch (e) { + logger('content conversion failed:', e); + res.status(500).json({ error: 'An error occurred during conversion' }); + } +}; diff --git a/src/frontend/servers/y-provider/src/handlers/index.ts b/src/frontend/servers/y-provider/src/handlers/index.ts index 26b0ebed..19666250 100644 --- a/src/frontend/servers/y-provider/src/handlers/index.ts +++ b/src/frontend/servers/y-provider/src/handlers/index.ts @@ -1,4 +1,5 @@ export * from './collaborationResetConnectionsHandler'; export * from './collaborationWSHandler'; export * from './convertHandler'; +export * from './contentHandler'; export * from './getDocumentConnectionInfoHandler'; diff --git a/src/frontend/servers/y-provider/src/routes.ts b/src/frontend/servers/y-provider/src/routes.ts index 5bb73365..4c5d8ca2 100644 --- a/src/frontend/servers/y-provider/src/routes.ts +++ b/src/frontend/servers/y-provider/src/routes.ts @@ -2,5 +2,6 @@ export const routes = { COLLABORATION_WS: '/collaboration/ws/', COLLABORATION_RESET_CONNECTIONS: '/collaboration/api/reset-connections/', CONVERT: '/api/convert/', + CONTENT: '/api/content/', COLLABORATION_GET_CONNECTIONS: '/collaboration/api/get-connections/', }; diff --git a/src/frontend/servers/y-provider/src/servers/appServer.ts b/src/frontend/servers/y-provider/src/servers/appServer.ts index 0c355fee..ae008b36 100644 --- a/src/frontend/servers/y-provider/src/servers/appServer.ts +++ b/src/frontend/servers/y-provider/src/servers/appServer.ts @@ -8,6 +8,7 @@ import expressWebsockets from 'express-ws'; import { collaborationResetConnectionsHandler, collaborationWSHandler, + contentHandler, convertHandler, getDocumentConnectionInfoHandler, } from '@/handlers'; @@ -61,6 +62,11 @@ export const initApp = () => { convertHandler, ); + /** + * Route to convert base64 Yjs content to different formats + */ + app.post(routes.CONTENT, httpSecurity, express.json(), contentHandler); + Sentry.setupExpressErrorHandler(app); app.get('/ping', (req, res) => {