♻️(frontend) raw payloads on convert endpoint

Accept raw payload on convert-endpoint and respond with raw Yjs payload

This change replaces Base64-encoded I/O with direct binary streaming,
yielding several benefits:
- **Network efficiency**: Eliminates the ~33% size inflation of Base64,
cutting bandwidth and latency.
- **Memory savings**: Enables piping DOCX (already compressed) buffers
straight to DocSpec API without holding, encoding and decoding multi-MB
payload in RAM.

Signed-off-by: Stephan Meijer <me@stephanmeijer.com>

src/frontend/servers/y-provider/__tests__/convert.test.ts

@@ -26,8 +26,9 @@ describe('Server Tests', () => {
 
     const response = await request(app)
       .post('/api/convert')
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', 'wrong-api-key');
+      .set('authorization', 'wrong-api-key')
+      .set('content-type', 'application/json');
 
     expect(response.status).toBe(401);
     expect(response.body).toStrictEqual({
@@ -40,8 +41,9 @@ describe('Server Tests', () => {
 
     const response = await request(app)
       .post('/api/convert')
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', 'Bearer test-secret-api-key');
+      .set('authorization', 'Bearer test-secret-api-key')
+      .set('content-type', 'application/json');
 
     // Warning: Changing the authorization header to Bearer token format will break backend compatibility with this microservice.
     expect(response.status).toBe(401);
@@ -55,8 +57,9 @@ describe('Server Tests', () => {
 
     const response = await request(app)
       .post('/api/convert')
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', apiKey);
+      .set('authorization', apiKey)
+      .set('content-type', 'application/json');
 
     expect(response.status).toBe(400);
     expect(response.body).toStrictEqual({
@@ -69,11 +72,10 @@ describe('Server Tests', () => {
 
     const response = await request(app)
       .post('/api/convert')
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', apiKey)
+      .set('authorization', apiKey)
-      .send({
+      .set('content-type', 'application/json')
-        content: '',
+      .send('');
-      });
 
     expect(response.status).toBe(400);
     expect(response.body).toStrictEqual({
@@ -93,20 +95,17 @@ describe('Server Tests', () => {
 
     const response = await request(app)
       .post('/api/convert')
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', apiKey)
+      .set('authorization', apiKey)
-      .send({
+      .set('content-type', 'application/json')
-        content: document,
+      .send(document);
-      });
 
     expect(response.status).toBe(200);
-    expect(response.body).toStrictEqual({
+    expect(response.body).toBeInstanceOf(Buffer);
-      content: expect.any(String),
-    });
 
     const editor = ServerBlockNoteEditor.create();
     const doc = new Y.Doc();
-    Y.applyUpdate(doc, Buffer.from(response.body.content, 'base64'));
+    Y.applyUpdate(doc, response.body);
     const blocks = editor.yDocToBlocks(doc, 'document-store');
 
     expect(blocks).toStrictEqual([

src/frontend/servers/y-provider/__tests__/server.test.ts

@@ -40,30 +40,30 @@ describe('Server Tests', () => {
     });
   });
 
-  it('allows JSON payloads up to 500kb for the CONVERT route', async () => {
+  it('allows payloads up to 500kb for the CONVERT route', async () => {
     const app = initApp();
 
     const largePayload = 'a'.repeat(400 * 1024); // 400kb payload
     const response = await request(app)
       .post(routes.CONVERT)
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', apiKey)
+      .set('authorization', apiKey)
-      .set('Content-Type', 'application/json')
+      .set('content-type', 'text/markdown')
-      .send({ data: largePayload });
+      .send(largePayload);
 
     expect(response.status).not.toBe(413);
   });
 
-  it('rejects JSON payloads larger than 500kb for the CONVERT route', async () => {
+  it('rejects payloads larger than 500kb for the CONVERT route', async () => {
     const app = initApp();
 
     const oversizedPayload = 'a'.repeat(501 * 1024); // 501kb payload
     const response = await request(app)
       .post(routes.CONVERT)
-      .set('Origin', origin)
+      .set('origin', origin)
-      .set('Authorization', apiKey)
+      .set('authorization', apiKey)
-      .set('Content-Type', 'application/json')
+      .set('content-type', 'text/markdown')
-      .send({ data: oversizedPayload });
+      .send(oversizedPayload);
 
     expect(response.status).toBe(413);
   });

src/frontend/servers/y-provider/src/handlers/convertHandler.ts

@@ -2,15 +2,7 @@ import { ServerBlockNoteEditor } from '@blocknote/server-util';
 import { Request, Response } from 'express';
 import * as Y from 'yjs';
 
-import { logger, toBase64 } from '@/utils';
+import { logger } from '@/utils';
-
-interface ConversionRequest {
-  content: string;
-}
-
-interface ConversionResponse {
-  content: string;
-}
 
 interface ErrorResponse {
   error: string;
@@ -19,35 +11,30 @@ interface ErrorResponse {
 const editor = ServerBlockNoteEditor.create();
 
 export const convertHandler = async (
-  req: Request<
+  req: Request<object, Uint8Array | ErrorResponse, Buffer, object>,
-    object,
+  res: Response<Uint8Array | ErrorResponse>,
-    ConversionResponse | ErrorResponse,
-    ConversionRequest,
-    object
-  >,
-  res: Response<ConversionResponse | ErrorResponse>,
 ) => {
-  const content = req.body?.content;
+  if (!req.body || req.body.length === 0) {
-
-  if (!content) {
     res.status(400).json({ error: 'Invalid request: missing content' });
     return;
   }
 
   try {
     // Perform the conversion from markdown to Blocknote.js blocks
-    const blocks = await editor.tryParseMarkdownToBlocks(content);
+    const blocks = await editor.tryParseMarkdownToBlocks(req.body.toString());
 
     if (!blocks || blocks.length === 0) {
       res.status(500).json({ error: 'No valid blocks were generated' });
       return;
     }
 
-    // Create a Yjs Document from blocks, and encode it as a base64 string
+    // Create a Yjs Document from blocks
     const yDocument = editor.blocksToYDoc(blocks, 'document-store');
-    const documentContent = toBase64(Y.encodeStateAsUpdate(yDocument));
 
-    res.status(200).json({ content: documentContent });
+    res
+      .status(200)
+      .setHeader('content-type', 'application/octet-stream')
+      .send(Y.encodeStateAsUpdate(yDocument));
   } catch (e) {
     logger('conversion failed:', e);
     res.status(500).json({ error: 'An error occurred' });

src/frontend/servers/y-provider/src/servers/appServer.ts

@@ -47,7 +47,10 @@ export const initApp = () => {
   app.post(
     routes.CONVERT,
     httpSecurity,
-    express.json({ limit: '500kb' }),
+    express.raw({
+      limit: '500kb',
+      type: '*/*',
+    }),
     convertHandler,
   );