✨(frontend) add ai blocknote feature

Add AI button to the editor toolbar.
We can use AI to generate content with our editor.
A list of predefined actions are available to use.

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to
 
 ## Added
 
+- ✨AI to doc editor #250
+- ✨(backend) allow uploading more types of attachments #309
 - ✨(frontend) add buttons to copy document to clipboard as HTML/Markdown #300
 
 ## Changed
@@ -22,9 +24,6 @@ and this project adheres to
 ## Fixed
 
 - 🐛(frontend) invalidate queries after removing user #336
-
-## Fixed
-
 - 🐛(backend) Fix dysfunctional permissions on document create #329
 
 ## [1.5.1] - 2024-10-10
@@ -37,7 +36,6 @@ and this project adheres to
 
 ## Added
 
-- ✨(backend) allow uploading more types of attachments #309
 - ✨(backend) add name fields to the user synchronized with OIDC #301
 - ✨(ci) add security scan #291
 - ♻️(frontend) Add versions #277

src/frontend/apps/e2e/__tests__/app-impress/common.ts

@@ -51,6 +51,7 @@ export const createDoc = async (
 
       await page.locator('.c__modal__backdrop').click({
         position: { x: 0, y: 0 },
+        force: true,
       });
 
       await expect(

src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts

@@ -181,4 +181,57 @@ test.describe('Doc Editor', () => {
       /http:\/\/localhost:8083\/media\/.*\/attachments\/.*.png/,
     );
   });
+
+  test('it checks the AI buttons', async ({ page, browserName }) => {
+    await page.route(/.*\/ai-translate\//, async (route) => {
+      const request = route.request();
+      if (request.method().includes('POST')) {
+        await route.fulfill({
+          json: {
+            answer: 'Bonjour le monde',
+          },
+        });
+      } else {
+        await route.continue();
+      }
+    });
+
+    await createDoc(page, 'doc-ai', browserName, 1);
+
+    await page.locator('.bn-block-outer').last().fill('Hello World');
+
+    const editor = page.locator('.ProseMirror');
+    await editor.getByText('Hello').dblclick();
+
+    await page.getByRole('button', { name: 'AI' }).click();
+
+    await expect(
+      page.getByRole('menuitem', { name: 'Use as prompt' }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('menuitem', { name: 'Rephrase' }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('menuitem', { name: 'Summarize' }),
+    ).toBeVisible();
+    await expect(page.getByRole('menuitem', { name: 'Correct' })).toBeVisible();
+    await expect(
+      page.getByRole('menuitem', { name: 'Language' }),
+    ).toBeVisible();
+
+    await page.getByRole('menuitem', { name: 'Language' }).hover();
+    await expect(
+      page.getByRole('menuitem', { name: 'English', exact: true }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('menuitem', { name: 'French', exact: true }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('menuitem', { name: 'German', exact: true }),
+    ).toBeVisible();
+
+    await page.getByRole('menuitem', { name: 'English', exact: true }).click();
+
+    await expect(editor.getByText('Bonjour le monde')).toBeVisible();
+  });
 });

src/frontend/apps/impress/src/api/APIError.ts

@@ -18,3 +18,7 @@ export class APIError<T = unknown> extends Error implements IAPIError<T> {
     this.data = data;
   }
 }
+
+export const isAPIError = (error: unknown): error is APIError => {
+  return error instanceof APIError;
+};

src/frontend/apps/impress/src/features/docs/doc-editor/api/index.ts

@@ -0,0 +1,3 @@
+export * from './useCreateDocUpload';
+export * from './useDocAITransform';
+export * from './useDocAITranslate';

src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITransform.tsx

@@ -0,0 +1,46 @@
+import { useMutation } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+export type AITransformActions =
+  | 'correct'
+  | 'prompt'
+  | 'rephrase'
+  | 'summarize';
+
+export type DocAITransform = {
+  docId: string;
+  text: string;
+  action: AITransformActions;
+};
+
+export type DocAITransformResponse = {
+  answer: string;
+};
+
+export const docAITransform = async ({
+  docId,
+  ...params
+}: DocAITransform): Promise<DocAITransformResponse> => {
+  const response = await fetchAPI(`documents/${docId}/ai-transform/`, {
+    method: 'POST',
+    body: JSON.stringify({
+      ...params,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new APIError(
+      'Failed to request ai transform',
+      await errorCauses(response),
+    );
+  }
+
+  return response.json() as Promise<DocAITransformResponse>;
+};
+
+export function useDocAITransform() {
+  return useMutation<DocAITransformResponse, APIError, DocAITransform>({
+    mutationFn: docAITransform,
+  });
+}

src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITranslate.tsx

@@ -0,0 +1,40 @@
+import { useMutation } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+export type DocAITranslate = {
+  docId: string;
+  text: string;
+  language: string;
+};
+
+export type DocAITranslateResponse = {
+  answer: string;
+};
+
+export const docAITranslate = async ({
+  docId,
+  ...params
+}: DocAITranslate): Promise<DocAITranslateResponse> => {
+  const response = await fetchAPI(`documents/${docId}/ai-translate/`, {
+    method: 'POST',
+    body: JSON.stringify({
+      ...params,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new APIError(
+      'Failed to request ai translate',
+      await errorCauses(response),
+    );
+  }
+
+  return response.json() as Promise<DocAITranslateResponse>;
+};
+
+export function useDocAITranslate() {
+  return useMutation<DocAITranslateResponse, APIError, DocAITranslate>({
+    mutationFn: docAITranslate,
+  });
+}

src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx

@@ -0,0 +1,383 @@
+import {
+  ComponentProps,
+  useBlockNoteEditor,
+  useComponentsContext,
+  useSelectedBlocks,
+} from '@blocknote/react';
+import {
+  Loader,
+  VariantType,
+  useToastProvider,
+} from '@openfun/cunningham-react';
+import {
+  PropsWithChildren,
+  ReactNode,
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { isAPIError } from '@/api';
+import { Box, Text } from '@/components';
+import { useDocOptions } from '@/features/docs/doc-management/';
+
+import {
+  AITransformActions,
+  useDocAITransform,
+  useDocAITranslate,
+} from '../api/';
+import { useDocStore } from '../stores';
+
+type LanguageTranslate = {
+  value: string;
+  display_name: string;
+};
+
+const sortByPopularLanguages = (
+  languages: LanguageTranslate[],
+  popularLanguages: string[],
+) => {
+  languages.sort((a, b) => {
+    const indexA = popularLanguages.indexOf(a.value);
+    const indexB = popularLanguages.indexOf(b.value);
+
+    // If both languages are in the popular list, sort based on their order in popularLanguages
+    if (indexA !== -1 && indexB !== -1) {
+      return indexA - indexB;
+    }
+
+    // If only a is in the popular list, it should come first
+    if (indexA !== -1) {
+      return -1;
+    }
+
+    // If only b is in the popular list, it should come first
+    if (indexB !== -1) {
+      return 1;
+    }
+
+    // If neither a nor b is in the popular list, maintain their relative order
+    return 0;
+  });
+};
+
+export function AIGroupButton() {
+  const editor = useBlockNoteEditor();
+  const Components = useComponentsContext();
+  const selectedBlocks = useSelectedBlocks(editor);
+  const { t } = useTranslation();
+  const { currentDoc } = useDocStore();
+  const { data: docOptions } = useDocOptions();
+  const [languages, setLanguages] = useState<LanguageTranslate[]>([]);
+
+  useEffect(() => {
+    const languages = docOptions?.actions.POST.language.choices;
+
+    if (!languages) {
+      return;
+    }
+
+    sortByPopularLanguages(languages, [
+      'fr',
+      'en',
+      'de',
+      'es',
+      'it',
+      'pt',
+      'nl',
+      'pl',
+    ]);
+
+    setLanguages(languages);
+  }, [docOptions?.actions.POST.language.choices]);
+
+  const show = useMemo(() => {
+    return !!selectedBlocks.find((block) => block.content !== undefined);
+  }, [selectedBlocks]);
+
+  if (!show || !editor.isEditable || !Components || !currentDoc || !languages) {
+    return null;
+  }
+
+  return (
+    <Components.Generic.Menu.Root>
+      <Components.Generic.Menu.Trigger>
+        <Components.FormattingToolbar.Button
+          className="bn-button bn-menu-item"
+          data-test="ai-actions"
+          label="AI"
+          mainTooltip={t('AI Actions')}
+          icon={
+            <Text $isMaterialIcon $size="l">
+              auto_awesome
+            </Text>
+          }
+        />
+      </Components.Generic.Menu.Trigger>
+      <Components.Generic.Menu.Dropdown
+        className="bn-menu-dropdown bn-drag-handle-menu"
+        sub={true}
+      >
+        <AIMenuItemTransform
+          action="prompt"
+          docId={currentDoc.id}
+          icon={
+            <Text $isMaterialIcon $size="s">
+              text_fields
+            </Text>
+          }
+        >
+          {t('Use as prompt')}
+        </AIMenuItemTransform>
+        <AIMenuItemTransform
+          action="rephrase"
+          docId={currentDoc.id}
+          icon={
+            <Text $isMaterialIcon $size="s">
+              refresh
+            </Text>
+          }
+        >
+          {t('Rephrase')}
+        </AIMenuItemTransform>
+        <AIMenuItemTransform
+          action="summarize"
+          docId={currentDoc.id}
+          icon={
+            <Text $isMaterialIcon $size="s">
+              summarize
+            </Text>
+          }
+        >
+          {t('Summarize')}
+        </AIMenuItemTransform>
+        <AIMenuItemTransform
+          action="correct"
+          docId={currentDoc.id}
+          icon={
+            <Text $isMaterialIcon $size="s">
+              check
+            </Text>
+          }
+        >
+          {t('Correct')}
+        </AIMenuItemTransform>
+        <Components.Generic.Menu.Root position="right" sub={true}>
+          <Components.Generic.Menu.Trigger sub={false}>
+            <Components.Generic.Menu.Item
+              className="bn-menu-item"
+              subTrigger={true}
+            >
+              <Box $direction="row" $gap="0.6rem">
+                <Text $isMaterialIcon $size="s">
+                  translate
+                </Text>
+                {t('Language')}
+              </Box>
+            </Components.Generic.Menu.Item>
+          </Components.Generic.Menu.Trigger>
+          <Components.Generic.Menu.Dropdown
+            sub={true}
+            className="bn-menu-dropdown"
+          >
+            {languages.map((language) => (
+              <AIMenuItemTranslate
+                key={language.value}
+                language={language.value}
+                docId={currentDoc.id}
+              >
+                {language.display_name}
+              </AIMenuItemTranslate>
+            ))}
+          </Components.Generic.Menu.Dropdown>
+        </Components.Generic.Menu.Root>
+      </Components.Generic.Menu.Dropdown>
+    </Components.Generic.Menu.Root>
+  );
+}
+
+/**
+ * Item is derived from Mantime, some props seem lacking or incorrect.
+ */
+type ItemDefault = ComponentProps['Generic']['Menu']['Item'];
+type ItemProps = Omit<ItemDefault, 'onClick'> & {
+  rightSection?: ReactNode;
+  closeMenuOnClick?: boolean;
+  onClick: (e: React.MouseEvent) => void;
+};
+
+interface AIMenuItemTransform {
+  action: AITransformActions;
+  docId: string;
+  icon?: ReactNode;
+}
+
+const AIMenuItemTransform = ({
+  docId,
+  action,
+  children,
+  icon,
+}: PropsWithChildren<AIMenuItemTransform>) => {
+  const editor = useBlockNoteEditor();
+  const { mutateAsync: requestAI, isPending } = useDocAITransform();
+  const handleAIError = useHandleAIError();
+
+  const handleAIAction = useCallback(async () => {
+    const selectedBlocks = editor.getSelection()?.blocks;
+
+    if (!selectedBlocks || selectedBlocks.length === 0) {
+      return;
+    }
+
+    const markdown = await editor.blocksToMarkdownLossy(selectedBlocks);
+
+    try {
+      const responseAI = await requestAI({
+        text: markdown,
+        action,
+        docId,
+      });
+
+      if (!responseAI.answer) {
+        return;
+      }
+
+      const blockMarkdown = await editor.tryParseMarkdownToBlocks(
+        responseAI.answer,
+      );
+      editor.replaceBlocks(selectedBlocks, blockMarkdown);
+    } catch (error) {
+      handleAIError(error);
+    }
+  }, [editor, requestAI, action, docId, handleAIError]);
+
+  return (
+    <AIMenuItem
+      icon={icon}
+      handleAIAction={handleAIAction}
+      isPending={isPending}
+    >
+      {children}
+    </AIMenuItem>
+  );
+};
+
+interface AIMenuItemTranslate {
+  language: string;
+  docId: string;
+  icon?: ReactNode;
+}
+
+const AIMenuItemTranslate = ({
+  children,
+  docId,
+  icon,
+  language,
+}: PropsWithChildren<AIMenuItemTranslate>) => {
+  const editor = useBlockNoteEditor();
+  const { mutateAsync: requestAI, isPending } = useDocAITranslate();
+  const handleAIError = useHandleAIError();
+
+  const handleAIAction = useCallback(async () => {
+    const selectedBlocks = editor.getSelection()?.blocks;
+
+    if (!selectedBlocks || selectedBlocks.length === 0) {
+      return;
+    }
+
+    const markdown = await editor.blocksToMarkdownLossy(selectedBlocks);
+
+    try {
+      const responseAI = await requestAI({
+        text: markdown,
+        language,
+        docId,
+      });
+
+      if (!responseAI.answer) {
+        return;
+      }
+
+      const blockMarkdown = await editor.tryParseMarkdownToBlocks(
+        responseAI.answer,
+      );
+      editor.replaceBlocks(selectedBlocks, blockMarkdown);
+    } catch (error) {
+      handleAIError(error);
+    }
+  }, [editor, requestAI, language, docId, handleAIError]);
+
+  return (
+    <AIMenuItem
+      icon={icon}
+      handleAIAction={handleAIAction}
+      isPending={isPending}
+    >
+      {children}
+    </AIMenuItem>
+  );
+};
+
+interface AIMenuItemProps {
+  handleAIAction: () => Promise<void>;
+  isPending: boolean;
+  icon?: ReactNode;
+}
+
+const AIMenuItem = ({
+  handleAIAction,
+  isPending,
+  children,
+  icon,
+}: PropsWithChildren<AIMenuItemProps>) => {
+  const Components = useComponentsContext();
+
+  if (!Components) {
+    return null;
+  }
+
+  const Item = Components.Generic.Menu.Item as React.FC<ItemProps>;
+
+  return (
+    <Item
+      closeMenuOnClick={false}
+      icon={icon}
+      onClick={(e: React.MouseEvent) => {
+        e.stopPropagation();
+        void handleAIAction();
+      }}
+      rightSection={isPending ? <Loader size="small" /> : undefined}
+    >
+      {children}
+    </Item>
+  );
+};
+
+const useHandleAIError = () => {
+  const { toast } = useToastProvider();
+  const { t } = useTranslation();
+
+  const handleAIError = useCallback(
+    (error: unknown) => {
+      if (isAPIError(error)) {
+        error.cause?.forEach((cause) => {
+          if (
+            cause === 'Request was throttled. Expected available in 60 seconds.'
+          ) {
+            toast(
+              t('Too many requests. Please wait 60 seconds.'),
+              VariantType.ERROR,
+            );
+          }
+        });
+      }
+
+      console.error(error);
+    },
+    [toast, t],
+  );
+
+  return handleAIError;
+};

src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx

@@ -12,6 +12,7 @@ import {
 } from '@blocknote/react';
 import React from 'react';
 
+import { AIGroupButton } from './AIButton';
 import { MarkdownButton } from './MarkdownButton';
 
 export const BlockNoteToolbar = () => {
@@ -21,6 +22,9 @@ export const BlockNoteToolbar = () => {
         <FormattingToolbar>
           <BlockTypeSelect key="blockTypeSelect" />
 
+          {/* Extra button to do some AI powered actions */}
+          <AIGroupButton key="AIButton" />
+
           {/* Extra button to convert from markdown to json */}
           <MarkdownButton key="customButton" />
 

src/frontend/apps/impress/src/features/docs/doc-management/api/index.ts

@@ -1,5 +1,6 @@
 export * from './useCreateDoc';
 export * from './useDoc';
+export * from './useDocOptions';
 export * from './useDocs';
 export * from './useUpdateDoc';
 export * from './useUpdateDocLink';

src/frontend/apps/impress/src/features/docs/doc-management/api/useDocOptions.tsx

@@ -0,0 +1,47 @@
+import { UseQueryOptions, useQuery } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+type DocOptionsResponse = {
+  actions: {
+    POST: {
+      language: {
+        choices: {
+          value: string;
+          display_name: string;
+        }[];
+      };
+    };
+  };
+};
+
+export const docOptions = async (): Promise<DocOptionsResponse> => {
+  const response = await fetchAPI(`documents/`, {
+    method: 'OPTIONS',
+  });
+
+  if (!response.ok) {
+    throw new APIError(
+      'Failed to get the doc options',
+      await errorCauses(response),
+    );
+  }
+
+  return response.json() as Promise<DocOptionsResponse>;
+};
+
+export const KEY_DOC_OPTIONS = 'doc-options';
+
+export function useDocOptions(
+  queryConfig?: UseQueryOptions<
+    DocOptionsResponse,
+    APIError,
+    DocOptionsResponse
+  >,
+) {
+  return useQuery<DocOptionsResponse, APIError, DocOptionsResponse>({
+    queryKey: [KEY_DOC_OPTIONS],
+    queryFn: docOptions,
+    ...queryConfig,
+  });
+}