From 5dc43cbc8bdbbf5f7b583406fc55f82129365c68 Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Fri, 20 Sep 2024 22:45:06 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(frontend)=20add=20ai=20blocknote=20fe?= =?UTF-8?q?ature?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add AI button to the editor toolbar. We can use AI to generate content with our editor. A list of predefined actions are available to use. --- CHANGELOG.md | 6 +- .../apps/e2e/__tests__/app-impress/common.ts | 1 + .../__tests__/app-impress/doc-editor.spec.ts | 53 +++ src/frontend/apps/impress/src/api/APIError.ts | 4 + .../src/features/docs/doc-editor/api/index.ts | 3 + .../docs/doc-editor/api/useDocAITransform.tsx | 46 +++ .../docs/doc-editor/api/useDocAITranslate.tsx | 40 ++ .../docs/doc-editor/components/AIButton.tsx | 383 ++++++++++++++++++ .../components/BlockNoteToolbar.tsx | 4 + .../features/docs/doc-management/api/index.ts | 1 + .../docs/doc-management/api/useDocOptions.tsx | 47 +++ 11 files changed, 584 insertions(+), 4 deletions(-) create mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/api/index.ts create mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITransform.tsx create mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITranslate.tsx create mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx create mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/api/useDocOptions.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 5be3ef36..4f23594a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to ## Added +- ✨AI to doc editor #250 +- ✨(backend) allow uploading more types of attachments #309 - ✨(frontend) add buttons to copy document to clipboard as HTML/Markdown #300 ## Changed @@ -22,9 +24,6 @@ and this project adheres to ## Fixed - 🐛(frontend) invalidate queries after removing user #336 - -## Fixed - - 🐛(backend) Fix dysfunctional permissions on document create #329 ## [1.5.1] - 2024-10-10 @@ -37,7 +36,6 @@ and this project adheres to ## Added -- ✨(backend) allow uploading more types of attachments #309 - ✨(backend) add name fields to the user synchronized with OIDC #301 - ✨(ci) add security scan #291 - ♻️(frontend) Add versions #277 diff --git a/src/frontend/apps/e2e/__tests__/app-impress/common.ts b/src/frontend/apps/e2e/__tests__/app-impress/common.ts index 5b8dfe68..4d5c43c4 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/common.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/common.ts @@ -51,6 +51,7 @@ export const createDoc = async ( await page.locator('.c__modal__backdrop').click({ position: { x: 0, y: 0 }, + force: true, }); await expect( diff --git a/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts index a39362e2..4d8f77fa 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts @@ -181,4 +181,57 @@ test.describe('Doc Editor', () => { /http:\/\/localhost:8083\/media\/.*\/attachments\/.*.png/, ); }); + + test('it checks the AI buttons', async ({ page, browserName }) => { + await page.route(/.*\/ai-translate\//, async (route) => { + const request = route.request(); + if (request.method().includes('POST')) { + await route.fulfill({ + json: { + answer: 'Bonjour le monde', + }, + }); + } else { + await route.continue(); + } + }); + + await createDoc(page, 'doc-ai', browserName, 1); + + await page.locator('.bn-block-outer').last().fill('Hello World'); + + const editor = page.locator('.ProseMirror'); + await editor.getByText('Hello').dblclick(); + + await page.getByRole('button', { name: 'AI' }).click(); + + await expect( + page.getByRole('menuitem', { name: 'Use as prompt' }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'Rephrase' }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'Summarize' }), + ).toBeVisible(); + await expect(page.getByRole('menuitem', { name: 'Correct' })).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'Language' }), + ).toBeVisible(); + + await page.getByRole('menuitem', { name: 'Language' }).hover(); + await expect( + page.getByRole('menuitem', { name: 'English', exact: true }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'French', exact: true }), + ).toBeVisible(); + await expect( + page.getByRole('menuitem', { name: 'German', exact: true }), + ).toBeVisible(); + + await page.getByRole('menuitem', { name: 'English', exact: true }).click(); + + await expect(editor.getByText('Bonjour le monde')).toBeVisible(); + }); }); diff --git a/src/frontend/apps/impress/src/api/APIError.ts b/src/frontend/apps/impress/src/api/APIError.ts index 2aaf715c..bed42a3e 100644 --- a/src/frontend/apps/impress/src/api/APIError.ts +++ b/src/frontend/apps/impress/src/api/APIError.ts @@ -18,3 +18,7 @@ export class APIError extends Error implements IAPIError { this.data = data; } } + +export const isAPIError = (error: unknown): error is APIError => { + return error instanceof APIError; +}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/api/index.ts b/src/frontend/apps/impress/src/features/docs/doc-editor/api/index.ts new file mode 100644 index 00000000..8173bc32 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/api/index.ts @@ -0,0 +1,3 @@ +export * from './useCreateDocUpload'; +export * from './useDocAITransform'; +export * from './useDocAITranslate'; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITransform.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITransform.tsx new file mode 100644 index 00000000..f38e1176 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITransform.tsx @@ -0,0 +1,46 @@ +import { useMutation } from '@tanstack/react-query'; + +import { APIError, errorCauses, fetchAPI } from '@/api'; + +export type AITransformActions = + | 'correct' + | 'prompt' + | 'rephrase' + | 'summarize'; + +export type DocAITransform = { + docId: string; + text: string; + action: AITransformActions; +}; + +export type DocAITransformResponse = { + answer: string; +}; + +export const docAITransform = async ({ + docId, + ...params +}: DocAITransform): Promise => { + const response = await fetchAPI(`documents/${docId}/ai-transform/`, { + method: 'POST', + body: JSON.stringify({ + ...params, + }), + }); + + if (!response.ok) { + throw new APIError( + 'Failed to request ai transform', + await errorCauses(response), + ); + } + + return response.json() as Promise; +}; + +export function useDocAITransform() { + return useMutation({ + mutationFn: docAITransform, + }); +} diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITranslate.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITranslate.tsx new file mode 100644 index 00000000..504d79b3 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAITranslate.tsx @@ -0,0 +1,40 @@ +import { useMutation } from '@tanstack/react-query'; + +import { APIError, errorCauses, fetchAPI } from '@/api'; + +export type DocAITranslate = { + docId: string; + text: string; + language: string; +}; + +export type DocAITranslateResponse = { + answer: string; +}; + +export const docAITranslate = async ({ + docId, + ...params +}: DocAITranslate): Promise => { + const response = await fetchAPI(`documents/${docId}/ai-translate/`, { + method: 'POST', + body: JSON.stringify({ + ...params, + }), + }); + + if (!response.ok) { + throw new APIError( + 'Failed to request ai translate', + await errorCauses(response), + ); + } + + return response.json() as Promise; +}; + +export function useDocAITranslate() { + return useMutation({ + mutationFn: docAITranslate, + }); +} diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx new file mode 100644 index 00000000..b0987792 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AIButton.tsx @@ -0,0 +1,383 @@ +import { + ComponentProps, + useBlockNoteEditor, + useComponentsContext, + useSelectedBlocks, +} from '@blocknote/react'; +import { + Loader, + VariantType, + useToastProvider, +} from '@openfun/cunningham-react'; +import { + PropsWithChildren, + ReactNode, + useCallback, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; + +import { isAPIError } from '@/api'; +import { Box, Text } from '@/components'; +import { useDocOptions } from '@/features/docs/doc-management/'; + +import { + AITransformActions, + useDocAITransform, + useDocAITranslate, +} from '../api/'; +import { useDocStore } from '../stores'; + +type LanguageTranslate = { + value: string; + display_name: string; +}; + +const sortByPopularLanguages = ( + languages: LanguageTranslate[], + popularLanguages: string[], +) => { + languages.sort((a, b) => { + const indexA = popularLanguages.indexOf(a.value); + const indexB = popularLanguages.indexOf(b.value); + + // If both languages are in the popular list, sort based on their order in popularLanguages + if (indexA !== -1 && indexB !== -1) { + return indexA - indexB; + } + + // If only a is in the popular list, it should come first + if (indexA !== -1) { + return -1; + } + + // If only b is in the popular list, it should come first + if (indexB !== -1) { + return 1; + } + + // If neither a nor b is in the popular list, maintain their relative order + return 0; + }); +}; + +export function AIGroupButton() { + const editor = useBlockNoteEditor(); + const Components = useComponentsContext(); + const selectedBlocks = useSelectedBlocks(editor); + const { t } = useTranslation(); + const { currentDoc } = useDocStore(); + const { data: docOptions } = useDocOptions(); + const [languages, setLanguages] = useState([]); + + useEffect(() => { + const languages = docOptions?.actions.POST.language.choices; + + if (!languages) { + return; + } + + sortByPopularLanguages(languages, [ + 'fr', + 'en', + 'de', + 'es', + 'it', + 'pt', + 'nl', + 'pl', + ]); + + setLanguages(languages); + }, [docOptions?.actions.POST.language.choices]); + + const show = useMemo(() => { + return !!selectedBlocks.find((block) => block.content !== undefined); + }, [selectedBlocks]); + + if (!show || !editor.isEditable || !Components || !currentDoc || !languages) { + return null; + } + + return ( + + + + auto_awesome + + } + /> + + + + text_fields + + } + > + {t('Use as prompt')} + + + refresh + + } + > + {t('Rephrase')} + + + summarize + + } + > + {t('Summarize')} + + + check + + } + > + {t('Correct')} + + + + + + + translate + + {t('Language')} + + + + + {languages.map((language) => ( + + {language.display_name} + + ))} + + + + + ); +} + +/** + * Item is derived from Mantime, some props seem lacking or incorrect. + */ +type ItemDefault = ComponentProps['Generic']['Menu']['Item']; +type ItemProps = Omit & { + rightSection?: ReactNode; + closeMenuOnClick?: boolean; + onClick: (e: React.MouseEvent) => void; +}; + +interface AIMenuItemTransform { + action: AITransformActions; + docId: string; + icon?: ReactNode; +} + +const AIMenuItemTransform = ({ + docId, + action, + children, + icon, +}: PropsWithChildren) => { + const editor = useBlockNoteEditor(); + const { mutateAsync: requestAI, isPending } = useDocAITransform(); + const handleAIError = useHandleAIError(); + + const handleAIAction = useCallback(async () => { + const selectedBlocks = editor.getSelection()?.blocks; + + if (!selectedBlocks || selectedBlocks.length === 0) { + return; + } + + const markdown = await editor.blocksToMarkdownLossy(selectedBlocks); + + try { + const responseAI = await requestAI({ + text: markdown, + action, + docId, + }); + + if (!responseAI.answer) { + return; + } + + const blockMarkdown = await editor.tryParseMarkdownToBlocks( + responseAI.answer, + ); + editor.replaceBlocks(selectedBlocks, blockMarkdown); + } catch (error) { + handleAIError(error); + } + }, [editor, requestAI, action, docId, handleAIError]); + + return ( + + {children} + + ); +}; + +interface AIMenuItemTranslate { + language: string; + docId: string; + icon?: ReactNode; +} + +const AIMenuItemTranslate = ({ + children, + docId, + icon, + language, +}: PropsWithChildren) => { + const editor = useBlockNoteEditor(); + const { mutateAsync: requestAI, isPending } = useDocAITranslate(); + const handleAIError = useHandleAIError(); + + const handleAIAction = useCallback(async () => { + const selectedBlocks = editor.getSelection()?.blocks; + + if (!selectedBlocks || selectedBlocks.length === 0) { + return; + } + + const markdown = await editor.blocksToMarkdownLossy(selectedBlocks); + + try { + const responseAI = await requestAI({ + text: markdown, + language, + docId, + }); + + if (!responseAI.answer) { + return; + } + + const blockMarkdown = await editor.tryParseMarkdownToBlocks( + responseAI.answer, + ); + editor.replaceBlocks(selectedBlocks, blockMarkdown); + } catch (error) { + handleAIError(error); + } + }, [editor, requestAI, language, docId, handleAIError]); + + return ( + + {children} + + ); +}; + +interface AIMenuItemProps { + handleAIAction: () => Promise; + isPending: boolean; + icon?: ReactNode; +} + +const AIMenuItem = ({ + handleAIAction, + isPending, + children, + icon, +}: PropsWithChildren) => { + const Components = useComponentsContext(); + + if (!Components) { + return null; + } + + const Item = Components.Generic.Menu.Item as React.FC; + + return ( + { + e.stopPropagation(); + void handleAIAction(); + }} + rightSection={isPending ? : undefined} + > + {children} + + ); +}; + +const useHandleAIError = () => { + const { toast } = useToastProvider(); + const { t } = useTranslation(); + + const handleAIError = useCallback( + (error: unknown) => { + if (isAPIError(error)) { + error.cause?.forEach((cause) => { + if ( + cause === 'Request was throttled. Expected available in 60 seconds.' + ) { + toast( + t('Too many requests. Please wait 60 seconds.'), + VariantType.ERROR, + ); + } + }); + } + + console.error(error); + }, + [toast, t], + ); + + return handleAIError; +}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx index d7deeee2..3d6b77e1 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteToolbar.tsx @@ -12,6 +12,7 @@ import { } from '@blocknote/react'; import React from 'react'; +import { AIGroupButton } from './AIButton'; import { MarkdownButton } from './MarkdownButton'; export const BlockNoteToolbar = () => { @@ -21,6 +22,9 @@ export const BlockNoteToolbar = () => { + {/* Extra button to do some AI powered actions */} + + {/* Extra button to convert from markdown to json */} diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/index.ts b/src/frontend/apps/impress/src/features/docs/doc-management/api/index.ts index 5cf42cd7..65c28fbd 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/index.ts +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/index.ts @@ -1,5 +1,6 @@ export * from './useCreateDoc'; export * from './useDoc'; +export * from './useDocOptions'; export * from './useDocs'; export * from './useUpdateDoc'; export * from './useUpdateDocLink'; diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocOptions.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocOptions.tsx new file mode 100644 index 00000000..6cfde96f --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocOptions.tsx @@ -0,0 +1,47 @@ +import { UseQueryOptions, useQuery } from '@tanstack/react-query'; + +import { APIError, errorCauses, fetchAPI } from '@/api'; + +type DocOptionsResponse = { + actions: { + POST: { + language: { + choices: { + value: string; + display_name: string; + }[]; + }; + }; + }; +}; + +export const docOptions = async (): Promise => { + const response = await fetchAPI(`documents/`, { + method: 'OPTIONS', + }); + + if (!response.ok) { + throw new APIError( + 'Failed to get the doc options', + await errorCauses(response), + ); + } + + return response.json() as Promise; +}; + +export const KEY_DOC_OPTIONS = 'doc-options'; + +export function useDocOptions( + queryConfig?: UseQueryOptions< + DocOptionsResponse, + APIError, + DocOptionsResponse + >, +) { + return useQuery({ + queryKey: [KEY_DOC_OPTIONS], + queryFn: docOptions, + ...queryConfig, + }); +}