WIP

env.d/development/common.dist

@@ -10,6 +10,10 @@ LOGGING_LEVEL_HANDLERS_CONSOLE=INFO
 LOGGING_LEVEL_LOGGERS_ROOT=INFO
 LOGGING_LEVEL_LOGGERS_APP=INFO
 
+# Y-Provider
+Y_PROVIDER_API_KEY="yprovider-api-key" 
+Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/
+
 # Python
 PYTHONPATH=/app
 
@@ -54,6 +58,9 @@ AI_BASE_URL=https://openaiendpoint.com
 AI_API_KEY=password
 AI_MODEL=llama
 
+# Accessibility API
+ACCESSIBILITY_API_BASE_URL=https://localhost:8000
+
 # Collaboration
 COLLABORATION_API_URL=http://nginx:8083/collaboration/api/
 COLLABORATION_SERVER_ORIGIN=http://localhost:3000

src/backend/core/api/serializers.py

@@ -16,6 +16,7 @@
     ConversionError,
     YdocConverter,
 )
+from core.services.ai_services import AIService
 
 
 class UserSerializer(serializers.ModelSerializer):
@@ -306,7 +307,7 @@ def create(self, validated_data):
         if user:
             email = user.email
             language = user.language or language
-
+        
         try:
             document_content = YdocConverter().convert_markdown(
                 validated_data["content"]
@@ -568,6 +569,56 @@ def validate_text(self, value):
         if len(value.strip()) == 0:
             raise serializers.ValidationError("Text field cannot be empty.")
         return value
+
+
+class AIPdfTranscribeSerializer(serializers.Serializer):
+    """Serializer for AI PDF transcribe requests."""
+
+    pdfUrl = serializers.CharField(required=True)
+
+    def __init__(self, *args, **kwargs):
+        """Initialize with user."""
+        self.user = kwargs.pop('user', None)
+        super().__init__(*args, **kwargs)
+
+    def validate_pdfUrl(self, value):
+        """Ensure the pdfUrl field is a valid URL."""
+        if not value.startswith(settings.MEDIA_BASE_URL):
+            raise serializers.ValidationError("Invalid PDF URL format.")
+        return value
+
+    def create(self, validated_data):
+        """Create a new document for the transcribed content."""
+        if not self.user:
+            raise serializers.ValidationError("User is required")
+
+        # Get the transcribed content from AI service
+        pdf_url = validated_data["pdfUrl"]
+        response = AIService().transcribe_pdf(pdf_url)
+
+        try:
+            # Convert the markdown content to YDoc format
+            document_content = YdocConverter().convert_markdown(response)
+        except ConversionError as err:
+            raise serializers.ValidationError(
+                {"content": [f"Could not convert transcribed content: {str(err)}"]}
+            ) from err
+
+        # Create the document as root node with converted content
+        document = models.Document.add_root(
+            title="PDF Transcription", 
+            content=document_content,
+            creator=self.user,
+        )
+
+        # Create owner access for the user
+        models.DocumentAccess.objects.create(
+            document=document,
+            role=models.RoleChoices.OWNER,
+            user=self.user,
+        )
+
+        return document
 
 
 class MoveDocumentSerializer(serializers.Serializer):

src/backend/core/api/viewsets.py

@@ -1142,6 +1142,32 @@ def ai_translate(self, request, *args, **kwargs):
 
         return drf.response.Response(response, status=drf.status.HTTP_200_OK)
 
+    @drf.decorators.action(
+        detail=True,
+        methods=["post"],
+        name="Transcribe PDF with AI",
+        url_path="ai-pdf-transcribe",
+        throttle_classes=[utils.AIDocumentRateThrottle, utils.AIUserRateThrottle],
+    )
+    def ai_pdf_transcribe(self, request, *args, **kwargs):
+        """
+        POST /api/v1.0/documents/<resource_id>/ai-pdf-transcribe
+        with expected data:
+        - pdfUrl: str
+        Return JSON response with the new document ID containing the transcription.
+        """
+        serializer = serializers.AIPdfTranscribeSerializer(
+            data=request.data,
+            user=request.user
+        )
+        serializer.is_valid(raise_exception=True)
+        document = serializer.save()
+
+        return drf.response.Response(
+            {"document_id": str(document.id)}, 
+            status=drf.status.HTTP_201_CREATED
+        )
+
 
 class DocumentAccessViewSet(
     ResourceAccessViewsetMixin,

src/backend/core/services/ai_services.py

@@ -2,13 +2,18 @@
 
 import json
 import re
+import os
+import requests
+import botocore
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from django.core.files.storage import default_storage
 
 from openai import OpenAI
 
 from core import enums
+from core.models import Document
 
 AI_ACTIONS = {
     "prompt": (
@@ -112,3 +117,32 @@ def translate(self, text, language):
         language_display = enums.ALL_LANGUAGES.get(language, language)
         system_content = AI_TRANSLATE.format(language=language_display)
         return self.call_ai_api(system_content, text)
+
+    def transcribe_pdf(self, pdf_url):
+        """Transcribe PDF using the accessibility hackathon API and create a new document."""
+        try:
+            # Extract the key from the media URL
+            media_prefix = os.path.join(settings.MEDIA_BASE_URL, "media")
+            key = pdf_url[len(media_prefix):]
+
+            # Get the PDF directly from MinIO
+            pdf_response = default_storage.connection.meta.client.get_object(
+                Bucket=default_storage.bucket_name,
+                Key=key
+            )
+            # pdf_content = pdf_response['Body'].read()
+
+            # Call the Albert / Mistral API
+            # api_url = f"{settings.ACCESSIBILITY_API_BASE_URL}/transcribe/pdf"
+            # files = {'pdf': ('document.pdf', pdf_content)}
+
+            # response = requests.post(api_url, files=files)
+            # response.raise_for_status()
+
+            # transcribed_text = response.json()['text']
+
+            transcribed_text = "Hello world"
+
+            return transcribed_text
+        except Exception as e:
+            raise RuntimeError(f"Failed to transcribe PDF: {str(e)}")

src/backend/impress/settings.py

@@ -517,6 +517,12 @@ class Base(Configuration):
     AI_BASE_URL = values.Value(None, environ_name="AI_BASE_URL", environ_prefix=None)
     AI_MODEL = values.Value(None, environ_name="AI_MODEL", environ_prefix=None)
 
+    ACCESSIBILITY_API_BASE_URL = values.Value(
+        None,
+        environ_name="ACCESSIBILITY_API_BASE_URL",
+        environ_prefix=None,
+    )
+
     AI_DOCUMENT_RATE_THROTTLE_RATES = {
         "minute": 5,
         "hour": 100,

src/frontend/apps/impress/src/features/docs/doc-editor/api/useDocAIPdfTranscribe.tsx

@@ -0,0 +1,39 @@
+import { useMutation } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+export type DocAIPdfTranscribe = {
+  docId: string;
+  pdfUrl: string;
+};
+
+export type DocAIPdfTranscribeResponse = {
+  document_id: string;
+};
+
+export const docAIPdfTranscribe = async ({
+  docId,
+  ...params
+}: DocAIPdfTranscribe): Promise<DocAIPdfTranscribeResponse> => {
+  const response = await fetchAPI(`documents/${docId}/ai-pdf-transcribe/`, {
+    method: 'POST',
+    body: JSON.stringify({
+      ...params,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new APIError(
+      'Failed to request pdf transcription',
+      await errorCauses(response),
+    );
+  }
+
+  return response.json() as Promise<DocAIPdfTranscribeResponse>;
+};
+
+export function useDocAIPdfTranscribe() {
+  return useMutation<DocAIPdfTranscribeResponse, APIError, DocAIPdfTranscribe>({
+    mutationFn: docAIPdfTranscribe,
+  });
+}

src/frontend/apps/impress/src/features/docs/doc-editor/components/AIPdfButton.tsx

@@ -3,14 +3,26 @@ import {
   useComponentsContext,
   useSelectedBlocks,
 } from '@blocknote/react';
+import { VariantType, useToastProvider } from '@openfun/cunningham-react';
+import { useRouter } from 'next/router';
+import { useTranslation } from 'react-i18next';
 
 import { Text } from '@/components';
+import { useDocStore } from '@/features/docs/doc-management/';
+
+import { useDocAIPdfTranscribe } from '../api/useDocAIPdfTranscribe';
 
 export const AIPdfButton = () => {
   const editor = useBlockNoteEditor();
   const Components = useComponentsContext();
   const selectedBlocks = useSelectedBlocks(editor);
-  if (!Components) {
+  const { t } = useTranslation();
+  const { currentDoc } = useDocStore();
+  const { toast } = useToastProvider();
+  const router = useRouter();
+  const { mutateAsync: requestAIPdf, isPending } = useDocAIPdfTranscribe();
+
+  if (!Components || !currentDoc) {
     return null;
   }
 
@@ -19,20 +31,42 @@ export const AIPdfButton = () => {
     return null;
   }
 
+  const handlePdfTranscription = async () => {
+    console.log('selectedBlocks', selectedBlocks);
+    const pdfBlock = selectedBlocks[0];
+    const props = pdfBlock.props as { url?: string };
+    const pdfUrl = props?.url;
+    console.log('pdfUrl', pdfUrl);
+    if (!props || !pdfUrl) {
+      toast(t('No PDF file found'), VariantType.ERROR);
+      return;
+    }
+
+    try {
+      const response = await requestAIPdf({
+        docId: currentDoc.id,
+        pdfUrl,
+      });
+
+      void router.push(`/docs/${response.document_id}`);
+    } catch (error) {
+      console.error('error', error);
+      toast(t('Failed to transcribe PDF'), VariantType.ERROR);
+    }
+  };
+
   return (
     <Components.FormattingToolbar.Button
       className="bn-button bn-menu-item"
-      data-test="ai-actions"
+      data-test="ai-pdf-transcribe"
       label="AI"
-      mainTooltip="Chat avec le PDF"
+      mainTooltip={t('Transcribe PDF')}
       icon={
         <Text $isMaterialIcon $size="l">
           auto_awesome
         </Text>
       }
-      onClick={() => {
-        console.log('selectedBlocks', selectedBlocks);
-      }}
+      onClick={() => void handlePdfTranscription()}
     />
   );
 };