fix: audio, recording format error

gpustack · Dec 11, 2024 · fef5d0e · fef5d0e
1 parent f64e696
commit fef5d0e
Show file tree

Hide file tree

Showing 10 changed files with 131 additions and 69 deletions.
diff --git a/package.json b/package.json
@@ -17,6 +17,7 @@
   "dependencies": {
     "@ant-design/icons": "^5.5.1",
     "@ant-design/pro-components": "^2.7.18",
+    "@ffmpeg/ffmpeg": "^0.12.10",
     "@huggingface/gguf": "^0.1.7",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/tasks": "^0.11.6",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/components/audio-player/index.tsx b/src/components/audio-player/index.tsx
@@ -175,6 +175,18 @@ const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
     console.log('onload', e);
   };
 
+  const onDownload = useCallback(() => {
+    const url = props.url || '';
+    const filename = props.name;
+
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = filename;
+    document.body.appendChild(link);
+    link.click();
+    link.remove();
+  }, [props.url, props.name]);
+
   useEffect(() => {
     if (audioRef.current) {
       initPlayerConfig();

diff --git a/src/pages/llmodels/components/deploy-modal.tsx b/src/pages/llmodels/components/deploy-modal.tsx
@@ -94,17 +94,15 @@ const AddModal: React.FC<AddModalProps> = (props) => {
     if (!open) {
       setIsGGUF(false);
       form.current?.setFieldValue?.('backend', backendOptionsMap.vllm);
+    } else if (source === modelSourceMap.ollama_library_value) {
+      form.current?.setFieldValue?.('backend', backendOptionsMap.llamaBox);
+      setIsGGUF(true);
     }
+
     return () => {
       setSelectedModel({});
     };
-  }, [open]);
-
-  useEffect(() => {
-    if (source === modelSourceMap.ollama_library_value) {
-      setIsGGUF(true);
-    }
-  }, [source]);
+  }, [open, source]);
 
   return (
     <Drawer

diff --git a/src/pages/playground/components/audio-input.tsx b/src/pages/playground/components/audio-input.tsx
@@ -1,5 +1,6 @@
 import { externalRefer } from '@/constants/external-links';
 import { AudioOutlined } from '@ant-design/icons';
+import { FFmpeg } from '@ffmpeg/ffmpeg';
 import { useIntl } from '@umijs/max';
 import { Button, Space, Tooltip } from 'antd';
 import dayjs from 'dayjs';
@@ -43,8 +44,8 @@ const audioFormat: Record<string, string> = {
 };
 
 const recordingFormat = {
-  type: 'audio/wav',
-  suffix: '.wav'
+  type: 'audio/mpeg',
+  suffix: '.mp3'
 };
 
 const AudioInput: React.FC<AudioInputProps> = (props) => {
@@ -59,6 +60,7 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
   const analyser = useRef<any>(null);
   const dataArray = useRef<any>(null);
   const audioUrl = useRef<string>('');
+  const ffmpeg = useRef<any>(null);
 
   const initAudioContext = useCallback(() => {
     audioContext.current = new (window.AudioContext ||
@@ -90,6 +92,53 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
     // props.onRecord?.(false);
   };
 
+  const getAudioFormat = (type?: string) => {
+    const mimeType = type || audioRecorder.current?.mimeType;
+
+    let resultFormat = recordingFormat;
+
+    Object.keys(audioFormat).forEach((key: string) => {
+      if (mimeType.includes(key)) {
+        resultFormat = {
+          type: mimeType,
+          suffix: `.${audioFormat[key]}`
+        };
+      }
+    });
+    return resultFormat;
+  };
+
+  const convertAudioBlob2Mp3 = useCallback(
+    async (audioBlob: any, mimeType: string): Promise<Blob> => {
+      try {
+        if (!ffmpeg.current.loaded) {
+          await ffmpeg.current.load();
+        }
+        const format = getAudioFormat(mimeType);
+        const recordFile = new Uint8Array(await audioBlob.arrayBuffer());
+
+        await ffmpeg.current.writeFile(`input${format.suffix}`, recordFile);
+
+        await ffmpeg.current.exec([
+          '-i',
+          `input${format.suffix}`,
+          '-f',
+          'mp3',
+          'output.mp3'
+        ]);
+
+        const mp3Data = await ffmpeg.current.readFile('output.mp3');
+
+        const mp3Blob = new Blob([mp3Data.buffer], { type: 'audio/mpeg' });
+
+        return mp3Blob;
+      } catch (error) {
+        return new Blob();
+      }
+    },
+    []
+  );
+
   // get all audio tracks
   const getAudioTracks = () => {
     const audioTracks = audioStream.current.getAudioTracks();
@@ -174,26 +223,11 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
   const handleAudioData = (audioData: any) => {
     props.onAudioData?.(audioData);
   };
-  const getAudioFormat = (type?: string) => {
-    const mimeType = type || audioRecorder.current?.mimeType;
-
-    let resultFormat = recordingFormat;
-
-    Object.keys(audioFormat).forEach((key: string) => {
-      if (mimeType.includes(key)) {
-        resultFormat = {
-          type: mimeType,
-          suffix: `.${audioFormat[key]}`
-        };
-      }
-    });
-    return resultFormat;
-  };
 
   const generateFileNameByTime = (type?: string) => {
-    const res = getAudioFormat(type);
+    const format = getAudioFormat(type);
     // format: recording-YYYY-MM-DD-HH_mm_ss.wav
-    return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${res.suffix}`;
+    return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${format.suffix}`;
   };
   // start recording
   const StartRecording = async () => {
@@ -209,6 +243,8 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
 
       audioRecorder.current = new MediaRecorder(audioStream.current);
 
+      console.log('audioRecorder:', audioRecorder.current);
+
       const audioChunks: any[] = [];
 
       audioRecorder.current.ondataavailable = (event: any) => {
@@ -221,20 +257,27 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
       };
 
       // stop recording
-      audioRecorder.current.onstop = () => {
-        const res = getAudioFormat();
+      audioRecorder.current.onstop = async () => {
         const audioBlob = new Blob(audioChunks, {
-          type: 'audio/mpeg'
+          type: audioRecorder.current.mimeType
         });
-        console.log('audioBlob:', res, audioBlob);
-        audioUrl.current = URL.createObjectURL(audioBlob);
+
+        // convert audio blob to mp3
+        const mp3Blob = await convertAudioBlob2Mp3(
+          audioBlob,
+          audioRecorder.current.mimeType
+        );
+
+        console.log('mp3Blob:', mp3Blob, audioBlob);
+
+        audioUrl.current = mp3Blob.size ? URL.createObjectURL(mp3Blob) : '';
 
         handleAudioData({
-          chunks: audioBlob,
-          size: audioBlob.size,
-          type: audioBlob.type,
+          chunks: mp3Blob,
+          size: mp3Blob.size,
+          type: mp3Blob.type,
           url: audioUrl.current,
-          name: generateFileNameByTime(audioBlob.type),
+          name: generateFileNameByTime(mp3Blob.type),
           duration: Math.floor((Date.now() - startTime.current) / 1000)
         });
 
@@ -302,6 +345,13 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
     checkMicrophonePermission();
   }, []);
 
+  useEffect(() => {
+    ffmpeg.current = new FFmpeg();
+    return () => {
+      ffmpeg.current = null;
+    };
+  }, []);
+
   return (
     <div className="audio-input">
       <Space size={40} className="btns">

diff --git a/src/pages/playground/components/ground-images.tsx b/src/pages/playground/components/ground-images.tsx
@@ -11,7 +11,7 @@ import {
 } from '@/utils/fetch-chunk-data';
 import { FileImageOutlined, SwapOutlined } from '@ant-design/icons';
 import { useIntl, useSearchParams } from '@umijs/max';
-import { Button, Divider, Form, Tooltip } from 'antd';
+import { Button, Form, Tooltip } from 'antd';
 import classNames from 'classnames';
 import _ from 'lodash';
 import 'overlayscrollbars/overlayscrollbars.css';
@@ -540,11 +540,16 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
             placeholer={intl.formatMessage({
               id: 'playground.input.prompt.holder'
             })}
-            actions={['clear']}
+            actions={['clear', 'upload']}
             defaultSize={{
               minRows: 5,
               maxRows: 5
             }}
+            title={
+              <span className="font-600">
+                {intl.formatMessage({ id: 'playground.image.prompt' })}
+              </span>
+            }
             loading={loading}
             disabled={!parameters.model}
             isEmpty={!imageList.length}
@@ -555,10 +560,6 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
             clearAll={handleClear}
             tools={
               <>
-                <span className="font-600">
-                  {intl.formatMessage({ id: 'playground.image.prompt' })}
-                </span>
-                <Divider type="vertical" />
                 <Tooltip
                   title={intl.formatMessage({
                     id: 'playground.image.prompt.random'

diff --git a/src/pages/playground/components/message-input.tsx b/src/pages/playground/components/message-input.tsx
@@ -355,6 +355,7 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
       <div className="messageInput" style={{ ...style }}>
         <div className="tool-bar">
           <div className="actions">
+            {title}
             {
               <>
                 {actions.includes('role') && (

diff --git a/src/pages/playground/components/thumb-img.tsx b/src/pages/playground/components/thumb-img.tsx
@@ -169,6 +169,7 @@ const ThumbImg: React.FC<{
                 return (
                   <SingleImage
                     {...item}
+                    key={item.uid}
                     autoSize={autoSize}
                     editable={editable}
                     autoBgColor={autoBgColor}

diff --git a/src/pages/playground/config/params-config.ts b/src/pages/playground/config/params-config.ts
@@ -319,6 +319,9 @@ export const ImageAdvancedParamsConfig: ParamsSchema[] = [
       text: 'playground.image.params.negativePrompt',
       isLocalized: true
     },
+    attrs: {
+      trim: false
+    },
     rules: [
       {
         required: false

diff --git a/src/pages/playground/style/thumb-img.less b/src/pages/playground/style/thumb-img.less
@@ -34,6 +34,10 @@
     width: 16px;
     overflow: hidden;
     pointer-events: all;
+
+    &:hover {
+      transform: scale(1.1);
+    }
   }
 
   &:hover {