-
Notifications
You must be signed in to change notification settings - Fork 367
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
examples: file upload & ffmpeg example (#705)
- Loading branch information
Showing
17 changed files
with
856 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"extends": "next/core-web-vitals" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
.vercel |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
"use client"; | ||
|
||
import { AssistantRuntimeProvider, useEdgeRuntime } from "@assistant-ui/react"; | ||
import { AttachmentAdapter } from "@assistant-ui/react"; | ||
import { INTERNAL } from "@assistant-ui/react"; | ||
|
||
const { generateId } = INTERNAL; | ||
|
||
const attachmentAdapter: AttachmentAdapter = { | ||
async add({ file }) { | ||
return { | ||
id: generateId(), | ||
type: "file", | ||
name: file.name, | ||
file, | ||
}; | ||
}, | ||
async send(attachment) { | ||
return { | ||
content: [ | ||
{ | ||
type: "text", | ||
text: `[User attached a file: ${attachment.name}]`, | ||
}, | ||
], | ||
}; | ||
}, | ||
async remove() { | ||
// noop | ||
}, | ||
}; | ||
|
||
export function MyRuntimeProvider({ | ||
children, | ||
}: Readonly<{ | ||
children: React.ReactNode; | ||
}>) { | ||
const runtime = useEdgeRuntime({ | ||
api: "/api/chat", | ||
maxToolRoundtrips: 3, | ||
adapters: { | ||
attachments: attachmentAdapter, | ||
}, | ||
}); | ||
|
||
return ( | ||
<AssistantRuntimeProvider runtime={runtime}> | ||
{children} | ||
</AssistantRuntimeProvider> | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
import dynamic from "next/dynamic"; | ||
import { FC, PropsWithChildren } from "react"; | ||
|
||
const NoSSRWrapper: FC<PropsWithChildren> = (props) => <>{props.children}</>; | ||
|
||
export default dynamic(() => Promise.resolve(NoSSRWrapper), { | ||
ssr: false, | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
import { openai } from "@ai-sdk/openai"; | ||
import { createEdgeRuntimeAPI } from "@assistant-ui/react/edge"; | ||
|
||
export const runtime = "edge"; | ||
|
||
export const { POST } = createEdgeRuntimeAPI({ | ||
model: openai("gpt-4o-2024-08-06"), | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
@tailwind base; | ||
@tailwind components; | ||
@tailwind utilities; | ||
|
||
@layer base { | ||
:root { | ||
--background: 0 0% 100%; | ||
--foreground: 240 10% 3.9%; | ||
|
||
--card: 0 0% 100%; | ||
--card-foreground: 240 10% 3.9%; | ||
|
||
--popover: 0 0% 100%; | ||
--popover-foreground: 240 10% 3.9%; | ||
|
||
--primary: 240 5.9% 10%; | ||
--primary-foreground: 0 0% 98%; | ||
|
||
--secondary: 240 4.8% 95.9%; | ||
--secondary-foreground: 240 5.9% 10%; | ||
|
||
--muted: 240 4.8% 95.9%; | ||
--muted-foreground: 240 3.8% 46.1%; | ||
|
||
--accent: 240 4.8% 95.9%; | ||
--accent-foreground: 240 5.9% 10%; | ||
|
||
--destructive: 0 84.2% 60.2%; | ||
--destructive-foreground: 0 0% 98%; | ||
|
||
--border: 240 5.9% 90%; | ||
--input: 240 5.9% 90%; | ||
--ring: 240 10% 3.9%; | ||
|
||
--radius: 0.5rem; | ||
} | ||
|
||
.dark { | ||
--background: 240 10% 3.9%; | ||
--foreground: 0 0% 98%; | ||
|
||
--card: 240 10% 3.9%; | ||
--card-foreground: 0 0% 98%; | ||
|
||
--popover: 240 10% 3.9%; | ||
--popover-foreground: 0 0% 98%; | ||
|
||
--primary: 0 0% 98%; | ||
--primary-foreground: 240 5.9% 10%; | ||
|
||
--secondary: 240 3.7% 15.9%; | ||
--secondary-foreground: 0 0% 98%; | ||
|
||
--muted: 240 3.7% 15.9%; | ||
--muted-foreground: 240 5% 64.9%; | ||
|
||
--accent: 240 3.7% 15.9%; | ||
--accent-foreground: 0 0% 98%; | ||
|
||
--destructive: 0 62.8% 30.6%; | ||
--destructive-foreground: 0 0% 98%; | ||
|
||
--border: 240 3.7% 15.9%; | ||
--input: 240 3.7% 15.9%; | ||
--ring: 240 4.9% 83.9%; | ||
} | ||
} | ||
|
||
@layer base { | ||
* { | ||
@apply border-border; | ||
} | ||
body { | ||
@apply bg-background text-foreground; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
import "./globals.css"; | ||
|
||
import { cn } from "@/lib/utils"; | ||
import { Montserrat } from "next/font/google"; | ||
import { MyRuntimeProvider } from "./MyRuntimeProvider"; | ||
|
||
const montserrat = Montserrat({ subsets: ["latin"] }); | ||
|
||
export default function RootLayout({ | ||
children, | ||
}: Readonly<{ | ||
children: React.ReactNode; | ||
}>) { | ||
return ( | ||
<MyRuntimeProvider> | ||
<html lang="en"> | ||
<body className={cn(montserrat.className, "h-dvh")}> | ||
{children} | ||
</body> | ||
</html> | ||
</MyRuntimeProvider> | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,179 @@ | ||
"use client"; | ||
|
||
import { | ||
Thread, | ||
useAssistantInstructions, | ||
useAssistantTool, | ||
useThreadContext, | ||
} from "@assistant-ui/react"; | ||
import { z } from "zod"; | ||
import { FFmpeg } from "@ffmpeg/ffmpeg"; | ||
import { toBlobURL } from "@ffmpeg/util"; | ||
import { FC, useEffect, useRef, useState } from "react"; | ||
import { | ||
CircleCheckIcon, | ||
RefreshCcwIcon, | ||
TriangleAlertIcon, | ||
} from "lucide-react"; | ||
|
||
// MVP: upload file, enter command | ||
// MVP: convert command to tool call | ||
// MVP: tool call: ffmpeg | ||
|
||
const FfmpegTool: FC<{ file: File }> = ({ file }) => { | ||
const loadingRef = useRef(false); | ||
const ffmpegRef = useRef(new FFmpeg()); | ||
|
||
useEffect(() => { | ||
if (loadingRef.current) return; | ||
loadingRef.current = true; | ||
|
||
const load = async () => { | ||
const baseURL = "https://unpkg.com/@ffmpeg/[email protected]/dist/umd"; | ||
const ffmpeg = ffmpegRef.current; | ||
// toBlobURL is used to bypass CORS issue, urls with the same | ||
// domain can be used directly. | ||
await ffmpeg.load({ | ||
coreURL: await toBlobURL( | ||
`${baseURL}/ffmpeg-core.js`, | ||
"text/javascript", | ||
), | ||
wasmURL: await toBlobURL( | ||
`${baseURL}/ffmpeg-core.wasm`, | ||
"application/wasm", | ||
), | ||
}); | ||
}; | ||
load(); | ||
}, []); | ||
|
||
useAssistantInstructions("The user has attached a file: " + file.name); | ||
|
||
useAssistantTool({ | ||
toolName: "run_ffmpeg", | ||
parameters: z.object({ | ||
command: z | ||
.string() | ||
.array() | ||
.describe("The ffmpeg command line arguments to provide"), | ||
outputFileName: z | ||
.string() | ||
.describe( | ||
"The name of the output file including extension, corresponding to the command provided", | ||
), | ||
outputMimeType: z | ||
.string() | ||
.describe("The mime type of the output file, e.g. image/png"), | ||
}), | ||
execute: async ({ command }) => { | ||
const transcode = async () => { | ||
const ffmpeg = ffmpegRef.current; | ||
|
||
const logs: string[] = []; | ||
const logger = ({ message }: { message: string }) => { | ||
logs.push(message); | ||
}; | ||
ffmpeg.on("log", logger); | ||
|
||
await ffmpeg.writeFile( | ||
file.name, | ||
new Uint8Array(await file.arrayBuffer()), | ||
); | ||
|
||
const code = await ffmpeg.exec(command); | ||
ffmpeg.off("log", logger); | ||
|
||
return { code, logs }; | ||
}; | ||
const { code, logs } = await transcode(); | ||
|
||
return { | ||
success: code === 0, | ||
hint: | ||
code === 0 | ||
? "note: a download button is appearing in the chat for the user" | ||
: "some error happened, logs: " + logs.join("\n"), | ||
}; | ||
}, | ||
render: function RenderFfmpeg({ | ||
part: { | ||
args: { command, outputFileName, outputMimeType }, | ||
result: { success } = {}, | ||
}, | ||
}) { | ||
const handleDownload = async () => { | ||
const ffmpeg = ffmpegRef.current; | ||
const data = (await ffmpeg.readFile(outputFileName)) as any; | ||
window.open( | ||
URL.createObjectURL( | ||
new Blob([data.buffer], { type: outputMimeType }), | ||
), | ||
"_blank", | ||
); | ||
}; | ||
return ( | ||
<div className="flex flex-col gap-2 rounded-lg border px-5 py-4"> | ||
<div> | ||
<div className="flex items-center gap-2"> | ||
{success == null && ( | ||
<RefreshCcwIcon className="size-4 animate-spin text-blue-600" /> | ||
)} | ||
{success === false && ( | ||
<TriangleAlertIcon className="size-4 text-red-600" /> | ||
)} | ||
{success === true && ( | ||
<CircleCheckIcon className="size-4 text-green-600" /> | ||
)} | ||
<p>Running ffmpeg</p> | ||
</div> | ||
<pre className="font-sm overflow-y-scroll"> | ||
ffmpeg {command?.join(" ")} | ||
</pre> | ||
</div> | ||
{!!success && ( | ||
<div className="mt-2 border-t border-dashed pt-3"> | ||
<button onClick={handleDownload}> | ||
Download {outputFileName} | ||
</button> | ||
</div> | ||
)} | ||
{success === false && ( | ||
<div className="mt-2 border-t border-dashed pt-3"> | ||
Encountered an error. | ||
</div> | ||
)} | ||
</div> | ||
); | ||
}, | ||
}); | ||
|
||
return null; | ||
}; | ||
|
||
export default function Home() { | ||
const [lastFile, setLastFile] = useState<File | null>(null); | ||
const { useComposer } = useThreadContext(); | ||
const attachments = useComposer((c) => c.attachments); | ||
useEffect(() => { | ||
const lastAttachment = attachments[attachments.length - 1]; | ||
if (!lastAttachment) return; | ||
setLastFile(lastAttachment.file!); | ||
}, [attachments]); | ||
|
||
console.log(lastFile); | ||
return ( | ||
<div className="flex h-full flex-col"> | ||
<div className="border-b"> | ||
<p className="my-4 ml-8 text-xl font-bold"> | ||
ConvertGPT (built with{" "} | ||
<a href="https://github.com/Yonom/assistant-ui" className="underline"> | ||
assistant-ui | ||
</a> | ||
) | ||
</p> | ||
</div> | ||
<Thread /> | ||
{lastFile && <FfmpegTool file={lastFile} />} | ||
</div> | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
{ | ||
"$schema": "https://ui.shadcn.com/schema.json", | ||
"style": "new-york", | ||
"rsc": true, | ||
"tsx": true, | ||
"tailwind": { | ||
"config": "tailwind.config.ts", | ||
"css": "app/globals.css", | ||
"baseColor": "zinc", | ||
"cssVariables": true, | ||
"prefix": "" | ||
}, | ||
"aliases": { | ||
"components": "@/components", | ||
"utils": "@/lib/utils" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
import { type ClassValue, clsx } from "clsx"; | ||
import { twMerge } from "tailwind-merge"; | ||
|
||
export function cn(...inputs: ClassValue[]) { | ||
return twMerge(clsx(inputs)); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
/// <reference types="next" /> | ||
/// <reference types="next/image-types/global" /> | ||
|
||
// NOTE: This file should not be edited | ||
// see https://nextjs.org/docs/basic-features/typescript for more information. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
/** @type {import('next').NextConfig} */ | ||
const nextConfig = {}; | ||
|
||
export default nextConfig; |
Oops, something went wrong.