Skip to content

Commit

Permalink
examples: file upload & ffmpeg example (#705)
Browse files Browse the repository at this point in the history
  • Loading branch information
Yonom authored Sep 8, 2024
1 parent 44d08bd commit 21716fd
Show file tree
Hide file tree
Showing 17 changed files with 856 additions and 3 deletions.
3 changes: 3 additions & 0 deletions examples/with-ffmpeg/.eslintrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"extends": "next/core-web-vitals"
}
1 change: 1 addition & 0 deletions examples/with-ffmpeg/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.vercel
51 changes: 51 additions & 0 deletions examples/with-ffmpeg/app/MyRuntimeProvider.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"use client";

import { AssistantRuntimeProvider, useEdgeRuntime } from "@assistant-ui/react";
import { AttachmentAdapter } from "@assistant-ui/react";
import { INTERNAL } from "@assistant-ui/react";

const { generateId } = INTERNAL;

const attachmentAdapter: AttachmentAdapter = {
async add({ file }) {
return {
id: generateId(),
type: "file",
name: file.name,
file,
};
},
async send(attachment) {
return {
content: [
{
type: "text",
text: `[User attached a file: ${attachment.name}]`,
},
],
};
},
async remove() {
// noop
},
};

export function MyRuntimeProvider({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
const runtime = useEdgeRuntime({
api: "/api/chat",
maxToolRoundtrips: 3,
adapters: {
attachments: attachmentAdapter,
},
});

return (
<AssistantRuntimeProvider runtime={runtime}>
{children}
</AssistantRuntimeProvider>
);
}
8 changes: 8 additions & 0 deletions examples/with-ffmpeg/app/NoSSRWrapper.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import dynamic from "next/dynamic";
import { FC, PropsWithChildren } from "react";

const NoSSRWrapper: FC<PropsWithChildren> = (props) => <>{props.children}</>;

export default dynamic(() => Promise.resolve(NoSSRWrapper), {
ssr: false,
});
8 changes: 8 additions & 0 deletions examples/with-ffmpeg/app/api/chat/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { openai } from "@ai-sdk/openai";
import { createEdgeRuntimeAPI } from "@assistant-ui/react/edge";

export const runtime = "edge";

export const { POST } = createEdgeRuntimeAPI({
model: openai("gpt-4o-2024-08-06"),
});
76 changes: 76 additions & 0 deletions examples/with-ffmpeg/app/globals.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
@tailwind base;
@tailwind components;
@tailwind utilities;

@layer base {
:root {
--background: 0 0% 100%;
--foreground: 240 10% 3.9%;

--card: 0 0% 100%;
--card-foreground: 240 10% 3.9%;

--popover: 0 0% 100%;
--popover-foreground: 240 10% 3.9%;

--primary: 240 5.9% 10%;
--primary-foreground: 0 0% 98%;

--secondary: 240 4.8% 95.9%;
--secondary-foreground: 240 5.9% 10%;

--muted: 240 4.8% 95.9%;
--muted-foreground: 240 3.8% 46.1%;

--accent: 240 4.8% 95.9%;
--accent-foreground: 240 5.9% 10%;

--destructive: 0 84.2% 60.2%;
--destructive-foreground: 0 0% 98%;

--border: 240 5.9% 90%;
--input: 240 5.9% 90%;
--ring: 240 10% 3.9%;

--radius: 0.5rem;
}

.dark {
--background: 240 10% 3.9%;
--foreground: 0 0% 98%;

--card: 240 10% 3.9%;
--card-foreground: 0 0% 98%;

--popover: 240 10% 3.9%;
--popover-foreground: 0 0% 98%;

--primary: 0 0% 98%;
--primary-foreground: 240 5.9% 10%;

--secondary: 240 3.7% 15.9%;
--secondary-foreground: 0 0% 98%;

--muted: 240 3.7% 15.9%;
--muted-foreground: 240 5% 64.9%;

--accent: 240 3.7% 15.9%;
--accent-foreground: 0 0% 98%;

--destructive: 0 62.8% 30.6%;
--destructive-foreground: 0 0% 98%;

--border: 240 3.7% 15.9%;
--input: 240 3.7% 15.9%;
--ring: 240 4.9% 83.9%;
}
}

@layer base {
* {
@apply border-border;
}
body {
@apply bg-background text-foreground;
}
}
23 changes: 23 additions & 0 deletions examples/with-ffmpeg/app/layout.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import "./globals.css";

import { cn } from "@/lib/utils";
import { Montserrat } from "next/font/google";
import { MyRuntimeProvider } from "./MyRuntimeProvider";

const montserrat = Montserrat({ subsets: ["latin"] });

export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
return (
<MyRuntimeProvider>
<html lang="en">
<body className={cn(montserrat.className, "h-dvh")}>
{children}
</body>
</html>
</MyRuntimeProvider>
);
}
179 changes: 179 additions & 0 deletions examples/with-ffmpeg/app/page.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
"use client";

import {
Thread,
useAssistantInstructions,
useAssistantTool,
useThreadContext,
} from "@assistant-ui/react";
import { z } from "zod";
import { FFmpeg } from "@ffmpeg/ffmpeg";
import { toBlobURL } from "@ffmpeg/util";
import { FC, useEffect, useRef, useState } from "react";
import {
CircleCheckIcon,
RefreshCcwIcon,
TriangleAlertIcon,
} from "lucide-react";

// MVP: upload file, enter command
// MVP: convert command to tool call
// MVP: tool call: ffmpeg

const FfmpegTool: FC<{ file: File }> = ({ file }) => {
const loadingRef = useRef(false);
const ffmpegRef = useRef(new FFmpeg());

useEffect(() => {
if (loadingRef.current) return;
loadingRef.current = true;

const load = async () => {
const baseURL = "https://unpkg.com/@ffmpeg/[email protected]/dist/umd";
const ffmpeg = ffmpegRef.current;
// toBlobURL is used to bypass CORS issue, urls with the same
// domain can be used directly.
await ffmpeg.load({
coreURL: await toBlobURL(
`${baseURL}/ffmpeg-core.js`,
"text/javascript",
),
wasmURL: await toBlobURL(
`${baseURL}/ffmpeg-core.wasm`,
"application/wasm",
),
});
};
load();
}, []);

useAssistantInstructions("The user has attached a file: " + file.name);

useAssistantTool({
toolName: "run_ffmpeg",
parameters: z.object({
command: z
.string()
.array()
.describe("The ffmpeg command line arguments to provide"),
outputFileName: z
.string()
.describe(
"The name of the output file including extension, corresponding to the command provided",
),
outputMimeType: z
.string()
.describe("The mime type of the output file, e.g. image/png"),
}),
execute: async ({ command }) => {
const transcode = async () => {
const ffmpeg = ffmpegRef.current;

const logs: string[] = [];
const logger = ({ message }: { message: string }) => {
logs.push(message);
};
ffmpeg.on("log", logger);

await ffmpeg.writeFile(
file.name,
new Uint8Array(await file.arrayBuffer()),
);

const code = await ffmpeg.exec(command);
ffmpeg.off("log", logger);

return { code, logs };
};
const { code, logs } = await transcode();

return {
success: code === 0,
hint:
code === 0
? "note: a download button is appearing in the chat for the user"
: "some error happened, logs: " + logs.join("\n"),
};
},
render: function RenderFfmpeg({
part: {
args: { command, outputFileName, outputMimeType },
result: { success } = {},
},
}) {
const handleDownload = async () => {
const ffmpeg = ffmpegRef.current;
const data = (await ffmpeg.readFile(outputFileName)) as any;
window.open(
URL.createObjectURL(
new Blob([data.buffer], { type: outputMimeType }),
),
"_blank",
);
};
return (
<div className="flex flex-col gap-2 rounded-lg border px-5 py-4">
<div>
<div className="flex items-center gap-2">
{success == null && (
<RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
)}
{success === false && (
<TriangleAlertIcon className="size-4 text-red-600" />
)}
{success === true && (
<CircleCheckIcon className="size-4 text-green-600" />
)}
<p>Running ffmpeg</p>
</div>
<pre className="font-sm overflow-y-scroll">
ffmpeg {command?.join(" ")}
</pre>
</div>
{!!success && (
<div className="mt-2 border-t border-dashed pt-3">
<button onClick={handleDownload}>
Download {outputFileName}
</button>
</div>
)}
{success === false && (
<div className="mt-2 border-t border-dashed pt-3">
Encountered an error.
</div>
)}
</div>
);
},
});

return null;
};

export default function Home() {
const [lastFile, setLastFile] = useState<File | null>(null);
const { useComposer } = useThreadContext();
const attachments = useComposer((c) => c.attachments);
useEffect(() => {
const lastAttachment = attachments[attachments.length - 1];
if (!lastAttachment) return;
setLastFile(lastAttachment.file!);
}, [attachments]);

console.log(lastFile);
return (
<div className="flex h-full flex-col">
<div className="border-b">
<p className="my-4 ml-8 text-xl font-bold">
ConvertGPT (built with{" "}
<a href="https://github.com/Yonom/assistant-ui" className="underline">
assistant-ui
</a>
)
</p>
</div>
<Thread />
{lastFile && <FfmpegTool file={lastFile} />}
</div>
);
}
17 changes: 17 additions & 0 deletions examples/with-ffmpeg/components.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"$schema": "https://ui.shadcn.com/schema.json",
"style": "new-york",
"rsc": true,
"tsx": true,
"tailwind": {
"config": "tailwind.config.ts",
"css": "app/globals.css",
"baseColor": "zinc",
"cssVariables": true,
"prefix": ""
},
"aliases": {
"components": "@/components",
"utils": "@/lib/utils"
}
}
6 changes: 6 additions & 0 deletions examples/with-ffmpeg/lib/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from "clsx";
import { twMerge } from "tailwind-merge";

export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}
5 changes: 5 additions & 0 deletions examples/with-ffmpeg/next-env.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />

// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.
4 changes: 4 additions & 0 deletions examples/with-ffmpeg/next.config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
/** @type {import('next').NextConfig} */
const nextConfig = {};

export default nextConfig;
Loading

0 comments on commit 21716fd

Please sign in to comment.