From 44d08bdca62aa9d53c3f9ddf1be13a5002f67338 Mon Sep 17 00:00:00 2001 From: Simon Farshid Date: Sat, 7 Sep 2024 17:52:45 -0700 Subject: [PATCH] feat: Edge/Local runtime AttachmentAdapter support (#774) --- .changeset/early-ears-juggle.md | 5 ++ .changeset/unlucky-pants-destroy.md | 6 +++ .../src/lib/playground-runtime.ts | 1 + packages/react/src/context/stores/Composer.ts | 17 ++----- .../composer/useComposerAddAttachment.tsx | 8 +-- .../runtimes/attachment/AttachmentAdapter.ts | 10 ++++ .../react/src/runtimes/attachment/index.ts | 1 + .../react/src/runtimes/core/ThreadRuntime.tsx | 6 ++- .../ExternalStoreThreadRuntime.tsx | 1 + packages/react/src/runtimes/index.ts | 1 + .../runtimes/local/LocalRuntimeOptions.tsx | 2 + .../src/runtimes/local/LocalThreadRuntime.tsx | 14 ++++- .../runtimes/utils/ThreadRuntimeComposer.tsx | 51 +++++++++++++++++-- packages/react/src/ui/composer.tsx | 5 +- 14 files changed, 99 insertions(+), 29 deletions(-) create mode 100644 .changeset/early-ears-juggle.md create mode 100644 .changeset/unlucky-pants-destroy.md create mode 100644 packages/react/src/runtimes/attachment/AttachmentAdapter.ts create mode 100644 packages/react/src/runtimes/attachment/index.ts diff --git a/.changeset/early-ears-juggle.md b/.changeset/early-ears-juggle.md new file mode 100644 index 000000000..3c4cc3b4f --- /dev/null +++ b/.changeset/early-ears-juggle.md @@ -0,0 +1,5 @@ +--- +"@assistant-ui/react": patch +--- + +feat: styled components for attachments diff --git a/.changeset/unlucky-pants-destroy.md b/.changeset/unlucky-pants-destroy.md new file mode 100644 index 000000000..720d0a48f --- /dev/null +++ b/.changeset/unlucky-pants-destroy.md @@ -0,0 +1,6 @@ +--- +"@assistant-ui/react-playground": patch +"@assistant-ui/react": patch +--- + +feat: Edge/Local runtime AttachmentAdapter support diff --git a/packages/react-playground/src/lib/playground-runtime.ts b/packages/react-playground/src/lib/playground-runtime.ts index a7f9ad379..03d412afc 100644 --- a/packages/react-playground/src/lib/playground-runtime.ts +++ b/packages/react-playground/src/lib/playground-runtime.ts @@ -110,6 +110,7 @@ export class PlaygroundThreadRuntime implements ReactThreadRuntime { private configProvider = new ProxyConfigProvider(); public readonly composer = new ThreadRuntimeComposer( + this, this.notifySubscribers.bind(this), ); diff --git a/packages/react/src/context/stores/Composer.ts b/packages/react/src/context/stores/Composer.ts index 7e5423438..e1b9fd691 100644 --- a/packages/react/src/context/stores/Composer.ts +++ b/packages/react/src/context/stores/Composer.ts @@ -11,7 +11,7 @@ export type ComposerState = Readonly<{ setValue: (value: string) => void; attachments: readonly Attachment[]; - addAttachment: (attachment: Attachment) => void; + addAttachment: (file: File) => void; removeAttachment: (attachmentId: string) => void; text: string; @@ -43,8 +43,8 @@ export const makeComposerStore = ( }, attachments: runtime.composer.attachments, - addAttachment: (attachment) => { - useThreadRuntime.getState().composer.addAttachment(attachment); + addAttachment: (file) => { + useThreadRuntime.getState().composer.addAttachment(file); }, removeAttachment: (attachmentId) => { useThreadRuntime.getState().composer.removeAttachment(attachmentId); @@ -63,16 +63,7 @@ export const makeComposerStore = ( send: () => { const runtime = useThreadRuntime.getState(); - const text = runtime.composer.text; - const attachments = runtime.composer.attachments; - runtime.composer.reset(); - - runtime.append({ - parentId: runtime.messages.at(-1)?.id ?? null, - role: "user", - content: text ? [{ type: "text", text }] : [], - attachments, - }); + runtime.composer.send(); }, cancel: () => { useThreadRuntime.getState().cancelRun(); diff --git a/packages/react/src/primitive-hooks/composer/useComposerAddAttachment.tsx b/packages/react/src/primitive-hooks/composer/useComposerAddAttachment.tsx index 950cf26bb..0f0e7eacf 100644 --- a/packages/react/src/primitive-hooks/composer/useComposerAddAttachment.tsx +++ b/packages/react/src/primitive-hooks/composer/useComposerAddAttachment.tsx @@ -1,6 +1,5 @@ import { useCallback } from "react"; import { useThreadContext } from "../../context"; -import { generateId } from "../../internal"; export const useComposerAddAttachment = () => { const { useComposer } = useThreadContext(); @@ -16,12 +15,7 @@ export const useComposerAddAttachment = () => { input.onchange = (e) => { const file = (e.target as HTMLInputElement).files?.[0]; if (!file) return; - addAttachment({ - id: generateId(), - type: "file", // TODO infer type from file extension or mimetype - name: file.name, - file, - }); + addAttachment(file); }; input.click(); diff --git a/packages/react/src/runtimes/attachment/AttachmentAdapter.ts b/packages/react/src/runtimes/attachment/AttachmentAdapter.ts new file mode 100644 index 000000000..eb4617725 --- /dev/null +++ b/packages/react/src/runtimes/attachment/AttachmentAdapter.ts @@ -0,0 +1,10 @@ +import { Attachment } from "../../context/stores/Attachment"; +import { CoreUserContentPart } from "../../types"; + +export type AttachmentAdapter = { + add(state: { file: File }): Promise; + send(attachment: Attachment): Promise<{ + content: CoreUserContentPart[]; + }>; + remove(attachment: Attachment): Promise; +}; diff --git a/packages/react/src/runtimes/attachment/index.ts b/packages/react/src/runtimes/attachment/index.ts new file mode 100644 index 000000000..78a8fc180 --- /dev/null +++ b/packages/react/src/runtimes/attachment/index.ts @@ -0,0 +1 @@ +export type { AttachmentAdapter } from "./AttachmentAdapter"; diff --git a/packages/react/src/runtimes/core/ThreadRuntime.tsx b/packages/react/src/runtimes/core/ThreadRuntime.tsx index 7199cd236..a7c8e1760 100644 --- a/packages/react/src/runtimes/core/ThreadRuntime.tsx +++ b/packages/react/src/runtimes/core/ThreadRuntime.tsx @@ -17,12 +17,14 @@ export type ThreadRuntime = ThreadActionsState & export declare namespace ThreadRuntime { export type Composer = Readonly<{ attachments: Attachment[]; - addAttachment: (attachment: Attachment) => void; - removeAttachment: (attachmentId: string) => void; + addAttachment: (file: File) => Promise; + removeAttachment: (attachmentId: string) => Promise; text: string; setText: (value: string) => void; reset: () => void; + + send: () => void; }>; } diff --git a/packages/react/src/runtimes/external-store/ExternalStoreThreadRuntime.tsx b/packages/react/src/runtimes/external-store/ExternalStoreThreadRuntime.tsx index d25cc2da3..fdc4b328c 100644 --- a/packages/react/src/runtimes/external-store/ExternalStoreThreadRuntime.tsx +++ b/packages/react/src/runtimes/external-store/ExternalStoreThreadRuntime.tsx @@ -49,6 +49,7 @@ export class ExternalStoreThreadRuntime implements ReactThreadRuntime { private _store!: ExternalStoreAdapter; public readonly composer = new ThreadRuntimeComposer( + this, this.notifySubscribers.bind(this), ); diff --git a/packages/react/src/runtimes/index.ts b/packages/react/src/runtimes/index.ts index fd90b90c0..580cdf9ec 100644 --- a/packages/react/src/runtimes/index.ts +++ b/packages/react/src/runtimes/index.ts @@ -4,3 +4,4 @@ export * from "./edge"; export * from "./external-store"; export * from "./dangerous-in-browser"; export * from "./speech"; +export * from "./attachment"; diff --git a/packages/react/src/runtimes/local/LocalRuntimeOptions.tsx b/packages/react/src/runtimes/local/LocalRuntimeOptions.tsx index adaef7166..ec3045bda 100644 --- a/packages/react/src/runtimes/local/LocalRuntimeOptions.tsx +++ b/packages/react/src/runtimes/local/LocalRuntimeOptions.tsx @@ -1,4 +1,5 @@ import type { CoreMessage } from "../../types"; +import { AttachmentAdapter } from "../attachment/AttachmentAdapter"; import { SpeechSynthesisAdapter } from "../speech/SpeechAdapterTypes"; export type LocalRuntimeOptions = { @@ -6,6 +7,7 @@ export type LocalRuntimeOptions = { maxToolRoundtrips?: number | undefined; adapters?: | { + attachments?: AttachmentAdapter | undefined; speech?: SpeechSynthesisAdapter | undefined; } | undefined; diff --git a/packages/react/src/runtimes/local/LocalThreadRuntime.tsx b/packages/react/src/runtimes/local/LocalThreadRuntime.tsx index fb4522aa2..5e3088a73 100644 --- a/packages/react/src/runtimes/local/LocalThreadRuntime.tsx +++ b/packages/react/src/runtimes/local/LocalThreadRuntime.tsx @@ -43,6 +43,7 @@ export class LocalThreadRuntime implements ThreadRuntime { } public readonly composer = new ThreadRuntimeComposer( + this, this.notifySubscribers.bind(this), ); @@ -72,11 +73,22 @@ export class LocalThreadRuntime implements ThreadRuntime { public set options({ initialMessages, ...options }: LocalRuntimeOptions) { this._options = options; + let hasUpdates = false; + const canSpeak = options.adapters?.speech !== undefined; if (this.capabilities.speak !== canSpeak) { this.capabilities.speak = canSpeak; - this.notifySubscribers(); + hasUpdates = true; } + + this.composer.adapter = options.adapters?.attachments; + const canAttach = this.composer.adapter !== undefined; + if (this.capabilities.attachments !== canAttach) { + this.capabilities.attachments = canAttach; + hasUpdates = true; + } + + if (hasUpdates) this.notifySubscribers(); } public getBranches(messageId: string): string[] { diff --git a/packages/react/src/runtimes/utils/ThreadRuntimeComposer.tsx b/packages/react/src/runtimes/utils/ThreadRuntimeComposer.tsx index 5701b85eb..03bd0ddf8 100644 --- a/packages/react/src/runtimes/utils/ThreadRuntimeComposer.tsx +++ b/packages/react/src/runtimes/utils/ThreadRuntimeComposer.tsx @@ -1,8 +1,18 @@ import { Attachment } from "../../context/stores/Attachment"; +import { AppendMessage } from "../../types"; +import { AttachmentAdapter } from "../attachment/AttachmentAdapter"; import { ThreadRuntime } from "../core"; export class ThreadRuntimeComposer implements ThreadRuntime.Composer { - constructor(private notifySubscribers: () => void) {} + public adapter?: AttachmentAdapter | undefined; + + constructor( + private runtime: { + messages: ThreadRuntime["messages"]; + append: (message: AppendMessage) => void; + }, + private notifySubscribers: () => void, + ) {} private _attachments: Attachment[] = []; @@ -10,13 +20,25 @@ export class ThreadRuntimeComposer implements ThreadRuntime.Composer { return this._attachments; } - addAttachment(attachment: Attachment) { + async addAttachment(file: File) { + if (!this.adapter) throw new Error("Attachments are not supported"); + + const attachment = await this.adapter.add({ file }); + this._attachments = [...this._attachments, attachment]; this.notifySubscribers(); } - removeAttachment(attachmentId: string) { - this._attachments = this._attachments.filter((a) => a.id !== attachmentId); + async removeAttachment(attachmentId: string) { + if (!this.adapter) throw new Error("Attachments are not supported"); + + const index = this._attachments.findIndex((a) => a.id === attachmentId); + if (index === -1) throw new Error("Attachment not found"); + const attachment = this._attachments[index]!; + + await this.adapter.remove(attachment); + + this._attachments = this._attachments.toSpliced(index, 1); this.notifySubscribers(); } @@ -36,4 +58,25 @@ export class ThreadRuntimeComposer implements ThreadRuntime.Composer { this._attachments = []; this.notifySubscribers(); } + + public async send() { + const attachmentContentParts = this.adapter + ? await Promise.all( + this.attachments.map(async (a) => { + const { content } = await this.adapter!.send(a); + return content; + }), + ) + : []; + + this.runtime.append({ + parentId: this.runtime.messages.at(-1)?.id ?? null, + role: "user", + content: this.text + ? [{ type: "text", text: this.text }, ...attachmentContentParts.flat()] + : [], + attachments: this.attachments, + }); + this.reset(); + } } diff --git a/packages/react/src/ui/composer.tsx b/packages/react/src/ui/composer.tsx index 82baac59e..22bfef72c 100644 --- a/packages/react/src/ui/composer.tsx +++ b/packages/react/src/ui/composer.tsx @@ -106,12 +106,13 @@ const ComposerRemoveAttachment = forwardRef< } = useThreadConfig(); const { useComposer } = useThreadContext(); + const { useAttachment } = useAttachmentContext(); const handleRemoveAttachment = () => { - // TODO delete the correct attachment useComposer .getState() - .removeAttachment(useComposer.getState().attachments[0]?.id!); + .removeAttachment(useAttachment.getState().attachment.id); }; + return (