Introduces Streaming

examples/json.ts

@@ -0,0 +1,39 @@
+#!/usr/bin/env -S npx ts-node --transpileOnly
+
+import { Substrate, GenerateJSON, GenerateText, sb } from "substrate";
+
+async function main() {
+  const SUBSTRATE_API_KEY = process.env["SUBSTRATE_API_KEY"];
+
+  const substrate = new Substrate({ apiKey: SUBSTRATE_API_KEY });
+
+  const author = new GenerateJSON({
+    prompt: "Who wrote Don Quixote?",
+    json_schema: {
+      type: "object",
+      properties: {
+        name: {
+          type: "string",
+          description: "The name of the author.",
+        },
+        bio: {
+          type: "string",
+          description: "Concise biography of the author.",
+        },
+      },
+    },
+    temperature: 0.4,
+    max_tokens: 800,
+  });
+
+  const name = author.future.json_object.get("name");
+  const bio = author.future.json_object.get("bio");
+
+  const report = new GenerateText({
+    prompt: sb.interpolate`Write a short summary about ${name} and make sure to use the following bio: ${bio}`,
+  });
+
+  const res = await substrate.run(author, report);
+  console.log(JSON.stringify(res.json, null, 2));
+}
+main();

examples/streaming.ts

@@ -0,0 +1,23 @@
+#!/usr/bin/env -S npx ts-node --transpileOnly
+
+import { Substrate, Llama3Instruct70B } from "substrate";
+
+async function main() {
+  const SUBSTRATE_API_KEY = process.env["SUBSTRATE_API_KEY"];
+
+  const substrate = new Substrate({ apiKey: SUBSTRATE_API_KEY });
+
+  const a = new Llama3Instruct70B({
+    prompt: "what are server side events useful for?",
+    max_tokens: 50,
+  });
+
+  const stream = await substrate.stream(a);
+
+  for await (let message of stream.get(a)) {
+    if (message.object === "node.delta") {
+      console.log(message);
+    }
+  }
+}
+main();

examples/streaming/nextjs-basic/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/streaming/nextjs-basic/README.md

@@ -0,0 +1,31 @@
+# Simple NextJS example
+
+This example uses a NextJS app that makes a request to Substrate in a route handler and sends the Server-Sent Events directly back to the client.
+
+On the client we can use a helper to decode these messsages into JavaScript objects that can be used to render out the content as it arrives.
+
+## Setup
+
+I'm using my local package for running this example using `npm link`.
+
+```
+# in the project root
+nvm use
+npm link
+
+# in this example directory
+npm link substrate
+```
+
+## Running the example
+
+```
+# install the dependencies
+npm install
+
+# run the dev server
+npm run dev
+
+# open your browser to use it (on localhost:3000 by default)
+open http://localhost:3000
+```

examples/streaming/nextjs-basic/app/Demo.tsx

@@ -0,0 +1,50 @@
+"use client";
+
+import { useState } from "react";
+import { sb } from "substrate";
+
+export default function Prompt() {
+  const [output, setOutput] = useState<string>("");
+
+  async function submitPrompt(event: any) {
+    event.preventDefault();
+
+    const formData = new FormData(event.currentTarget);
+
+    const request = new Request("/api/generate-text", {
+      method: "POST",
+      body: formData,
+    });
+    const response = await fetch(request);
+
+    if (response.ok) {
+      setOutput("");
+      const stream = await sb.streaming.fromSSEResponse(response);
+
+      for await (let message of stream) {
+        if (message.object === "node.delta") {
+          setOutput((state) => state + message.data.choices.item.text);
+        }
+      }
+    }
+  }
+
+  return (
+    <div className="w-full flex flex-col space-y-10">
+      <form className="w-full flex flex-row space-x-2" onSubmit={submitPrompt}>
+        <input
+          name="prompt"
+          autoFocus={true}
+          placeholder="Enter your prompt..."
+          type="text"
+          className="p-2 grow"
+        />
+        <button type="submit" className="rounded p-2">
+          Submit
+        </button>
+      </form>
+
+      <pre className="whitespace-pre-wrap">{output}</pre>
+    </div>
+  );
+}

examples/streaming/nextjs-basic/app/api/generate-text/route.ts

@@ -0,0 +1,22 @@
+"use server";
+
+import { Substrate, Llama3Instruct70B } from "substrate";
+
+const SUBSTRATE_API_KEY = process.env["SUBSTRATE_API_KEY"];
+
+const substrate = new Substrate({ apiKey: SUBSTRATE_API_KEY });
+
+export async function POST(request: Request) {
+  const formData = await request.formData();
+  const prompt =
+    formData.get("prompt")?.toString() ||
+    "write a short error message that informs a user they should fill in the prompt field";
+
+  const node = new Llama3Instruct70B({ prompt });
+
+  const streamResponse = await substrate.stream(node);
+  const body = streamResponse.apiResponse.body;
+  return new Response(body, {
+    headers: { "Content-Type": "text/event-stream" },
+  });
+}

examples/streaming/nextjs-basic/app/globals.css

@@ -0,0 +1,33 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --foreground-rgb: 0, 0, 0;
+  --background-start-rgb: 214, 219, 220;
+  --background-end-rgb: 255, 255, 255;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --foreground-rgb: 255, 255, 255;
+    --background-start-rgb: 0, 0, 0;
+    --background-end-rgb: 0, 0, 0;
+  }
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}
+
+@layer utilities {
+  .text-balance {
+    text-wrap: balance;
+  }
+}

examples/streaming/nextjs-basic/app/layout.tsx

@@ -0,0 +1,22 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "Create Next App",
+  description: "Generated by create next app",
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

examples/streaming/nextjs-basic/app/page.tsx

@@ -0,0 +1,9 @@
+import Demo from "./Demo";
+
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-between p-24">
+      <Demo />
+    </main>
+  );
+}

examples/streaming/nextjs-basic/next.config.mjs

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;