From 58c341141a4a3e6ee237205c56dad30d4df73c29 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 24 Feb 2025 23:16:52 -0800 Subject: [PATCH] feat (provider/amazon-bedrock): add generate image support for Amazon Nova Canvas (#4974) Co-authored-by: Andrea Amorosi --- .changeset/eight-ducks-wait.md | 5 + .../03-ai-sdk-core/35-image-generation.mdx | 83 +++---- .../01-ai-sdk-providers/08-amazon-bedrock.mdx | 80 +++++++ .../src/generate-image/amazon-bedrock.ts | 23 ++ .../src/bedrock-image-model.test.ts | 223 ++++++++++++++++++ .../amazon-bedrock/src/bedrock-image-model.ts | 130 ++++++++++ .../src/bedrock-image-settings.ts | 14 ++ .../src/bedrock-provider.test.ts | 35 ++- .../amazon-bedrock/src/bedrock-provider.ts | 28 +++ 9 files changed, 579 insertions(+), 42 deletions(-) create mode 100644 .changeset/eight-ducks-wait.md create mode 100644 examples/ai-core/src/generate-image/amazon-bedrock.ts create mode 100644 packages/amazon-bedrock/src/bedrock-image-model.test.ts create mode 100644 packages/amazon-bedrock/src/bedrock-image-model.ts create mode 100644 packages/amazon-bedrock/src/bedrock-image-settings.ts diff --git a/.changeset/eight-ducks-wait.md b/.changeset/eight-ducks-wait.md new file mode 100644 index 000000000000..d5129ce02f94 --- /dev/null +++ b/.changeset/eight-ducks-wait.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/amazon-bedrock': patch +--- + +feat (provider/amazon-bedrock): add generate image support for Amazon Nova Canvas diff --git a/content/docs/03-ai-sdk-core/35-image-generation.mdx b/content/docs/03-ai-sdk-core/35-image-generation.mdx index cdb1f289001c..deb4a2fa7e7a 100644 --- a/content/docs/03-ai-sdk-core/35-image-generation.mdx +++ b/content/docs/03-ai-sdk-core/35-image-generation.mdx @@ -210,46 +210,47 @@ try { ## Image Models -| Provider | Model | Support sizes (`width x height`) or aspect ratios (`width : height`) | -| ----------------------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Replicate](/providers/ai-sdk-providers/replicate) | `black-forest-labs/flux-schnell` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | -| [Replicate](/providers/ai-sdk-providers/replicate) | `recraft-ai/recraft-v3` | 1024x1024, 1365x1024, 1024x1365, 1536x1024, 1024x1536, 1820x1024, 1024x1820, 1024x2048, 2048x1024, 1434x1024, 1024x1434, 1024x1280, 1280x1024, 1024x1707, 1707x1024 | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-dev-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-schnell-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-5-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/japanese-stable-diffusion-xl` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/SSD-1B` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | -| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | -| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-flash-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/flux/dev` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/fast-sdxl` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/flux-pro/v1.1-ultra` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/ideogram/v2` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/recraft-v3` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/stable-diffusion-3.5-large` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/hyper-sdxl` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `stabilityai/stable-diffusion-xl-base-1.0` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-dev` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-dev-lora` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-schnell` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-canny` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-depth` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-redux` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1.1-pro` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-pro` | 512x512, 768x768, 1024x1024 | -| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-schnell-Free` | 512x512, 768x768, 1024x1024 | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sd3.5` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1.1-pro` | 256-1440 (multiples of 32) | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1-schnell` | 256-1440 (multiples of 32) | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1-dev` | 256-1440 (multiples of 32) | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-pro` | 256-1440 (multiples of 32) | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sd3.5-medium` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | -| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sdxl-turbo` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | +| Provider | Model | Support sizes (`width x height`) or aspect ratios (`width : height`) | +| ------------------------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Amazon Bedrock](/providers/ai-sdk-providers/amazon-bedrock#image-models) | `amazon.nova-canvas-v1:0` | 320-4096 (multiples of 16), 1:4 to 4:1, max 4.2M pixels | +| [Replicate](/providers/ai-sdk-providers/replicate) | `black-forest-labs/flux-schnell` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | +| [Replicate](/providers/ai-sdk-providers/replicate) | `recraft-ai/recraft-v3` | 1024x1024, 1365x1024, 1024x1365, 1536x1024, 1024x1536, 1820x1024, 1024x1820, 1024x2048, 2048x1024, 1434x1024, 1024x1434, 1024x1280, 1280x1024, 1024x1707, 1707x1024 | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-dev-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-schnell-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-5-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/japanese-stable-diffusion-xl` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/SSD-1B` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-flash-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/flux/dev` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/fast-sdxl` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/flux-pro/v1.1-ultra` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/ideogram/v2` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/recraft-v3` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/stable-diffusion-3.5-large` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Fal](/providers/ai-sdk-providers/fal#image-models) | `fal-ai/hyper-sdxl` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `stabilityai/stable-diffusion-xl-base-1.0` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-dev` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-dev-lora` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-schnell` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-canny` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-depth` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-redux` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1.1-pro` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-pro` | 512x512, 768x768, 1024x1024 | +| [Together.ai](/providers/ai-sdk-providers/togetherai#image-models) | `black-forest-labs/FLUX.1-schnell-Free` | 512x512, 768x768, 1024x1024 | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sd3.5` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1.1-pro` | 256-1440 (multiples of 32) | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1-schnell` | 256-1440 (multiples of 32) | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-1-dev` | 256-1440 (multiples of 32) | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `black-forest-labs/FLUX-pro` | 256-1440 (multiples of 32) | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sd3.5-medium` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | +| [DeepInfra](/providers/ai-sdk-providers/deepinfra#image-models) | `stabilityai/sdxl-turbo` | 1:1, 16:9, 1:9, 3:2, 2:3, 4:5, 5:4, 9:16, 9:21 | Above are a small subset of the image models supported by the AI SDK providers. For more, see the respective provider documentation. diff --git a/content/providers/01-ai-sdk-providers/08-amazon-bedrock.mdx b/content/providers/01-ai-sdk-providers/08-amazon-bedrock.mdx index 8b91d2c2ad8d..cc44cf5e01eb 100644 --- a/content/providers/01-ai-sdk-providers/08-amazon-bedrock.mdx +++ b/content/providers/01-ai-sdk-providers/08-amazon-bedrock.mdx @@ -391,6 +391,86 @@ The following optional settings are available for Bedrock Titan embedding models | `amazon.titan-embed-text-v1` | 1536 | | | `amazon.titan-embed-text-v2:0` | 1024 | | +## Image Models + +You can create models that call the Bedrock API [Bedrock API](https://docs.aws.amazon.com/nova/latest/userguide/image-generation.html) +using the `.image()` factory method. + +For more on the Amazon Nova Canvas image model, see the [Nova Canvas +Overview](https://docs.aws.amazon.com/ai/responsible-ai/nova-canvas/overview.html). + + + The `amazon.nova-canvas-v1:0` model is available in the `us-east-1` region. + + +```ts +const model = bedrock.image('amazon.nova-canvas-v1:0'); +``` + +You can then generate images with the `experimental_generateImage` function: + +```ts +import { bedrock } from '@ai-sdk/amazon-bedrock'; +import { experimental_generateImage as generateImage } from 'ai'; + +const { image } = await generateImage({ + model: bedrock.imageModel('amazon.nova-canvas-v1:0'), + prompt: 'A beautiful sunset over a calm ocean', + size: '512x512', + seed: 42, +}); +``` + +You can also pass the `providerOptions` object to the `generateImage` function to customize the generation behavior: + +```ts +import { bedrock } from '@ai-sdk/amazon-bedrock'; +import { experimental_generateImage as generateImage } from 'ai'; + +const { image } = await generateImage({ + model: bedrock.imageModel('amazon.nova-canvas-v1:0'), + prompt: 'A beautiful sunset over a calm ocean', + size: '512x512', + seed: 42, + providerOptions: { bedrock: { quality: 'premium' } }, +}); +``` + +Documentation for additional settings can be found within the [Amazon Bedrock +User Guide for Amazon Nova +Documentation](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html). + +### Image Model Settings + +When creating an image model, you can customize the generation behavior with optional settings: + +```ts +const model = bedrock.imageModel('amazon.nova-canvas-v1:0', { + maxImagesPerCall: 1, // Maximum number of images to generate per API call +}); +``` + +- **maxImagesPerCall** _number_ + + Override the maximum number of images generated per API call. Default can vary + by model, with 5 as a common default. + +### Model Capabilities + +The Amazon Nova Canvas model supports custom sizes with constraints as follows: + +- Each side must be between 320-4096 pixels, inclusive. +- Each side must be evenly divisible by 16. +- The aspect ratio must be between 1:4 and 4:1. That is, one side can't be more than 4 times longer than the other side. +- The total pixel count must be less than 4,194,304. + +For more, see [Image generation access and +usage](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html). + +| Model | Sizes | +| ------------------------- | ----------------------------------------------------------------------------------------------------- | +| `amazon.nova-canvas-v1:0` | Custom sizes: 320-4096px per side (must be divisible by 16), aspect ratio 1:4 to 4:1, max 4.2M pixels | + ## Response Headers The Amazon Bedrock provider will return the response headers associated with diff --git a/examples/ai-core/src/generate-image/amazon-bedrock.ts b/examples/ai-core/src/generate-image/amazon-bedrock.ts new file mode 100644 index 000000000000..ab49896e09e1 --- /dev/null +++ b/examples/ai-core/src/generate-image/amazon-bedrock.ts @@ -0,0 +1,23 @@ +import { bedrock } from '@ai-sdk/amazon-bedrock'; +import { experimental_generateImage as generateImage } from 'ai'; +import { presentImages } from '../lib/present-image'; +import 'dotenv/config'; + +async function main() { + const result = await generateImage({ + model: bedrock.imageModel('amazon.nova-canvas-v1:0'), + prompt: + 'A salamander at dusk in a forest pond with fireflies in the background, in the style of anime', + size: '512x512', + seed: 42, + providerOptions: { + bedrock: { + quality: 'premium', + }, + }, + }); + + await presentImages(result.images); +} + +main().catch(console.error); diff --git a/packages/amazon-bedrock/src/bedrock-image-model.test.ts b/packages/amazon-bedrock/src/bedrock-image-model.test.ts new file mode 100644 index 000000000000..3b083750032e --- /dev/null +++ b/packages/amazon-bedrock/src/bedrock-image-model.test.ts @@ -0,0 +1,223 @@ +import { createTestServer } from '@ai-sdk/provider-utils/test'; +import { createAmazonBedrock } from './bedrock-provider'; +import { BedrockImageModel } from './bedrock-image-model'; +import { injectFetchHeaders } from './inject-fetch-headers'; + +const prompt = 'A cute baby sea otter'; + +const provider = createAmazonBedrock(); +const fakeFetchWithAuth = injectFetchHeaders({ 'x-amz-auth': 'test-auth' }); + +const invokeUrl = `https://bedrock-runtime.us-east-1.amazonaws.com/model/${encodeURIComponent( + 'amazon.nova-canvas-v1:0', +)}/invoke`; + +describe('doGenerate', () => { + const mockConfigHeaders = { + 'config-header': 'config-value', + 'shared-header': 'config-shared', + }; + + const server = createTestServer({ + [invokeUrl]: { + response: { + type: 'binary', + headers: { + 'content-type': 'application/json', + }, + body: Buffer.from( + JSON.stringify({ + images: ['base64-image-1', 'base64-image-2'], + }), + ), + }, + }, + }); + + const model = new BedrockImageModel( + 'amazon.nova-canvas-v1:0', + {}, + { + baseUrl: () => 'https://bedrock-runtime.us-east-1.amazonaws.com', + headers: mockConfigHeaders, + fetch: fakeFetchWithAuth, + }, + ); + + it('should pass the model and the settings', async () => { + await model.doGenerate({ + prompt, + n: 1, + size: '1024x1024', + aspectRatio: undefined, + seed: 1234, + providerOptions: { + bedrock: { + negativeText: 'bad', + quality: 'premium', + cfgScale: 1.2, + }, + }, + }); + + const body = await server.calls[0].requestBody; + expect(body).toStrictEqual({ + taskType: 'TEXT_IMAGE', + textToImageParams: { + text: prompt, + negativeText: 'bad', + }, + imageGenerationConfig: { + numberOfImages: 1, + seed: 1234, + quality: 'premium', + cfgScale: 1.2, + width: 1024, + height: 1024, + }, + }); + }); + + it('should properly combine headers from all sources', async () => { + const optionsHeaders = { + 'options-header': 'options-value', + 'shared-header': 'options-shared', + }; + + const modelWithHeaders = new BedrockImageModel( + 'amazon.nova-canvas-v1:0', + {}, + { + baseUrl: () => 'https://bedrock-runtime.us-east-1.amazonaws.com', + headers: { + 'model-header': 'model-value', + 'shared-header': 'model-shared', + }, + fetch: injectFetchHeaders({ + 'signed-header': 'signed-value', + authorization: 'AWS4-HMAC-SHA256...', + }), + }, + ); + + await modelWithHeaders.doGenerate({ + prompt, + n: 1, + size: undefined, + aspectRatio: undefined, + seed: undefined, + providerOptions: {}, + headers: optionsHeaders, + }); + + const requestHeaders = server.calls[0].requestHeaders; + expect(requestHeaders['options-header']).toBe('options-value'); + expect(requestHeaders['model-header']).toBe('model-value'); + expect(requestHeaders['signed-header']).toBe('signed-value'); + expect(requestHeaders['authorization']).toBe('AWS4-HMAC-SHA256...'); + expect(requestHeaders['shared-header']).toBe('options-shared'); + }); + + it('should respect maxImagesPerCall setting', async () => { + const customModel = provider.image('amazon.nova-canvas-v1:0', { + maxImagesPerCall: 2, + }); + expect(customModel.maxImagesPerCall).toBe(2); + + const defaultModel = provider.image('amazon.nova-canvas-v1:0'); + expect(defaultModel.maxImagesPerCall).toBe(5); // 'amazon.nova-canvas-v1:0','s default from settings + + const unknownModel = provider.image('unknown-model' as any); + expect(unknownModel.maxImagesPerCall).toBe(1); // fallback for unknown models + }); + + it('should return warnings for unsupported settings', async () => { + const result = await model.doGenerate({ + prompt, + n: 1, + size: '1024x1024', + aspectRatio: '1:1', + seed: undefined, + providerOptions: {}, + }); + + expect(result.warnings).toStrictEqual([ + { + type: 'unsupported-setting', + setting: 'aspectRatio', + details: + 'This model does not support aspect ratio. Use `size` instead.', + }, + ]); + }); + + it('should extract the generated images', async () => { + const result = await model.doGenerate({ + prompt, + n: 1, + size: undefined, + aspectRatio: undefined, + seed: undefined, + providerOptions: {}, + }); + + expect(result.images).toStrictEqual(['base64-image-1', 'base64-image-2']); + }); + + it('should include response data with timestamp, modelId and headers', async () => { + const testDate = new Date('2024-03-15T12:00:00Z'); + + const customModel = new BedrockImageModel( + 'amazon.nova-canvas-v1:0', + {}, + { + baseUrl: () => 'https://bedrock-runtime.us-east-1.amazonaws.com', + headers: () => ({}), + _internal: { + currentDate: () => testDate, + }, + }, + ); + + const result = await customModel.doGenerate({ + prompt, + n: 1, + size: '1024x1024', + aspectRatio: undefined, + seed: undefined, + providerOptions: {}, + }); + + expect(result.response).toStrictEqual({ + timestamp: testDate, + modelId: 'amazon.nova-canvas-v1:0', + headers: { + 'content-length': '46', + 'content-type': 'application/json', + }, + }); + }); + + it('should use real date when no custom date provider is specified', async () => { + const beforeDate = new Date(); + + const result = await model.doGenerate({ + prompt, + n: 1, + size: undefined, + aspectRatio: undefined, + seed: 1234, + providerOptions: {}, + }); + + const afterDate = new Date(); + + expect(result.response.timestamp.getTime()).toBeGreaterThanOrEqual( + beforeDate.getTime(), + ); + expect(result.response.timestamp.getTime()).toBeLessThanOrEqual( + afterDate.getTime(), + ); + expect(result.response.modelId).toBe('amazon.nova-canvas-v1:0'); + }); +}); diff --git a/packages/amazon-bedrock/src/bedrock-image-model.ts b/packages/amazon-bedrock/src/bedrock-image-model.ts new file mode 100644 index 000000000000..cc57e42050eb --- /dev/null +++ b/packages/amazon-bedrock/src/bedrock-image-model.ts @@ -0,0 +1,130 @@ +import { ImageModelV1, ImageModelV1CallWarning } from '@ai-sdk/provider'; +import { + FetchFunction, + Resolvable, + combineHeaders, + createJsonErrorResponseHandler, + createJsonResponseHandler, + postJsonToApi, + resolve, +} from '@ai-sdk/provider-utils'; +import { + BedrockImageModelId, + BedrockImageSettings, + modelMaxImagesPerCall, +} from './bedrock-image-settings'; +import { BedrockErrorSchema } from './bedrock-error'; +import { z } from 'zod'; + +type BedrockImageModelConfig = { + baseUrl: () => string; + headers: Resolvable>; + fetch?: FetchFunction; + _internal?: { + currentDate?: () => Date; + }; +}; + +export class BedrockImageModel implements ImageModelV1 { + readonly specificationVersion = 'v1'; + readonly provider = 'amazon-bedrock'; + + get maxImagesPerCall(): number { + return ( + this.settings.maxImagesPerCall ?? modelMaxImagesPerCall[this.modelId] ?? 1 + ); + } + + private getUrl(modelId: string): string { + const encodedModelId = encodeURIComponent(modelId); + return `${this.config.baseUrl()}/model/${encodedModelId}/invoke`; + } + + constructor( + readonly modelId: BedrockImageModelId, + private readonly settings: BedrockImageSettings, + private readonly config: BedrockImageModelConfig, + ) {} + + async doGenerate({ + prompt, + n, + size, + aspectRatio, + seed, + providerOptions, + headers, + abortSignal, + }: Parameters[0]): Promise< + Awaited> + > { + const warnings: Array = []; + const [width, height] = size ? size.split('x').map(Number) : []; + const args = { + taskType: 'TEXT_IMAGE', + textToImageParams: { + text: prompt, + ...(providerOptions?.bedrock?.negativeText + ? { + negativeText: providerOptions.bedrock.negativeText, + } + : {}), + }, + imageGenerationConfig: { + ...(width ? { width } : {}), + ...(height ? { height } : {}), + ...(seed ? { seed } : {}), + ...(n ? { numberOfImages: n } : {}), + ...(providerOptions?.bedrock?.quality + ? { quality: providerOptions.bedrock.quality } + : {}), + ...(providerOptions?.bedrock?.cfgScale + ? { cfgScale: providerOptions.bedrock.cfgScale } + : {}), + }, + }; + + if (aspectRatio != undefined) { + warnings.push({ + type: 'unsupported-setting', + setting: 'aspectRatio', + details: + 'This model does not support aspect ratio. Use `size` instead.', + }); + } + + const currentDate = this.config._internal?.currentDate?.() ?? new Date(); + const { value: response, responseHeaders } = await postJsonToApi({ + url: this.getUrl(this.modelId), + headers: await resolve( + combineHeaders(await resolve(this.config.headers), headers), + ), + body: args, + failedResponseHandler: createJsonErrorResponseHandler({ + errorSchema: BedrockErrorSchema, + errorToMessage: error => `${error.type}: ${error.message}`, + }), + successfulResponseHandler: createJsonResponseHandler( + bedrockImageResponseSchema, + ), + abortSignal, + fetch: this.config.fetch, + }); + + return { + images: response.images, + warnings, + response: { + timestamp: currentDate, + modelId: this.modelId, + headers: responseHeaders, + }, + }; + } +} + +// minimal version of the schema, focussed on what is needed for the implementation +// this approach limits breakages when the API changes and increases efficiency +const bedrockImageResponseSchema = z.object({ + images: z.array(z.string()), +}); diff --git a/packages/amazon-bedrock/src/bedrock-image-settings.ts b/packages/amazon-bedrock/src/bedrock-image-settings.ts new file mode 100644 index 000000000000..6caf11c27ab9 --- /dev/null +++ b/packages/amazon-bedrock/src/bedrock-image-settings.ts @@ -0,0 +1,14 @@ +export type BedrockImageModelId = 'amazon.nova-canvas-v1:0' | (string & {}); + +// https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html +export const modelMaxImagesPerCall: Record = { + 'amazon.nova-canvas-v1:0': 5, +}; + +export interface BedrockImageSettings { + /** + * Override the maximum number of images per call (default is dependent on the + * model, or 1 for an unknown model). + */ + maxImagesPerCall?: number; +} diff --git a/packages/amazon-bedrock/src/bedrock-provider.test.ts b/packages/amazon-bedrock/src/bedrock-provider.test.ts index da357c1174ae..ccaff5d98a15 100644 --- a/packages/amazon-bedrock/src/bedrock-provider.test.ts +++ b/packages/amazon-bedrock/src/bedrock-provider.test.ts @@ -2,12 +2,13 @@ import { describe, it, expect, vi, beforeEach, Mock } from 'vitest'; import { createAmazonBedrock } from './bedrock-provider'; import { BedrockChatLanguageModel } from './bedrock-chat-language-model'; import { BedrockEmbeddingModel } from './bedrock-embedding-model'; -import { loadSetting } from '@ai-sdk/provider-utils'; +import { BedrockImageModel } from './bedrock-image-model'; // Add type assertions for the mocked classes const BedrockChatLanguageModelMock = BedrockChatLanguageModel as unknown as Mock; const BedrockEmbeddingModelMock = BedrockEmbeddingModel as unknown as Mock; +const BedrockImageModelMock = BedrockImageModel as unknown as Mock; vi.mock('./bedrock-chat-language-model', () => ({ BedrockChatLanguageModel: vi.fn(), @@ -17,6 +18,10 @@ vi.mock('./bedrock-embedding-model', () => ({ BedrockEmbeddingModel: vi.fn(), })); +vi.mock('./bedrock-image-model', () => ({ + BedrockImageModel: vi.fn(), +})); + vi.mock('@ai-sdk/provider-utils', () => ({ loadSetting: vi.fn().mockImplementation(({ settingValue }) => 'us-east-1'), withoutTrailingSlash: vi.fn(url => url), @@ -121,5 +126,33 @@ describe('AmazonBedrockProvider', () => { expect(constructorCall[1]).toEqual({ dimensions: 1024, normalize: true }); expect(model).toBeInstanceOf(BedrockEmbeddingModel); }); + + it('should create an image model', () => { + const provider = createAmazonBedrock(); + const modelId = 'amazon.titan-image-generator'; + + const model = provider.image(modelId, { + maxImagesPerCall: 5, + }); + + const constructorCall = BedrockImageModelMock.mock.calls[0]; + expect(constructorCall[0]).toBe(modelId); + expect(constructorCall[1]).toEqual({ maxImagesPerCall: 5 }); + expect(model).toBeInstanceOf(BedrockImageModel); + }); + + it('should create an image model via imageModel method', () => { + const provider = createAmazonBedrock(); + const modelId = 'amazon.titan-image-generator'; + + const model = provider.imageModel(modelId, { + maxImagesPerCall: 5, + }); + + const constructorCall = BedrockImageModelMock.mock.calls[0]; + expect(constructorCall[0]).toBe(modelId); + expect(constructorCall[1]).toEqual({ maxImagesPerCall: 5 }); + expect(model).toBeInstanceOf(BedrockImageModel); + }); }); }); diff --git a/packages/amazon-bedrock/src/bedrock-provider.ts b/packages/amazon-bedrock/src/bedrock-provider.ts index 68bc7217166a..3a3964ab8294 100644 --- a/packages/amazon-bedrock/src/bedrock-provider.ts +++ b/packages/amazon-bedrock/src/bedrock-provider.ts @@ -1,5 +1,6 @@ import { EmbeddingModelV1, + ImageModelV1, LanguageModelV1, ProviderV1, } from '@ai-sdk/provider'; @@ -20,6 +21,11 @@ import { BedrockEmbeddingModelId, BedrockEmbeddingSettings, } from './bedrock-embedding-settings'; +import { BedrockImageModel } from './bedrock-image-model'; +import { + BedrockImageModelId, + BedrockImageSettings, +} from './bedrock-image-settings'; import { createSigV4FetchFunction } from './bedrock-sigv4-fetch'; export interface AmazonBedrockProviderSettings { @@ -81,6 +87,16 @@ export interface AmazonBedrockProvider extends ProviderV1 { modelId: BedrockEmbeddingModelId, settings?: BedrockEmbeddingSettings, ): EmbeddingModelV1; + + image( + modelId: BedrockImageModelId, + settings?: BedrockImageSettings, + ): ImageModelV1; + + imageModel( + modelId: BedrockImageModelId, + settings?: BedrockImageSettings, + ): ImageModelV1; } /** @@ -162,10 +178,22 @@ export function createAmazonBedrock( fetch: sigv4Fetch, }); + const createImageModel = ( + modelId: BedrockImageModelId, + settings: BedrockImageSettings = {}, + ) => + new BedrockImageModel(modelId, settings, { + baseUrl: getBaseUrl, + headers: options.headers ?? {}, + fetch: sigv4Fetch, + }); + provider.languageModel = createChatModel; provider.embedding = createEmbeddingModel; provider.textEmbedding = createEmbeddingModel; provider.textEmbeddingModel = createEmbeddingModel; + provider.image = createImageModel; + provider.imageModel = createImageModel; return provider; }