diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..969ac3d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,33 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[Bug] " +labels: bug +assignees: '' + +--- + +**Is this a new bug?** +In other words: Is this an error, flaw, failure or fault? Please search Github issues and check our [Community Forum](https://community.pinecone.io/) to see if someone has already reported the bug you encountered. + +If this is a request for help or troubleshooting code in your own Pinecone project, please join the [Pinecone Community Forum](https://community.pinecone.io/). + +- [ ] I believe this is a new bug +- [ ] I have searched the existing Github issues and Community Forum, and I could not find an existing post for this bug + +**Describe the bug** +Describe the functionality that was working before but is broken now. + +**Error information** +If you have one, please include the full stack trace here. If not, please share as much as you can about the error. + +**Steps to reproduce the issue locally** +Include steps to reproduce the issue here. If you have sample code or a script that can be used to replicate this issue, please include that as well (including any dependent files to run the code). + +**Environment** +* OS Version: +* Go version: +* Go SDK version: + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000..77f5d9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,16 @@ +--- +name: Documentation +about: Report an issue in our docs +title: "[Docs] " +labels: 'documentation' +assignees: '' + +--- + +**Description** +Describe the issue that you've encountered with our documentation. + +**Suggested solution** +Describe how this issue could be fixed or improved. +**Link to page** +Add a link to the exact documentation page where the issue occurred. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..693231f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[Feature Request]" +labels: 'enhancement' +assignees: '' + +--- + +**What motivated you to submit this feature request?** +A clear and concise description of why you are requesting this feature - e.g. "Being able to do x would allow me to..." + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4f80531 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# Contributing + +## Prereqs + +1. A [current version of Go](https://go.dev/doc/install) (recommended 1.21+) +2. The [just](https://github.com/casey/just?tab=readme-ov-file#installation) command runner +3. The [protobuf-compiler](https://grpc.io/docs/protoc-installation/) + +Then, execute `just bootstrap` to install the necessary Go packages + +## .env Setup + +An easy way to keep track of necessary environment variables is to create a `.env` file in the root of the project. +This project comes with a sample `.env` file (`.env.sample`) that you can copy and modify. At the very least, you +will need to include the `PINECONE_API_KEY` variable in your `.env` file for the tests to run locally. + +````shell +### API Definitions submodule + +The API Definitions are in a private submodule. To checkout or update the submodules, execute the following command in the root of the project: + +```shell +git submodule update --init --recursive +```` + +For working with submodules, see the [Git Submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) +documentation. + +## Just commands + +`just test` : Executes all tests (unit & integration) for the pinecone package + +`jest test-unit` : Executes unit tests only for the pinecone package + +`just gen` : Generates Go client code from the API definitions + +`just docs` : Generates Go docs and starts http server on localhost + +`just bootstrap` : Installs necessary go packages for gen and docs diff --git a/README.md b/README.md index 4a928d9..758be5f 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,10 @@ -> **Warning** -> -> **Under active development** This SDK is pre-1.0 and should be considered unstable. Before a 1.0 release, there are -> no guarantees of backward compatibility between minor versions. +# Pinecone Go Client · ![License](https://img.shields.io/github/license/pinecone-io/go-pinecone?color=orange) [![Go Reference](https://pkg.go.dev/badge/github.com/pinecone-io/go-pinecone.svg)](https://pkg.go.dev/github.com/pinecone-io/go-pinecone@main/pinecone) [![Go Report Card](https://goreportcard.com/badge/github.com/pinecone-io/go-pinecone)](https://goreportcard.com/report/github.com/pinecone-io/go-pinecone) -# go-pinecone - -[![Go Reference](https://pkg.go.dev/badge/github.com/pinecone-io/go-pinecone.svg)](https://pkg.go.dev/github.com/pinecone-io/go-pinecone@main/pinecone) - -Official Pinecone Go Client +This is the official Go client for [Pinecone](https://www.pinecone.io). ## Documentation -To see the latest documentation on `main`, visit https://pkg.go.dev/github.com/pinecone-io/go-pinecone@main/pinecone. +To see the latest documentation for `main`, visit https://pkg.go.dev/github.com/pinecone-io/go-pinecone@main/pinecone. To see the latest versioned-release's documentation, visit https://pkg.go.dev/github.com/pinecone-io/go-pinecone/pinecone. @@ -20,23 +13,162 @@ visit https://pkg.go.dev/github.com/pinecone-io/go-pinecone/pinecone. go-pinecone contains -- gRPC bindings for Data Plane operations on Vectors -- REST bindings for Control Plane operations on Indexes and Collections +- gRPC bindings for [Data Plane](https://docs.pinecone.io/reference/api/2024-07/data-plane) operations +- REST bindings for [Control Plane](https://docs.pinecone.io/reference/api/2024-07/control-plane) + operations -See [Pinecone API Docs](https://docs.pinecone.io/reference/) for more info. +See the [Pinecone API Docs](https://docs.pinecone.io/reference/) for more information. -## Installation +## Upgrading your client + +To upgrade your client to the latest version, run: + +```shell +go get -u github.com/pinecone-io/go-pinecone/pinecone@latest +``` + +## Prerequisites -go-pinecone requires a Go version with [modules](https://go.dev/wiki/Modules) support. +`go-pinecone` requires a Go version with [modules](https://go.dev/wiki/Modules) support. -To add a dependency on go-pinecone: +## Installation + +To install the Pinecone Go client, run the following in your terminal: ```shell go get github.com/pinecone-io/go-pinecone/pinecone ``` +For more information on setting up a Go project, see the [Go documentation](https://golang.org/doc/). + ## Usage +### Initializing the client + +**Authenticating via an API key** + +When initializing the client with a Pinecone API key, you must construct a `NewClientParams` object and pass it to the +`NewClient` function. + +It's recommended that you set your Pinecone API key as an environment variable (`"PINECONE_API_KEY"`) and access it that +way. Alternatively, you can pass it in your code directly. + +```go +package main + +import ( + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } +} +``` + +**Authenticating via custom headers** + +If you choose to authenticate via custom headers (e.g. for OAuth), you must construct a `NewClientBaseParams` object +and pass it to `NewClientBase`. + +Note: you must include the `"X-Project-Id"` header with your Pinecone project ID +when authenticating via custom headers. + +```go +package main + +import ( + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" +) + +func main() { + clientParams := pinecone.NewClientBaseParams{ + Headers: map[string]string{ + "Authorization": "Bearer " + "" + "X-Project-Id": "" + }, + } + + pc, err := pinecone.NewClientBase(clientParams) + + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } +} +``` + +## Indexes + +### Create indexes + +**Create a serverless index** + +The following example creates a serverless index in the `us-east-1` +region of AWS. For more information on serverless and regional availability, +see [Understanding indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes). + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.CreateServerlessIndex(ctx, &pinecone.CreateServerlessIndexRequest{ + Name: "my-serverless-index", + Dimension: 3, + Metric: pinecone.Cosine, + Cloud: pinecone.Aws, + Region: "us-east-1", + }) + + if err != nil { + log.Fatalf("Failed to create serverless index: %s", idx.Name) + } else { + fmt.Printf("Successfully created serverless index: %s", idx.Name) + } +} +``` + +**Create a pod-based index** + +The following example creates a pod-based index with a metadata configuration. If no metadata configuration is +provided, all metadata fields are automatically indexed. + ```go package main @@ -44,86 +176,1095 @@ import ( "context" "fmt" "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" ) func main() { ctx := context.Background() - pc, err := pinecone.NewClient(pinecone.NewClientParams{ - ApiKey: "api-key", + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + podIndexMetadata := &pinecone.PodSpecMetadataConfig{ + Indexed: &[]string{"title", "description"}, + } + + idx, err := pc.CreatePodIndex(ctx, &pinecone.CreatePodIndexRequest{ + Name: "my-pod-index", + Dimension: 3, + Metric: pinecone.Cosine, + Environment: "us-west1-gcp", + PodType: "s1", + MetadataConfig: podIndexMetadata, }) if err != nil { - fmt.Println("Error:", err) - return + log.Fatalf("Failed to create pod index: %v", err) + } else { + fmt.Printf("Successfully created pod index: %s", idx.Name) + } + +} +``` + +### List indexes + +The following example lists all indexes in your Pinecone project. + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") } idxs, err := pc.ListIndexes(ctx) if err != nil { - fmt.Println("Error:", err) - return + log.Fatalf("Failed to list indexes: %v", err) + } else { + fmt.Println("Your project has the following indexes:") + for _, idx := range idxs { + fmt.Printf("- \"%s\"\n", idx.Name) + } + } +} +``` + +### Describe an index + +The following example describes an index by name. + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + indexName := "the-name-of-my-index" + + idx, err := pc.DescribeIndex(ctx, indexName) + if err != nil { + log.Fatalf("Failed to describe index: %s", err) + } else { + fmt.Printf("%+v", *idx) + } +} +``` + +### Delete an index + +The following example deletes an index by name. Note: only indexes not protected by deletion protection +may be deleted. + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + indexName := "the-name-of-my-index" + + err = pc.DeleteIndex(ctx, indexName) + if err != nil { + log.Fatalf("Error: %v", err) + } else { + fmt.Printf("Index \"%s\" deleted successfully", indexName) + } +} +``` + +### Configure an index + +There are multiple ways to configure Pinecone indexes. You are able to configure Deletion Protection for both +pod-based and Serverless indexes. Additionally, you can configure the size of your pods and the number of replicas +for pod-based indexes. Examples for each of these configurations are provided below. + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), } - for _, index := range idxs { - fmt.Println(index) + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") } - idx, err := pc.Index(idxs[0].Host) - defer idx.Close() + // To scale the size of your pods-based index from "x2" to "x4": + _, err := pc.ConfigureIndex(ctx, "my-pod-index", &ConfigureIndexParams{PodType: "p1.x4"}) + if err != nil { + fmt.Printf("Failed to configure index: %v\n", err) + } + // To scale the number of replicas to 4: + _, err := pc.ConfigureIndex(ctx, "my-pod-index", &ConfigureIndexParams{Replicas: 4}) if err != nil { - fmt.Println("Error:", err) - return + fmt.Printf("Failed to configure index: %v\n", err) } - res, err := idx.DescribeIndexStats(ctx) + // To scale both the size of your pods and the number of replicas: + _, err := pc.ConfigureIndex(ctx, "my-pod-index", &ConfigureIndexParams{PodType: "p1.x4", Replicas: 4}) if err != nil { - fmt.Println("Error:", err) - return + fmt.Printf("Failed to configure index: %v\n", err) } - fmt.Println(res) + // To enable deletion protection: + _, err := pc.ConfigureIndex(ctx, "my-index", &ConfigureIndexParams{DeletionProtection: "enabled"}) + if err != nil { + fmt.Printf("Failed to configure index: %v\n", err) + } } ``` -## Support +### Describe index statistics -To get help using go-pinecone, reach out to support@pinecone.io. +The following examlpe describes the statistics of an index by name. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() -## Development + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } -### Prereqs + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } -1. A [current version of Go](https://go.dev/doc/install) (recommended 1.21+) -2. The [just](https://github.com/casey/just?tab=readme-ov-file#installation) command runner -3. The [protobuf-compiler](https://grpc.io/docs/protoc-installation/) + indexName := "the-name-of-my-index" -Then, execute `just bootstrap` to install the necessary Go packages + idx, err := pc.DescribeIndex(ctx, indexName) + if err != nil { + log.Fatalf("Failed to describe index \"%s\". Error:%s", idx.Name, err) + } else { + fmt.Printf("Successfully found the \"%s\" index!\n", idx.Name) + } +} +``` -### .env Setup +## Index Operations -An easy way to keep track of necessary environment variables is to create a `.env` file in the root of the project. -This project comes with a sample `.env` file (`.env.sample`) that you can copy and modify. At the very least, you -will need to include the `PINECONE_API_KEY` variable in your `.env` file for the tests to run locally. +Pinecone indexes support working with vector data using operations such as upsert, query, fetch, and delete. -```shell -### API Definitions submodule +### Targeting an index -The API Definitions are in a private submodule. To checkout or update the submodules execute in the root of the project: +To perform data operations on an index, you target it using the `Index` method on a `Client` object. You will +need your index's `Host` value, which you can retrieve via `DescribeIndex` or `ListIndexes`. -```shell -git submodule update --init --recursive +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "pinecone-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v: %v", idx.Host, err) + } +} ``` -For working with submodules, see the [Git Submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) -documentation. +### Upsert vectors + +The following example upserts vectors to `example-index`. + +```go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } -### Just commands + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } -`just test` : Executes all tests for the pinecone package + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v: %v", idx.Host, err) + } -`just gen` : Generates Go client code from the API definitions + vectors := []*pinecone.Vector{ + { + Id: "A", + Values: []float32{0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1}, + }, + { + Id: "B", + Values: []float32{0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2}, + }, + { + Id: "C", + Values: []float32{0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3}, + }, + { + Id: "D", + Values: []float32{0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4}, + }, + } -`just docs` : Generates Go docs and starts http server on localhost + count, err := idxConnection.UpsertVectors(ctx, vectors) + if err != nil { + log.Fatalf("Failed to upsert vectors: %v", err) + } else { + fmt.Printf("Successfully upserted %d vector(s)", count) + } +} +``` -`just bootstrap` : Installs necessary go packages for gen and docs +### Query an index + +#### Query by vector values + +The following example queries the index `example-index` with vector values and metadata filtering. + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func prettifyStruct(obj interface{}) string { + bytes, _ := json.MarshalIndent(obj, "", " ") + return string(bytes) +} + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host %v: %v", idx.Host, err) + } + + queryVector := []float32{0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3} + + metadataFilter, err := structpb.NewStruct(map[string]interface{}{ + "genre": {"$eq": "documentary"}, + "year": 2019, + }) + if err != nil { + log.Fatalf("Failed to create metadataFilter: %v", err) + } + + res, err := idxConnection.QueryByVectorValues(ctx, &pinecone.QueryByVectorValuesRequest{ + Vector: queryVector, + TopK: 3, + Filter: metadataFilter, + IncludeValues: true, + }) + if err != nil { + log.Fatalf("Error encountered when querying by vector: %v", err) + } else { + fmt.Printf(prettifyStruct(res)) + } +} + +// Returns: +// { +// "matches": [ +// { +// "vector": { +// "id": "B", +// "values": [ +// 0.2, +// 0.2, +// 0.2, +// 0.2, +// 0.2, +// 0.2, +// 0.2, +// 0.2 +// ] +// }, +// "score": 1 +// }, +// { +// "vector": { +// "id": "C", +// "values": [ +// 0.3, +// 0.3, +// 0.3, +// 0.3, +// 0.3, +// 0.3, +// 0.3, +// 0.3 +// ] +// }, +// "score": 1 +// }, +// { +// "vector": { +// "id": "A", +// "values": [ +// 0.1, +// 0.1, +// 0.1, +// 0.1, +// 0.1, +// 0.1, +// 0.1, +// 0.1 +// ] +// }, +// "score": 1 +// } +// ], +// "usage": { +// "read_units": 6 +// } +// } +``` + +#### Query by vector id + +The following example queries the index `example-index` with a vector id value. + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func prettifyStruct(obj interface{}) string { + bytes, _ := json.MarshalIndent(obj, "", " ") + return string(bytes) +} + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host %v: %v", idx.Host, err) + } + + vectorId := "vector-id" + res, err := idxConnection.QueryByVectorId(ctx, &pinecone. & pinecone.QueryByVectorIdRequest{ + VectorId: vectorId, + TopK: 3, + IncludeValues: true, + }) + if err != nil { + log.Fatalf("Error encountered when querying by vector ID `%v`: %v", vectorId, err) + } else { + fmt.Printf(prettifyStruct(res)) + } +} +``` + +### Delete vectors + +#### Delete vectors by ID + +The following example deletes a vector by its ID value from `example-index` and `example-namespace`. You can pass a +slice of vector IDs to `DeleteVectorsById`. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%s\". Error:%s", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) + } + + vectorId := "your-vector-id" + err = idxConnection.DeleteVectorsById(ctx, []string{vectorId}) + + if err != nil { + log.Fatalf("Failed to delete vector with ID: %s. Error: %s\n", vectorId, err) + } +} +``` + +#### Delete vectors by filter + +The following example deletes vectors from `example-index` using a metadata filter. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%s\". Error:%s", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) + } + + filter, err := structpb.NewStruct(map[string]interface{}{ + "genre": "classical", + }) + if err != nil { + log.Fatalf("Failed to create metadata filter. Error: %v", err) + } + + err = idxConnection.DeleteVectorsByFilter(ctx, filter) + + if err != nil { + log.Fatalf("Failed to delete vector(s) with filter: %+v. Error: %s\n", filter, err) + } +} +``` + +#### Delete all vectors in a namespace + +The following example deletes all vectors from `example-index` and `example-namespace`. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%s\". Error:%s", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) + } + + // deletes all vectors in "example-namespace" + err = idxConnection.DeleteAllVectorsInNamespace(ctx) + if err != nil { + log.Fatalf("Failed to delete vectors in namespace: \"%s\". Error: %s", idxConnection.Namespace, err) + } +} +``` + +### Fetch vectors + +The following example fetches vectors by ID from `example-index` and `example-namespace`. + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func prettifyStruct(obj interface{}) string { + bytes, _ := json.MarshalIndent(obj, "", " ") + return string(bytes) +} + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host %v: %v", idx.Host, err) + } + + res, err := idxConnection.FetchVectors(ctx, []string{"id-1", "id-2"}) + if err != nil { + log.Fatalf("Failed to fetch vectors: %v", err) + } else { + fmt.Printf(prettifyStruct(res)) + } +} + +// Response: +// { +// "vectors": { +// "id-1": { +// "id": "id-1", +// "values": [ +// -0.0089730695, +// -0.020010853, +// -0.0042787646, +// ... +// ] +// }, +// "id-2": { +// "id": "id-2", +// "values": [ +// -0.005380766, +// 0.00215196, +// -0.014833462, +// ... +// ] +// } +// }, +// "usage": { +// "read_units": 1 +// } +// } +``` + +### Update vectors + +The following example updates vectors by ID in `example-index` and `example-namespace`. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "pinecone-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "ns1"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host %v: %v", idx.Host, err) + } + + id := "id-3" + + err = idxConnection.UpdateVector(ctx, &pinecone.UpdateVectorRequest{ + Id: id, + Values: []float32{4.0, 2.0}, + }) + if err != nil { + log.Fatalf("Failed to update vector with ID %v: %v", id, err) + } +} +``` + +### List vectors + +The `ListVectors` method can be used to list vector ids matching a particular id prefix. +With clever assignment of vector ids, you can model hierarchical relationships across embeddings within the same +document. + +The following example lists all vector ids in `example-index` and `example-namespace`, with the prefix `doc1`. + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func prettifyStruct(obj interface{}) string { + bytes, _ := json.MarshalIndent(obj, "", " ") + return string(bytes) +} + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + idx, err := pc.DescribeIndex(ctx, "example-index") + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "example-namespace"}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host %v: %v", idx.Host, err) + } + + limit := uint32(3) + prefix := "doc1" + + res, err := idxConnection.ListVectors(ctx, &pinecone.ListVectorsRequest{ + Limit: &limit, + Prefix: &prefix, + }) + if len(res.VectorIds) == 0 { + fmt.Println("No vectors found") + } else { + fmt.Printf(prettifyStruct(res)) + } +} + +// Response: +// { +// "vector_ids": [ +// "doc1#chunk1", +// "doc1#chunk2", +// "doc1#chunk3" +// ], +// "usage": { +// "read_units": 1 +// }, +// "next_pagination_token": "eyJza2lwX3Bhc3QiOiIwMDBkMTc4OC0zMDAxLTQwZmMtYjZjNC0wOWI2N2I5N2JjNDUiLCJwcmVmaXgiOm51bGx9" +// } +``` + +## Collections + +[A collection is a static copy of an index](https://docs.pinecone.io/guides/indexes/understanding-collections). +Collections are only available for pod-based indexes. + +### Create a collection + +The following example creates a collection from a source index. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + collection, err := pc.CreateCollection(ctx, &pinecone.CreateCollectionRequest{ + Name: "my-collection", + Source: "my-source-index", + }) + if err != nil { + log.Fatalf("Failed to create collection: %v", err) + } else { + fmt.Printf("Successfully created collection \"%s\".", collection.Name) + } +} +``` + +### List collections + +The following example lists all collections in your Pinecone project. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + collections, err := pc.ListCollections(ctx) + if err != nil { + log.Fatalf("Failed to list collections: %v", err) + } else { + if len(collections) == 0 { + fmt.Printf("No collections found in project") + } else { + fmt.Println("Collections in project:") + for _, collection := range collections { + fmt.Printf("- %s\n", collection.Name) + } + } + } +} +``` + +### Describe a collection + +The following example describes a collection by name. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + collection, err := pc.DescribeCollection(ctx, "my-collection") + if err != nil { + log.Fatalf("Error describing collection: %v", err) + } else { + fmt.Printf("Collection: %+v\n", *collection) + } +} +``` + +### Delete a collection + +The following example deletes a collection by name. + +```Go +package main + +import ( + "context" + "fmt" + "github.com/pinecone-io/go-pinecone/pinecone" + "log" + "os" +) + +func main() { + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + collectionName := "my-collection" + + err = pc.DeleteCollection(ctx, collectionName) + if err != nil { + log.Fatalf("Failed to create collection: %s\n", err) + } else { + log.Printf("Successfully deleted collection \"%s\"\n", collectionName) + } +} +``` + +## Support + +To get help using go-pinecone, reach out to support@pinecone.io. diff --git a/pinecone/client.go b/pinecone/client.go index 5788378..15bfabb 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -194,7 +194,7 @@ func NewClient(in NewClientParams) (*Client, error) { // // clientParams := pinecone.NewClientBaseParams{ // Headers: map[string]string{ -// "Authorization": "Bearer " + "" +// "Authorization": "Bearer " + "" // "X-Project-Id": "" // }, // SourceTag: "your_source_identifier", // optional @@ -838,14 +838,14 @@ type ConfigureIndexParams struct { // fmt.Printf("Failed to configure index: %v\n", err) // } // -// // To scale both the size of your pods and the number of replicas: +// // To scale both the size of your pods and the number of replicas to 4: // _, err := pc.ConfigureIndex(ctx, "my-pod-index", &ConfigureIndexParams{PodType: "p1.x4", Replicas: 4}) // if err != nil { // fmt.Printf("Failed to configure index: %v\n", err) // } // // // To enable deletion protection: -// _, err := pc.ConfigureIndex(ctx, "my-index", nil, nil, &ConfigureIndexParams{DeletionProtection: "enabled"}) +// _, err := pc.ConfigureIndex(ctx, "my-index", &ConfigureIndexParams{DeletionProtection: "enabled"}) // if err != nil { // fmt.Printf("Failed to configure index: %v\n", err) // } diff --git a/pinecone/index_connection.go b/pinecone/index_connection.go index 13f8d36..b7888e8 100644 --- a/pinecone/index_connection.go +++ b/pinecone/index_connection.go @@ -400,7 +400,7 @@ func (idx *IndexConnection) ListVectors(ctx context.Context, in *ListVectorsRequ type QueryByVectorValuesRequest struct { Vector []float32 TopK uint32 - Filter *MetadataFilter + MetadataFilter *MetadataFilter IncludeValues bool IncludeMetadata bool SparseValues *SparseValues @@ -465,7 +465,7 @@ type QueryVectorsResponse struct { // "genre": "classical", // } // -// metadataFilter, err := structpb.NewStruct(metadataMap) +// MetadataFilter, err := structpb.NewStruct(metadataMap) // // if err != nil { // log.Fatalf("Failed to create metadata map. Error: %v", err) @@ -479,7 +479,7 @@ type QueryVectorsResponse struct { // res, err := idxConnection.QueryByVectorValues(ctx, &pinecone.QueryByVectorValuesRequest{ // Vector: queryVector, // TopK: topK, // number of vectors to be returned -// MetadataFilter: metadataFilter, +// MetadataFilter: MetadataFilter, // SparseValues: &sparseValues, // IncludeValues: true, // IncludeMetadata: true, @@ -496,7 +496,7 @@ func (idx *IndexConnection) QueryByVectorValues(ctx context.Context, in *QueryBy req := &data.QueryRequest{ Namespace: idx.Namespace, TopK: in.TopK, - Filter: in.Filter, + Filter: in.MetadataFilter, IncludeValues: in.IncludeValues, IncludeMetadata: in.IncludeMetadata, Vector: in.Vector, @@ -519,7 +519,7 @@ func (idx *IndexConnection) QueryByVectorValues(ctx context.Context, in *QueryBy type QueryByVectorIdRequest struct { VectorId string TopK uint32 - Filter *MetadataFilter + metadataFilter *MetadataFilter IncludeValues bool IncludeMetadata bool SparseValues *SparseValues @@ -530,9 +530,8 @@ type QueryByVectorIdRequest struct { // // Returns a pointer to a QueryVectorsResponse object or an error if the request fails. // -// Note: QueryByVectorId executes a nearest neighbors search, -// meaning that unless TopK=1 in the QueryByVectorIdRequest object, -// it will return 2+ vectors. The vector with a score of 1.0 is the vector with the same ID as the query vector. +// Note: QueryByVectorId executes a nearest neighbors search, meaning that unless TopK=1 in the QueryByVectorIdRequest +// object, it will return 2+ vectors. The vector with a score of 1.0 is the vector with the same ID as the query vector. // // Parameters: // - ctx: A context.Context object controls the request's lifetime, @@ -588,7 +587,7 @@ func (idx *IndexConnection) QueryByVectorId(ctx context.Context, in *QueryByVect Id: in.VectorId, Namespace: idx.Namespace, TopK: in.TopK, - Filter: in.Filter, + Filter: in.metadataFilter, IncludeValues: in.IncludeValues, IncludeMetadata: in.IncludeMetadata, SparseVector: sparseValToGrpc(in.SparseValues), @@ -603,8 +602,8 @@ func (idx *IndexConnection) QueryByVectorId(ctx context.Context, in *QueryByVect // otherwise returns nil. This method will also return nil if the passed vector ID does not exist in the index or // namespace. // -// Note: You must instantiate an IndexWithNamespace connection in order to delete vectors by ID in namespaces other -// than the default. +// Note: You must instantiate an Index connection with a Namespace in NewIndexConnParams in order to delete vectors +// in a namespace other than the default: "". // // Parameters: // - ctx: A context.Context object controls the request's lifetime, @@ -658,13 +657,13 @@ func (idx *IndexConnection) DeleteVectorsById(ctx context.Context, ids []string) // Returns an error if the request fails, otherwise returns nil. // // Note: DeleteVectorsByFilter is only available on pods-based indexes. -// Additionally, you must instantiate an IndexWithNamespace connection in order to delete vectors in namespaces -// other than the default. +// Additionally, you must instantiate an IndexConnection using the Index method with a Namespace in NewIndexConnParams +// in order to delete vectors in a namespace other than the default. // // Parameters: // - ctx: A context.Context object controls the request's lifetime, // allowing for the request to be canceled or to timeout according to the context's deadline. -// - filter: The filter to apply to the deletion. +// - MetadataFilter: The filter to apply to the deletion. // // Example: // @@ -693,11 +692,11 @@ func (idx *IndexConnection) DeleteVectorsById(ctx context.Context, ids []string) // log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) // } // -// metadataFilter := map[string]interface{}{ +// MetadataFilter := map[string]interface{}{ // "genre": "classical", // } // -// filter, err := structpb.NewStruct(metadataFilter) +// filter, err := structpb.NewStruct(MetadataFilter) // // if err != nil { // log.Fatalf("Failed to create metadata filter. Error: %v", err) @@ -708,9 +707,9 @@ func (idx *IndexConnection) DeleteVectorsById(ctx context.Context, ids []string) // if err != nil { // log.Fatalf("Failed to delete vector(s) with filter: %+v. Error: %s\n", filter, err) // } -func (idx *IndexConnection) DeleteVectorsByFilter(ctx context.Context, filter *MetadataFilter) error { +func (idx *IndexConnection) DeleteVectorsByFilter(ctx context.Context, metadataFilter *MetadataFilter) error { req := data.DeleteRequest{ - Filter: filter, + Filter: metadataFilter, Namespace: idx.Namespace, } @@ -721,8 +720,8 @@ func (idx *IndexConnection) DeleteVectorsByFilter(ctx context.Context, filter *M // // Returns an error if the request fails, otherwise returns nil. // -// Note: You must instantiate an IndexWithNamespace connection in order to delete vectors by ID in namespaces other -// than the default. +// Note: You must instantiate an IndexConnection using the Index method with a Namespace in NewIndexConnParams +// in order to delete vectors in a namespace other than the default. // // Parameters: // - ctx: A context.Context object controls the request's lifetime, @@ -749,7 +748,7 @@ func (idx *IndexConnection) DeleteVectorsByFilter(ctx context.Context, filter *M // log.Fatalf("Failed to describe index \"%s\". Error:%s", idx.Name, err) // } // -// idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host}) +// idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host, Namespace: "your-namespace"}) // // if err != nil { // log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) @@ -913,7 +912,7 @@ func (idx *IndexConnection) DescribeIndexStats(ctx context.Context) (*DescribeIn // Parameters: // - ctx: A context.Context object controls the request's lifetime, // allowing for the request to be canceled or to timeout according to the context's deadline. -// - filter: The filter to apply to the request. +// - MetadataFilter: The filter to apply to the request. // // Example: // @@ -942,14 +941,14 @@ func (idx *IndexConnection) DescribeIndexStats(ctx context.Context) (*DescribeIn // log.Fatalf("Failed to create IndexConnection for Host: %v. Error: %v", idx.Host, err) // } // -// metadataFilter := map[string]interface{}{ +// MetadataFilter := map[string]interface{}{ // "genre": "classical", // } // -// filter, err := structpb.NewStruct(metadataFilter) +// filter, err := structpb.NewStruct(MetadataFilter) // // if err != nil { -// log.Fatalf("Failed to create filter %+v. Error: %s", metadataFilter, err) +// log.Fatalf("Failed to create filter %+v. Error: %s", MetadataFilter, err) // } // // res, err := idxConnection.DescribeIndexStatsFiltered(ctx, filter) @@ -961,9 +960,9 @@ func (idx *IndexConnection) DescribeIndexStats(ctx context.Context) (*DescribeIn // fmt.Printf("Namespace: \"%s\", has %d vector(s) that match the given filter\n", name, summary.VectorCount) // } // } -func (idx *IndexConnection) DescribeIndexStatsFiltered(ctx context.Context, filter *MetadataFilter) (*DescribeIndexStatsResponse, error) { +func (idx *IndexConnection) DescribeIndexStatsFiltered(ctx context.Context, metadataFilter *MetadataFilter) (*DescribeIndexStatsResponse, error) { req := &data.DescribeIndexStatsRequest{ - Filter: filter, + Filter: metadataFilter, } res, err := (*idx.dataClient).DescribeIndexStats(idx.akCtx(ctx), req) if err != nil {