From 9ada732d297247c2d4bfdbd74e6fa0116e383557 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 21 Oct 2024 13:09:14 -0700 Subject: [PATCH 1/4] Add rerank section --- README.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1fdf2a9..2153b95 100644 --- a/README.md +++ b/README.md @@ -1307,13 +1307,17 @@ func main() { ## Inference -The `Client` object has an `Inference` namespace which allows interacting with Pinecone's [Inference API](https://docs.pinecone.io/reference/api/2024-07/inference/generate-embeddings). The Inference API is a service that gives you access to embedding models hosted on Pinecone's infrastructure. Read more at [Understanding Pinecone Inference](https://docs.pinecone.io/guides/inference/understanding-inference). +The `Client` object has an `Inference` namespace which allows interacting with +Pinecone's [Inference API](https://docs.pinecone.io/reference/api/2024-07/inference/generate-embeddings). The Inference +API is a service that gives you access to embedding models hosted on Pinecone's infrastructure. Read more +at [Understanding Pinecone Inference](https://docs.pinecone.io/guides/inference/understanding-inference). **Notes:** Models currently supported: -- [multilingual-e5-large](https://docs.pinecone.io/guides/inference/understanding-inference#embedding-models) +- Embedding: [multilingual-e5-large](https://docs.pinecone.io/guides/inference/understanding-inference#embedding-models) +- Reranking: [bge-reranker-v2-m3](https://docs.pinecone.io/models/bge-reranker-v2-m3) ### Create Embeddings @@ -1368,11 +1372,67 @@ Send text to Pinecone's inference API to generate embeddings for documents and q } fmt.Printf("query embedding response: %+v", queryEmbeddingsResponse) - // << Send query to Pinecone to retrieve similar documents >> + // << Send query to Pinecone to retrieve similar documents >> +``` + +### Rerank documents + +Rerank documents in descending relevance-order against a query. + +**Note:** The `score` represents the absolute measure of relevance of a given query and passage pair. Normalized +between [0, 1], the `score` represents how closely relevant a specific item and query are, with scores closer to 1 +indicating higher relevance. + +```go + ctx := context.Background() + + pc, err := pinecone.NewClient(pinecone.NewClientParams{ + ApiKey: "YOUR-API-KEY" + }) + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } + + rerankModel := "bge-reranker-v2-m3" + query := "What are some good Turkey dishes for Thanksgiving?" + + documents := []pinecone.Document{ + {"title": "Turkey Sandwiches", "body": "Turkey is a classic meat to eat at American Thanksgiving."}, + {"title": "Lemon Turkey", "body": "A lemon brined Turkey with apple sausage stuffing is a classic Thanksgiving main course."}, + {"title": "Thanksgiving", "body": "My favorite Thanksgiving dish is pumpkin pie"}, + {"title": "Protein Sources", "body": "Turkey is a great source of protein."}, + } + + // Optional arguments + topN := 3 + returnDocuments := false + rankFields := []string{"body"} + modelParams := map[string]string{ + "truncate": "END", + } + + rerankRequest := pinecone.RerankRequest{ + Model: rerankModel, + Query: query, + Documents: documents, + TopN: &topN, + ReturnDocuments: &returnDocuments, + RankFields: &rankFields, + Parameters: &modelParams, + } + + rerankResponse, err := pc.Inference.Rerank(ctx, &rerankRequest) + + if err != nil { + log.Fatalf("Failed to rerank documents: %v", err) + } + + fmt.Printf("rerank response: %+v", rerankResponse) ``` ## Support -To get help using go-pinecone you can file an issue on [GitHub](https://github.com/pinecone-io/go-pinecone/issues), visit the [community forum](https://community.pinecone.io/), +To get help using go-pinecone you can file an issue on [GitHub](https://github.com/pinecone-io/go-pinecone/issues), +visit the [community forum](https://community.pinecone.io/), or reach out to support@pinecone.io. From 713abf00e9c1ee92591612644d5ee2d122a88b93 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 21 Oct 2024 13:22:03 -0700 Subject: [PATCH 2/4] Add import section to README --- README.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/README.md b/README.md index 2153b95..f5bb6c0 100644 --- a/README.md +++ b/README.md @@ -579,6 +579,74 @@ func main() { } ``` +### Import vectors from object storage + +You can now [import vectors en masse](https://docs.pinecone.io/guides/data/understanding-imports) from object +storage. `Import` is a long-running, asynchronous operation that imports large numbers of records into a Pinecone +serverless index. + +In order to import vectors from object storage, they must be stored in Parquet files and adhere to the necessary +[file format](https://docs.pinecone.io/guides/data/understanding-imports#parquet-file-format). Your object storage +must also adhere to the necessary [directory structure](https://docs.pinecone.io/guides/data/understanding-imports#directory-structure). + +The following example imports 10 vectors from an Amazon S3 bucket into a Pinecone serverless index: + +```go + ctx := context.Background() + + clientParams := pinecone.NewClientParams{ + ApiKey: os.Getenv("PINECONE_API_KEY"), + } + + pc, err := pinecone.NewClient(clientParams) + + if err != nil { + log.Fatalf("Failed to create Client: %v", err) + } else { + fmt.Println("Successfully created a new Client object!") + } + + indexName := "sample-index" + + idx, err := pc.CreateServerlessIndex(ctx, &pinecone.CreateServerlessIndexRequest{ + Name: indexName, + Dimension: 3, + Metric: pinecone.Cosine, + Cloud: pinecone.Aws, + Region: "us-east-1", + }) + + if err != nil { + log.Fatalf("Failed to create serverless index: %v", err) + } else { + fmt.Printf("Successfully created serverless index: %s", idx.Name) + } + + idx, err = pc.DescribeIndex(ctx, "pinecone-index") + + if err != nil { + log.Fatalf("Failed to describe index \"%v\": %v", idx.Name, err) + } + + idxConnection, err := pc.Index(pinecone.NewIndexConnParams{Host: idx.Host}) + if err != nil { + log.Fatalf("Failed to create IndexConnection for Host: %v: %v", idx.Host, err) + } + + storageURI := "s3://my-bucket/my-directory/" + + errorMode := "abort" // Will abort if error encountered; other option: "continue" + + importRes, err := idxConnection.StartImport(ctx, storageURI, nil, (*pinecone.ImportErrorMode)(&errorMode)) + + if err != nil { + log.Fatalf("Failed to start import: %v", err) + } + + fmt.Printf("import started with ID: %s", importRes.Id) +``` +You can [start, cancel, and check the status](https://docs.pinecone.io/guides/data/import-data) of all or one import operation(s). + ### Query an index #### Query by vector values From 6de6e38d124891733586048f5603c96470a2b00b Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 21 Oct 2024 13:24:21 -0700 Subject: [PATCH 3/4] Remove `10`; unnecessary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5bb6c0..8307192 100644 --- a/README.md +++ b/README.md @@ -589,7 +589,7 @@ In order to import vectors from object storage, they must be stored in Parquet f [file format](https://docs.pinecone.io/guides/data/understanding-imports#parquet-file-format). Your object storage must also adhere to the necessary [directory structure](https://docs.pinecone.io/guides/data/understanding-imports#directory-structure). -The following example imports 10 vectors from an Amazon S3 bucket into a Pinecone serverless index: +The following example imports vectors from an Amazon S3 bucket into a Pinecone serverless index: ```go ctx := context.Background() From 2b7d331ce1d6abf9682199db72cdbb9d5983c838 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 21 Oct 2024 16:50:08 -0700 Subject: [PATCH 4/4] Remove `else` stmts --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 8307192..68d8552 100644 --- a/README.md +++ b/README.md @@ -602,8 +602,6 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s if err != nil { log.Fatalf("Failed to create Client: %v", err) - } else { - fmt.Println("Successfully created a new Client object!") } indexName := "sample-index" @@ -618,8 +616,6 @@ The following example imports vectors from an Amazon S3 bucket into a Pinecone s if err != nil { log.Fatalf("Failed to create serverless index: %v", err) - } else { - fmt.Printf("Successfully created serverless index: %s", idx.Name) } idx, err = pc.DescribeIndex(ctx, "pinecone-index")