From ce920dcf22a5aee42d282468fd7a6b429ba62142 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 1 Jul 2024 11:27:05 -0700 Subject: [PATCH 01/19] Add ConfigureIndex method --- pinecone/client.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pinecone/client.go b/pinecone/client.go index 73aefc5..5265b80 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -273,6 +273,44 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { return nil } +type ConfigureIndexRequest struct { + Name string + Pods string // Changes the *size* of your current pod type; cannot change *type* of pod itself + Replicas int32 +} + +func (c *Client) ConfigureIndex(ctx context.Context, in *ConfigureIndexRequest) (*Index, error) { + podType := control.PodSpecPodType(in.Pods) + replicas := control.PodSpecReplicas(in.Replicas) + + request := control.ConfigureIndexRequest{ + Spec: struct { + Pod struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` + } `json:"pod"` + }{ + Pod: struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` + }{ + PodType: &podType, + Replicas: &replicas, + }, + }, + } + res, err := c.restClient.ConfigureIndex(ctx, in.Name, request) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusCreated { + handleErrorResponseBody(res, "failed to configure index: ")} + + return decodeIndex(res.Body) +} + func (c *Client) ListCollections(ctx context.Context) ([]*Collection, error) { res, err := c.restClient.ListCollections(ctx) if err != nil { From 1b6c8679dbefe0fc161b1351f852594f390e85ad Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 1 Jul 2024 12:30:02 -0700 Subject: [PATCH 02/19] Not sure if this is right, but it is working --- pinecone/client.go | 112 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 27 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index 5265b80..41b7c85 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -273,43 +273,101 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { return nil } -type ConfigureIndexRequest struct { - Name string - Pods string // Changes the *size* of your current pod type; cannot change *type* of pod itself - Replicas int32 -} - -func (c *Client) ConfigureIndex(ctx context.Context, in *ConfigureIndexRequest) (*Index, error) { - podType := control.PodSpecPodType(in.Pods) - replicas := control.PodSpecReplicas(in.Replicas) - - request := control.ConfigureIndexRequest{ - Spec: struct { - Pod struct { - PodType *control.PodSpecPodType `json:"pod_type,omitempty"` - Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` - } `json:"pod"` - }{ - Pod: struct { - PodType *control.PodSpecPodType `json:"pod_type,omitempty"` - Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` +func (c *Client) ConfigureIndex(ctx context.Context, name string, pods string, + replicas int32) (*control.ConfigureIndexResponse, + error) { + + podType := control.PodSpecPodType(pods) + replicas = control.PodSpecReplicas(replicas) + + request := control.ConfigureIndexRequest{ + Spec: struct { + Pod struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` + } `json:"pod"` }{ - PodType: &podType, - Replicas: &replicas, + Pod: struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` + }{ + PodType: &podType, + Replicas: &replicas, + }, }, - }, - } - res, err := c.restClient.ConfigureIndex(ctx, in.Name, request) + } + + res, err := c.restClient.ConfigureIndex(ctx, name, request) if err != nil { return nil, err } + defer res.Body.Close() if res.StatusCode != http.StatusCreated { + //bodyBytes, err := io.ReadAll(res.Body) + //if err != nil { + // return nil, fmt.Errorf("failed to read response body: %w", err) + //} + //fmt.Printf("Error response body: %s\n", string(bodyBytes)) + handleErrorResponseBody(res, "failed to configure index: ")} - return decodeIndex(res.Body) -} + + // Inspect the response body + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + response := &control.ConfigureIndexResponse{ + Body: bodyBytes, + HTTPResponse: res, + } + + return response, nil // TODO: why do I need nil here? + +} + + + + +//func (c *Client) ConfigureIndex(ctx context.Context, in *ConfigureIndexRequest) (*Index, error) { +// +// indexToConfigure, err := c.Index(in.Host) +// +// podType := control.PodSpecPodType(in.Pods) +// replicas := control.PodSpecReplicas(in.Replicas) +// +// request := control.ConfigureIndexRequest{ +// Spec: struct { +// Pod struct { +// PodType *control.PodSpecPodType `json:"pod_type,omitempty"` +// Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` +// } `json:"pod"` +// }{ +// Pod: struct { +// PodType *control.PodSpecPodType `json:"pod_type,omitempty"` +// Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` +// }{ +// PodType: &podType, +// Replicas: &replicas, +// }, +// }, +// } +// fmt.Printf("!! Request: %+v\n", request) +// fmt.Printf("Index name: %s\n", c.restClient.In) +// res, err := c.restClient.ConfigureIndex(ctx, in.Name, request) +// if err != nil { +// return nil, err +// } +// defer res.Body.Close() +// +// if res.StatusCode != http.StatusCreated { +// handleErrorResponseBody(res, "failed to configure index: ")} +// +// return decodeIndex(res.Body) +//} func (c *Client) ListCollections(ctx context.Context) ([]*Collection, error) { res, err := c.restClient.ListCollections(ctx) From 366b4bc2bc5db53a414bb0fe912c0cdd14c0fd30 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 11:28:28 -0700 Subject: [PATCH 03/19] Finishing touches --- pinecone/client.go | 160 ++++++++++++++++++++-------------------- pinecone/client_test.go | 35 +++++++++ 2 files changed, 115 insertions(+), 80 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index 41b7c85..8130111 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -273,102 +273,102 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { return nil } -func (c *Client) ConfigureIndex(ctx context.Context, name string, pods string, - replicas int32) (*control.ConfigureIndexResponse, +// ConfigureIndex is used to [scale a pods-based index] up or down by changing the size of pods or the number of +//replicas. +// +// Parameters: +// - name: The name of the index to configure. +// - pods: The size of pods to scale the index to (e.g. for a "p1" pod type, you could pass "p1.x2", "p1.x4", +// "p1.x6", etc. Optional. +// - replicas: The number of replicas to scale the index to. Optional. +// This is capped by the maximum number of replicas allowed in your Pinecone project. To configure this number, +// go to app.pinecone.io, select your project, and configure the maximum number of pods. +// +// Note: You can only scale an index up, not down. If you want to scale an index down, +// you must create a new index with the desired configuration. +// +// It returns a ConfigureIndexResponse object, which contains the new configuration of the index, or an error. +// +// Example for a pods-based index originally configured with 1 "p1" pod of size "x2" and 1 replica: +// // To scale the size of your pods from "x2" to "x4": +// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", nil) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } +// +// // To scale the number of replicas: +// _, err := pc.ConfigureIndex(ctx, "my-index", nil, 4) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } +// +// // To scale both the size of your pods and the number of replicas: +// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", 4) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } +// +// [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes +func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, + replicas *int32) (*control.ConfigureIndexResponse, error) { - podType := control.PodSpecPodType(pods) - replicas = control.PodSpecReplicas(replicas) + var podType *control.PodSpecPodType + var replicasAmt *control.PodSpecReplicas - request := control.ConfigureIndexRequest{ - Spec: struct { - Pod struct { - PodType *control.PodSpecPodType `json:"pod_type,omitempty"` - Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` - } `json:"pod"` + if pods == nil && replicas == nil { + return nil, fmt.Errorf("Must specify either pods or replicas") + } + + if pods != nil { + podTypeVal := control.PodSpecPodType(*pods) + podType = &podTypeVal + } + if replicas != nil { + replicasVal := control.PodSpecReplicas(*replicas) + replicasAmt = &replicasVal + } + + request := control.ConfigureIndexRequest{ + Spec: struct { + Pod struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` + } `json:"pod"` + }{ + Pod: struct { + PodType *control.PodSpecPodType `json:"pod_type,omitempty"` + Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` }{ - Pod: struct { - PodType *control.PodSpecPodType `json:"pod_type,omitempty"` - Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` - }{ - PodType: &podType, - Replicas: &replicas, - }, + PodType: podType, + Replicas: replicasAmt, }, - } + }, + } - res, err := c.restClient.ConfigureIndex(ctx, name, request) + req, err := c.restClient.ConfigureIndex(ctx, name, request) if err != nil { - return nil, err + log.Fatalf("Failed to configure index %s. Error: %v", name, err) + } + if req.StatusCode != http.StatusOK { + err := handleErrorResponseBody(req, "failed to configure index: ") + if err != nil { + return nil, err + } } - - defer res.Body.Close() - - if res.StatusCode != http.StatusCreated { - //bodyBytes, err := io.ReadAll(res.Body) - //if err != nil { - // return nil, fmt.Errorf("failed to read response body: %w", err) - //} - //fmt.Printf("Error response body: %s\n", string(bodyBytes)) - - handleErrorResponseBody(res, "failed to configure index: ")} - // Inspect the response body - bodyBytes, err := io.ReadAll(res.Body) + response, err := control.ParseConfigureIndexResponse(req) // TODO: need this? if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - response := &control.ConfigureIndexResponse{ - Body: bodyBytes, - HTTPResponse: res, + log.Fatalf("Failed to configure index %s. Error: %v", name, err) + return nil, err } - return response, nil // TODO: why do I need nil here? + defer req.Body.Close() + return response, nil } - - - -//func (c *Client) ConfigureIndex(ctx context.Context, in *ConfigureIndexRequest) (*Index, error) { -// -// indexToConfigure, err := c.Index(in.Host) -// -// podType := control.PodSpecPodType(in.Pods) -// replicas := control.PodSpecReplicas(in.Replicas) -// -// request := control.ConfigureIndexRequest{ -// Spec: struct { -// Pod struct { -// PodType *control.PodSpecPodType `json:"pod_type,omitempty"` -// Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` -// } `json:"pod"` -// }{ -// Pod: struct { -// PodType *control.PodSpecPodType `json:"pod_type,omitempty"` -// Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` -// }{ -// PodType: &podType, -// Replicas: &replicas, -// }, -// }, -// } -// fmt.Printf("!! Request: %+v\n", request) -// fmt.Printf("Index name: %s\n", c.restClient.In) -// res, err := c.restClient.ConfigureIndex(ctx, in.Name, request) -// if err != nil { -// return nil, err -// } -// defer res.Body.Close() -// -// if res.StatusCode != http.StatusCreated { -// handleErrorResponseBody(res, "failed to configure index: ")} -// -// return decodeIndex(res.Body) -//} - func (c *Client) ListCollections(ctx context.Context) ([]*Collection, error) { res, err := c.restClient.ListCollections(ctx) if err != nil { diff --git a/pinecone/client_test.go b/pinecone/client_test.go index f8b2c94..446f6f8 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -496,6 +496,41 @@ func (ts *ClientTests) TestDeleteCollection() { require.NoError(ts.T(), err) } +func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { + indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 + pods := "p1.x1" + replicas := int32(1) + _, err := ts.client.ConfigureIndex(context.Background(), indexName, &pods, &replicas) + require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) +} + +func (ts *ClientTests) TestConfigureIndexNoPods() { + indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 + replicas := int32(2) + _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, &replicas) + require.NoError(ts.T(), err) +} + +func (ts *ClientTests) TestConfigureIndexNoReplicas() { + indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 + pods := "p1.x4" + _, err := ts.client.ConfigureIndex(context.Background(), indexName, &pods, nil) + require.NoError(ts.T(), err) +} + +func (ts *ClientTests) TestConfigureIndexNoPodsOrReplicas() { + indexName := "go-pinecone-pods-test-index" // This index is created with p1.x8 + _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, nil) + require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) +} + +func (ts *ClientTests) TestConfigureIndexHitPodLimit() { + indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 + replicas := int32(30000) + _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, &replicas) + require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error()) +} + func (ts *ClientTests) deleteIndex(name string) error { return ts.client.DeleteIndex(context.Background(), name) } From 1b58b5fa26c405d79c3d2ddac8d96df44ca4e7b5 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 12:49:20 -0700 Subject: [PATCH 04/19] Make test indexes --- pinecone/client_test.go | 121 +++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 21 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index 446f6f8..d7d7ac6 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -3,18 +3,18 @@ package pinecone import ( "context" "fmt" + "github.com/google/uuid" + "github.com/pinecone-io/go-pinecone/internal/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "io" + "log" "net/http" "os" "reflect" "strings" "testing" - - "github.com/google/uuid" - "github.com/pinecone-io/go-pinecone/internal/mocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" ) type ClientTests struct { @@ -24,6 +24,7 @@ type ClientTests struct { sourceTag string podIndex string serverlessIndex string + configureIndex string } func TestClient(t *testing.T) { @@ -81,6 +82,9 @@ func (ts *ClientTests) SetupSuite() { ts.serverlessIndex = os.Getenv("TEST_SERVERLESS_INDEX_NAME") require.NotEmpty(ts.T(), ts.serverlessIndex, "TEST_SERVERLESS_INDEX_NAME env variable not set") + ts.configureIndex = os.Getenv("TEST_CONFIGURE_INDEX_NAME") + require.NotEmpty(ts.T(), ts.configureIndex, "TEST_CONFIGURE_INDEX_NAME env variable not set") + client, err := NewClient(NewClientParams{}) require.NoError(ts.T(), err) @@ -497,38 +501,113 @@ func (ts *ClientTests) TestDeleteCollection() { } func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { - indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 - pods := "p1.x1" - replicas := int32(1) - _, err := ts.client.ConfigureIndex(context.Background(), indexName, &pods, &replicas) + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + Name: ts.configureIndex, + Dimension: 10, + Metric: Cosine, + Environment: "us-east1-gcp", + PodType: "p1.x2", + },) + if erdr != nil { + log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + } + + pods := "p1.x1" // test index originally created with "p1.x2" pods + replicas := int32(1) // could be nil, but do not want to test nil case here + _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, &replicas) require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) + + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + } } -func (ts *ClientTests) TestConfigureIndexNoPods() { - indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 +func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + Name: ts.configureIndex, + Dimension: 10, + Metric: Cosine, + Environment: "us-east1-gcp", + PodType: "p1.x2", + },) + if erdr != nil { + log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + } + replicas := int32(2) - _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, &replicas) + _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) require.NoError(ts.T(), err) + + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + } } -func (ts *ClientTests) TestConfigureIndexNoReplicas() { - indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 +func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + Name: ts.configureIndex, + Dimension: 10, + Metric: Cosine, + Environment: "us-east1-gcp", + PodType: "p1.x2", + },) + if erdr != nil { + log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + } + pods := "p1.x4" - _, err := ts.client.ConfigureIndex(context.Background(), indexName, &pods, nil) + _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, nil) require.NoError(ts.T(), err) + + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + } } -func (ts *ClientTests) TestConfigureIndexNoPodsOrReplicas() { - indexName := "go-pinecone-pods-test-index" // This index is created with p1.x8 - _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, nil) +func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + Name: ts.configureIndex, + Dimension: 10, + Metric: Cosine, + Environment: "us-east1-gcp", + PodType: "p1.x2", + },) + if erdr != nil { + log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + } + + _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, nil) require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) + + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + } } func (ts *ClientTests) TestConfigureIndexHitPodLimit() { - indexName := "go-pinecone-pods-test-index" // This index is created with p1.x2 + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + Name: ts.configureIndex, + Dimension: 10, + Metric: Cosine, + Environment: "us-east1-gcp", + PodType: "p1.x2", + },) + if erdr != nil { + log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + } + replicas := int32(30000) - _, err := ts.client.ConfigureIndex(context.Background(), indexName, nil, &replicas) + _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error()) + + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + } } func (ts *ClientTests) deleteIndex(name string) error { From 89e33687aaa24c98e4f9cbb000f2b4059e35901f Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 12:50:55 -0700 Subject: [PATCH 05/19] Add new secret to ci yaml --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 60c697e..3d2e4d0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,3 +22,4 @@ jobs: TEST_POD_INDEX_NAME: ${{ secrets.TEST_POD_INDEX_NAME }} TEST_SERVERLESS_INDEX_NAME: ${{ secrets.TEST_SERVERLESS_INDEX_NAME }} PINECONE_API_KEY: ${{ secrets.API_KEY }} + TEST_CONFIGURE_INDEX_NAME: ${{ secrets.TEST_CONFIGURE_INDEX_NAME }} From 7fb2f0c45efabe04d79c25cbee8b7f51b6264b94 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 14:35:44 -0700 Subject: [PATCH 06/19] Think cache has cleared now --- pinecone/client_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index d7d7ac6..ea10a15 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -519,7 +519,7 @@ func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } } @@ -541,7 +541,7 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } } @@ -563,7 +563,7 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } } @@ -584,7 +584,7 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } } @@ -606,7 +606,7 @@ func (ts *ClientTests) TestConfigureIndexHitPodLimit() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error recreating index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } } From 1b499701759b5bb70f915f986dfebd7b319bc6b4 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 14:39:11 -0700 Subject: [PATCH 07/19] test out deleting, creating, deleting --- pinecone/client_test.go | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index ea10a15..0003a36 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -501,6 +501,10 @@ func (ts *ClientTests) TestDeleteCollection() { } func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, Dimension: 10, @@ -517,13 +521,17 @@ func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, &replicas) require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) } } func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, Dimension: 10, @@ -539,13 +547,17 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) require.NoError(ts.T(), err) - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) } } func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, Dimension: 10, @@ -561,13 +573,17 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, nil) require.NoError(ts.T(), err) - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) } } func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, Dimension: 10, @@ -582,13 +598,17 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, nil) require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) } } func (ts *ClientTests) TestConfigureIndexHitPodLimit() { + deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + if deleteErr != nil { + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, Dimension: 10, @@ -604,9 +624,9 @@ func (ts *ClientTests) TestConfigureIndexHitPodLimit() { _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error()) - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) + deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) } } From 018feb0094d58383d26076425f9bcda1b9c85a22 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 14:42:04 -0700 Subject: [PATCH 08/19] Trying again without log.Fatalf --- pinecone/client_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index 0003a36..e41fb0b 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -503,7 +503,9 @@ func (ts *ClientTests) TestDeleteCollection() { func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + fmt.Println("Found a delete error: ", deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, @@ -530,7 +532,9 @@ func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + fmt.Println("Found a delete error: ", deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, @@ -556,7 +560,9 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + fmt.Println("Found a delete error: ", deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, @@ -582,7 +588,8 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + fmt.Println("Found a delete error: ", deleteErr) + //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, @@ -607,7 +614,9 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { func (ts *ClientTests) TestConfigureIndexHitPodLimit() { deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) + fmt.Println("Found a delete error: ", deleteErr) + } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: ts.configureIndex, From 21ce7d62dde09098d86e44ee3cff09be16a992c7 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 15:28:04 -0700 Subject: [PATCH 09/19] Get rid of single name index --- pinecone/client_test.go | 111 +++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 65 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index e41fb0b..eff4405 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -501,142 +501,123 @@ func (ts *ClientTests) TestDeleteCollection() { } func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) - fmt.Println("Found a delete error: ", deleteErr) + name := uuid.New().String() + + defer func(ts *ClientTests, name string) { + err := ts.deleteIndex(name) + require.NoError(ts.T(), err) + }(ts, name) - } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: ts.configureIndex, + Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) if erdr != nil { - log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + log.Fatalf("Error creating index %s: %v", name, erdr) } pods := "p1.x1" // test index originally created with "p1.x2" pods replicas := int32(1) // could be nil, but do not want to test nil case here - _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, &replicas) + _, err := ts.client.ConfigureIndex(context.Background(), name, &pods, &replicas) require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) - - deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) - } } func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) - fmt.Println("Found a delete error: ", deleteErr) + name := uuid.New().String() + + defer func(ts *ClientTests, name string) { + err := ts.deleteIndex(name) + require.NoError(ts.T(), err) + }(ts, name) - } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: ts.configureIndex, + Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) if erdr != nil { - log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + log.Fatalf("Error creating index %s: %v", name, erdr) } replicas := int32(2) - _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) + _, err := ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) require.NoError(ts.T(), err) - - deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) - } } func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) - fmt.Println("Found a delete error: ", deleteErr) + name := uuid.New().String() + + defer func(ts *ClientTests, name string) { + err := ts.deleteIndex(name) + require.NoError(ts.T(), err) + }(ts, name) - } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: ts.configureIndex, + Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) if erdr != nil { - log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + log.Fatalf("Error creating index %s: %v", name, erdr) } pods := "p1.x4" - _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, &pods, nil) + _, err := ts.client.ConfigureIndex(context.Background(), name, &pods, nil) require.NoError(ts.T(), err) - - deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) - } } func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - fmt.Println("Found a delete error: ", deleteErr) - //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) - } + name := uuid.New().String() + + defer func(ts *ClientTests, name string) { + err := ts.deleteIndex(name) + require.NoError(ts.T(), err) + }(ts, name) + _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: ts.configureIndex, + Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) if erdr != nil { - log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + log.Fatalf("Error creating index %s: %v", name, erdr) } - _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, nil) + _, err := ts.client.ConfigureIndex(context.Background(), name, nil, nil) require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) - - deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) - } } func (ts *ClientTests) TestConfigureIndexHitPodLimit() { - deleteErr := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - //log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr) - fmt.Println("Found a delete error: ", deleteErr) + name := uuid.New().String() + + defer func(ts *ClientTests, name string) { + err := ts.deleteIndex(name) + require.NoError(ts.T(), err) + }(ts, name) - } _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: ts.configureIndex, + Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) if erdr != nil { - log.Fatalf("Error creating index %s: %v", ts.configureIndex, erdr) + log.Fatalf("Error creating index %s: %v", name, erdr) } replicas := int32(30000) - _, err := ts.client.ConfigureIndex(context.Background(), ts.configureIndex, nil, &replicas) + _, err := ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error()) - - deleteErr2 := ts.client.DeleteIndex(context.Background(), ts.configureIndex) - if deleteErr != nil { - log.Fatalf("Error deleting index %s: %v", ts.configureIndex, deleteErr2) - } } func (ts *ClientTests) deleteIndex(name string) error { From a99a56f64cdc18c417d81bd03a509b9b546c8d2b Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 15:32:43 -0700 Subject: [PATCH 10/19] Remove secret from CI + rename erdr to err --- .github/workflows/ci.yaml | 1 - pinecone/client_test.go | 44 ++++++++++++++++++--------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d2e4d0..60c697e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,4 +22,3 @@ jobs: TEST_POD_INDEX_NAME: ${{ secrets.TEST_POD_INDEX_NAME }} TEST_SERVERLESS_INDEX_NAME: ${{ secrets.TEST_SERVERLESS_INDEX_NAME }} PINECONE_API_KEY: ${{ secrets.API_KEY }} - TEST_CONFIGURE_INDEX_NAME: ${{ secrets.TEST_CONFIGURE_INDEX_NAME }} diff --git a/pinecone/client_test.go b/pinecone/client_test.go index eff4405..dbead37 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -24,7 +24,6 @@ type ClientTests struct { sourceTag string podIndex string serverlessIndex string - configureIndex string } func TestClient(t *testing.T) { @@ -82,9 +81,6 @@ func (ts *ClientTests) SetupSuite() { ts.serverlessIndex = os.Getenv("TEST_SERVERLESS_INDEX_NAME") require.NotEmpty(ts.T(), ts.serverlessIndex, "TEST_SERVERLESS_INDEX_NAME env variable not set") - ts.configureIndex = os.Getenv("TEST_CONFIGURE_INDEX_NAME") - require.NotEmpty(ts.T(), ts.configureIndex, "TEST_CONFIGURE_INDEX_NAME env variable not set") - client, err := NewClient(NewClientParams{}) require.NoError(ts.T(), err) @@ -508,20 +504,20 @@ func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { require.NoError(ts.T(), err) }(ts, name) - _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) - if erdr != nil { - log.Fatalf("Error creating index %s: %v", name, erdr) + if err != nil { + log.Fatalf("Error creating index %s: %v", name, err) } pods := "p1.x1" // test index originally created with "p1.x2" pods replicas := int32(1) // could be nil, but do not want to test nil case here - _, err := ts.client.ConfigureIndex(context.Background(), name, &pods, &replicas) + _, err = ts.client.ConfigureIndex(context.Background(), name, &pods, &replicas) require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) } @@ -533,19 +529,19 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { require.NoError(ts.T(), err) }(ts, name) - _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) - if erdr != nil { - log.Fatalf("Error creating index %s: %v", name, erdr) + if err != nil { + log.Fatalf("Error creating index %s: %v", name, err) } replicas := int32(2) - _, err := ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) + _, err = ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) require.NoError(ts.T(), err) } @@ -557,19 +553,19 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { require.NoError(ts.T(), err) }(ts, name) - _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) - if erdr != nil { - log.Fatalf("Error creating index %s: %v", name, erdr) + if err != nil { + log.Fatalf("Error creating index %s: %v", name, err) } pods := "p1.x4" - _, err := ts.client.ConfigureIndex(context.Background(), name, &pods, nil) + _, err = ts.client.ConfigureIndex(context.Background(), name, &pods, nil) require.NoError(ts.T(), err) } @@ -581,18 +577,18 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { require.NoError(ts.T(), err) }(ts, name) - _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) - if erdr != nil { - log.Fatalf("Error creating index %s: %v", name, erdr) + if err != nil { + log.Fatalf("Error creating index %s: %v", name, err) } - _, err := ts.client.ConfigureIndex(context.Background(), name, nil, nil) + _, err = ts.client.ConfigureIndex(context.Background(), name, nil, nil) require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) } @@ -604,19 +600,19 @@ func (ts *ClientTests) TestConfigureIndexHitPodLimit() { require.NoError(ts.T(), err) }(ts, name) - _, erdr := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ + _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ Name: name, Dimension: 10, Metric: Cosine, Environment: "us-east1-gcp", PodType: "p1.x2", },) - if erdr != nil { - log.Fatalf("Error creating index %s: %v", name, erdr) + if err != nil { + log.Fatalf("Error creating index %s: %v", name, err) } replicas := int32(30000) - _, err := ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) + _, err = ts.client.ConfigureIndex(context.Background(), name, nil, &replicas) require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error()) } From 75713e55bdfd441a27d2523f545d5fd26391dd0f Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 15:38:27 -0700 Subject: [PATCH 11/19] Clean up docstring --- pinecone/client.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index 8130111..fa83c67 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -273,21 +273,23 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { return nil } -// ConfigureIndex is used to [scale a pods-based index] up or down by changing the size of pods or the number of +// ConfigureIndex is used to [scale a pods-based index] up or down by changing the size of the pods or the number of //replicas. // // Parameters: -// - name: The name of the index to configure. -// - pods: The size of pods to scale the index to (e.g. for a "p1" pod type, you could pass "p1.x2", "p1.x4", -// "p1.x6", etc. Optional. -// - replicas: The number of replicas to scale the index to. Optional. -// This is capped by the maximum number of replicas allowed in your Pinecone project. To configure this number, -// go to app.pinecone.io, select your project, and configure the maximum number of pods. +// - name: The name of the index to configure. +// - pods: (Optional) The pod size to scale the index to (e.g. for a "p1" pod type, +// you could pass "p1.x2" to scale your index to the "x2" size, +// or you could pass "p1.x4" to scale your index to the "x4" size, and +// so forth. +// - replicas: (Optional) The number of replicas to scale the index to. +// This is capped by the maximum number of replicas allowed in your Pinecone project. To configure this number, +// go to [app.pinecone.io], select your project, and configure the maximum number of pods. // // Note: You can only scale an index up, not down. If you want to scale an index down, -// you must create a new index with the desired configuration. +// you must create a new index with the desired configuration. // -// It returns a ConfigureIndexResponse object, which contains the new configuration of the index, or an error. +// Returns a ConfigureIndexResponse object, which contains the new configuration of the index, or an error. // // Example for a pods-based index originally configured with 1 "p1" pod of size "x2" and 1 replica: // // To scale the size of your pods from "x2" to "x4": @@ -309,6 +311,7 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { // } // // [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes +// [app.pinecone.io]: https://app.pinecone.io func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, replicas *int32) (*control.ConfigureIndexResponse, error) { From 70e0bf2f29cf50a471a605c11e35dba0037c63b9 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Mon, 8 Jul 2024 15:42:08 -0700 Subject: [PATCH 12/19] Lint --- pinecone/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pinecone/client.go b/pinecone/client.go index fa83c67..6df1bf0 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -360,7 +360,6 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, } } - response, err := control.ParseConfigureIndexResponse(req) // TODO: need this? if err != nil { log.Fatalf("Failed to configure index %s. Error: %v", name, err) From cb6b71645e978f9347089097820ab00cccaa7ef4 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Wed, 10 Jul 2024 12:43:12 -0700 Subject: [PATCH 13/19] Run gofmt and goimports --- pinecone/client.go | 54 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index adc8273..b4fe59b 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -760,17 +760,17 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { } // ConfigureIndex is used to [scale a pods-based index] up or down by changing the size of the pods or the number of -//replicas. +// replicas. // // Parameters: -// - name: The name of the index to configure. -// - pods: (Optional) The pod size to scale the index to (e.g. for a "p1" pod type, -// you could pass "p1.x2" to scale your index to the "x2" size, -// or you could pass "p1.x4" to scale your index to the "x4" size, and -// so forth. -// - replicas: (Optional) The number of replicas to scale the index to. -// This is capped by the maximum number of replicas allowed in your Pinecone project. To configure this number, -// go to [app.pinecone.io], select your project, and configure the maximum number of pods. +// - name: The name of the index to configure. +// - pods: (Optional) The pod size to scale the index to (e.g. for a "p1" pod type, +// you could pass "p1.x2" to scale your index to the "x2" size, +// or you could pass "p1.x4" to scale your index to the "x4" size, and +// so forth. +// - replicas: (Optional) The number of replicas to scale the index to. +// This is capped by the maximum number of replicas allowed in your Pinecone project. To configure this number, +// go to [app.pinecone.io], select your project, and configure the maximum number of pods. // // Note: You can only scale an index up, not down. If you want to scale an index down, // you must create a new index with the desired configuration. @@ -778,23 +778,24 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { // Returns a ConfigureIndexResponse object, which contains the new configuration of the index, or an error. // // Example for a pods-based index originally configured with 1 "p1" pod of size "x2" and 1 replica: -// // To scale the size of your pods from "x2" to "x4": -// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", nil) -// if err != nil { -// fmt.Printf("Failed to configure index: %v\n", err) -// } -// -// // To scale the number of replicas: -// _, err := pc.ConfigureIndex(ctx, "my-index", nil, 4) -// if err != nil { -// fmt.Printf("Failed to configure index: %v\n", err) -// } -// -// // To scale both the size of your pods and the number of replicas: -// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", 4) -// if err != nil { -// fmt.Printf("Failed to configure index: %v\n", err) -// } +// +// // To scale the size of your pods from "x2" to "x4": +// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", nil) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } +// +// // To scale the number of replicas: +// _, err := pc.ConfigureIndex(ctx, "my-index", nil, 4) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } +// +// // To scale both the size of your pods and the number of replicas: +// _, err := pc.ConfigureIndex(ctx, "my-index", "p1.x4", 4) +// if err != nil { +// fmt.Printf("Failed to configure index: %v\n", err) +// } // // [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes // [app.pinecone.io]: https://app.pinecone.io @@ -857,7 +858,6 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, return response, nil } - // ListCollections retrieves a list of all Collections in a Pinecone [project]. See Collection for more information. // // Parameters: From f53f53663282889fa01a609c36b9903dbfc54e5f Mon Sep 17 00:00:00 2001 From: aulorbe Date: Wed, 10 Jul 2024 12:57:28 -0700 Subject: [PATCH 14/19] Rename vars, clean up logic --- pinecone/client.go | 34 ++++++++-------------------------- pinecone/client_test.go | 2 +- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index b4fe59b..df1b9c9 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -799,24 +799,12 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { // // [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes // [app.pinecone.io]: https://app.pinecone.io -func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, +func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *string, replicas *int32) (*control.ConfigureIndexResponse, error) { - var podType *control.PodSpecPodType - var replicasAmt *control.PodSpecReplicas - - if pods == nil && replicas == nil { - return nil, fmt.Errorf("Must specify either pods or replicas") - } - - if pods != nil { - podTypeVal := control.PodSpecPodType(*pods) - podType = &podTypeVal - } - if replicas != nil { - replicasVal := control.PodSpecReplicas(*replicas) - replicasAmt = &replicasVal + if podType == nil && replicas == nil { + return nil, fmt.Errorf("must specify either podType or replicas") } request := control.ConfigureIndexRequest{ @@ -831,29 +819,23 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, pods *string, Replicas *control.PodSpecReplicas `json:"replicas,omitempty"` }{ PodType: podType, - Replicas: replicasAmt, + Replicas: replicas, }, }, } - req, err := c.restClient.ConfigureIndex(ctx, name, request) + res, err := c.restClient.ConfigureIndex(ctx, name, request) if err != nil { - log.Fatalf("Failed to configure index %s. Error: %v", name, err) - } - if req.StatusCode != http.StatusOK { - err := handleErrorResponseBody(req, "failed to configure index: ") - if err != nil { - return nil, err - } + return nil, handleErrorResponseBody(res, "failed to configure index: ") } - response, err := control.ParseConfigureIndexResponse(req) // TODO: need this? + response, err := control.ParseConfigureIndexResponse(res) // TODO: get rid of internal here if err != nil { log.Fatalf("Failed to configure index %s. Error: %v", name, err) return nil, err } - defer req.Body.Close() + defer res.Body.Close() return response, nil } diff --git a/pinecone/client_test.go b/pinecone/client_test.go index 9446546..c7dc813 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -658,7 +658,7 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { } _, err = ts.client.ConfigureIndex(context.Background(), name, nil, nil) - require.ErrorContainsf(ts.T(), err, "Must specify either pods or replicas", err.Error()) + require.ErrorContainsf(ts.T(), err, "must specify either podType or replicas", err.Error()) } func (ts *ClientTests) TestConfigureIndexHitPodLimit() { From adc7e3a898a6cf3f9546e59a7c534550319d68e5 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Thu, 11 Jul 2024 09:53:58 -0700 Subject: [PATCH 15/19] Address final comments --- pinecone/client.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index df1b9c9..2eec8a4 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -775,7 +775,7 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { // Note: You can only scale an index up, not down. If you want to scale an index down, // you must create a new index with the desired configuration. // -// Returns a ConfigureIndexResponse object, which contains the new configuration of the index, or an error. +// Returns a pointer to a configured Index object or an error. // // Example for a pods-based index originally configured with 1 "p1" pod of size "x2" and 1 replica: // @@ -800,8 +800,7 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error { // [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes // [app.pinecone.io]: https://app.pinecone.io func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *string, - replicas *int32) (*control.ConfigureIndexResponse, - error) { + replicas *int32) (*Index, error) { if podType == nil && replicas == nil { return nil, fmt.Errorf("must specify either podType or replicas") @@ -829,14 +828,15 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *strin return nil, handleErrorResponseBody(res, "failed to configure index: ") } - response, err := control.ParseConfigureIndexResponse(res) // TODO: get rid of internal here + defer res.Body.Close() + + response, err := decodeIndex(res.Body) + if err != nil { log.Fatalf("Failed to configure index %s. Error: %v", name, err) return nil, err } - defer res.Body.Close() - return response, nil } From f3939ed03aed0bbdfe2d250008cac0b651776bb2 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Fri, 12 Jul 2024 08:18:31 -0700 Subject: [PATCH 16/19] Fix ConfigureIndex error handling --- pinecone/client.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pinecone/client.go b/pinecone/client.go index 2eec8a4..c000b2b 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -825,19 +825,16 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *strin res, err := c.restClient.ConfigureIndex(ctx, name, request) if err != nil { - return nil, handleErrorResponseBody(res, "failed to configure index: ") + return nil, err } defer res.Body.Close() - response, err := decodeIndex(res.Body) - - if err != nil { - log.Fatalf("Failed to configure index %s. Error: %v", name, err) - return nil, err + if res.StatusCode != http.StatusCreated { + return nil, handleErrorResponseBody(res, "failed to configure index: ") } - return response, nil + return decodeIndex(res.Body) } // ListCollections retrieves a list of all Collections in a Pinecone [project]. See Collection for more information. @@ -1229,6 +1226,7 @@ type errorResponseMap struct { } func handleErrorResponseBody(response *http.Response, errMsgPrefix string) error { + fmt.Println("Inside handleErrorResponseBody!, first line") resBodyBytes, err := io.ReadAll(response.Body) if err != nil { return fmt.Errorf("failed to read response body: %w", err) From a3049bd788fd211256d9e3d381082c70051a6c15 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Fri, 12 Jul 2024 08:19:28 -0700 Subject: [PATCH 17/19] Oops, remove debugging line --- pinecone/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pinecone/client.go b/pinecone/client.go index c000b2b..4fe5a82 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -1226,7 +1226,6 @@ type errorResponseMap struct { } func handleErrorResponseBody(response *http.Response, errMsgPrefix string) error { - fmt.Println("Inside handleErrorResponseBody!, first line") resBodyBytes, err := io.ReadAll(response.Body) if err != nil { return fmt.Errorf("failed to read response body: %w", err) From 104661935b0c7e9c3c9eca07ba5f7e3d2396a8e6 Mon Sep 17 00:00:00 2001 From: aulorbe Date: Fri, 12 Jul 2024 08:51:28 -0700 Subject: [PATCH 18/19] Change to StatusOk b/c not checking if index has been created --- pinecone/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinecone/client.go b/pinecone/client.go index 4fe5a82..906c309 100644 --- a/pinecone/client.go +++ b/pinecone/client.go @@ -830,7 +830,7 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *strin defer res.Body.Close() - if res.StatusCode != http.StatusCreated { + if res.StatusCode != http.StatusOK { return nil, handleErrorResponseBody(res, "failed to configure index: ") } From 676904120911b5c14c283a5af431da513439856d Mon Sep 17 00:00:00 2001 From: aulorbe Date: Fri, 12 Jul 2024 08:54:33 -0700 Subject: [PATCH 19/19] lint --- pinecone/client_test.go | 63 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/pinecone/client_test.go b/pinecone/client_test.go index c7dc813..80aa6a6 100644 --- a/pinecone/client_test.go +++ b/pinecone/client_test.go @@ -3,11 +3,6 @@ package pinecone import ( "context" "fmt" - "github.com/google/uuid" - "github.com/pinecone-io/go-pinecone/internal/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "io" "log" "net/http" @@ -15,6 +10,12 @@ import ( "reflect" "strings" "testing" + + "github.com/google/uuid" + "github.com/pinecone-io/go-pinecone/internal/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) type ClientTests struct { @@ -574,17 +575,17 @@ func (ts *ClientTests) TestConfigureIndexIllegalScaleDown() { }(ts, name) _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: name, - Dimension: 10, - Metric: Cosine, + Name: name, + Dimension: 10, + Metric: Cosine, Environment: "us-east1-gcp", - PodType: "p1.x2", - },) + PodType: "p1.x2", + }) if err != nil { log.Fatalf("Error creating index %s: %v", name, err) } - pods := "p1.x1" // test index originally created with "p1.x2" pods + pods := "p1.x1" // test index originally created with "p1.x2" pods replicas := int32(1) // could be nil, but do not want to test nil case here _, err = ts.client.ConfigureIndex(context.Background(), name, &pods, &replicas) require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error()) @@ -599,12 +600,12 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoPods() { }(ts, name) _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: name, - Dimension: 10, - Metric: Cosine, + Name: name, + Dimension: 10, + Metric: Cosine, Environment: "us-east1-gcp", - PodType: "p1.x2", - },) + PodType: "p1.x2", + }) if err != nil { log.Fatalf("Error creating index %s: %v", name, err) } @@ -623,12 +624,12 @@ func (ts *ClientTests) TestConfigureIndexScaleUpNoReplicas() { }(ts, name) _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: name, - Dimension: 10, - Metric: Cosine, + Name: name, + Dimension: 10, + Metric: Cosine, Environment: "us-east1-gcp", - PodType: "p1.x2", - },) + PodType: "p1.x2", + }) if err != nil { log.Fatalf("Error creating index %s: %v", name, err) } @@ -647,12 +648,12 @@ func (ts *ClientTests) TestConfigureIndexIllegalNoPodsOrReplicas() { }(ts, name) _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: name, - Dimension: 10, - Metric: Cosine, + Name: name, + Dimension: 10, + Metric: Cosine, Environment: "us-east1-gcp", - PodType: "p1.x2", - },) + PodType: "p1.x2", + }) if err != nil { log.Fatalf("Error creating index %s: %v", name, err) } @@ -670,12 +671,12 @@ func (ts *ClientTests) TestConfigureIndexHitPodLimit() { }(ts, name) _, err := ts.client.CreatePodIndex(context.Background(), &CreatePodIndexRequest{ - Name: name, - Dimension: 10, - Metric: Cosine, + Name: name, + Dimension: 10, + Metric: Cosine, Environment: "us-east1-gcp", - PodType: "p1.x2", - },) + PodType: "p1.x2", + }) if err != nil { log.Fatalf("Error creating index %s: %v", name, err) }