Skip to content

Commit

Permalink
add support for DeletionProtection to Index models and operations
Browse files Browse the repository at this point in the history
  • Loading branch information
austin-denoble committed Jul 24, 2024
1 parent be9fe92 commit f12a9b4
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 67 deletions.
104 changes: 73 additions & 31 deletions pinecone/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -429,15 +429,16 @@ func (c *Client) ListIndexes(ctx context.Context) ([]*Index, error) {
//
// [type of pods]: https://docs.pinecone.io/guides/indexes/choose-a-pod-type-and-size
type CreatePodIndexRequest struct {
Name string
Dimension int32
Metric IndexMetric
Environment string
PodType string
Shards int32
Replicas int32
SourceCollection *string
MetadataConfig *PodSpecMetadataConfig
Name string
Dimension int32
Metric IndexMetric
DeletionProtection string
Environment string
PodType string
Shards int32
Replicas int32
SourceCollection *string
MetadataConfig *PodSpecMetadataConfig
}

// ReplicaCount ensures the replica count of a pods-based Index is >1.
Expand Down Expand Up @@ -501,11 +502,14 @@ func (req CreatePodIndexRequest) TotalCount() int {
// fmt.Printf("Successfully created pod index: %s", idx.Name)
// }
func (c *Client) CreatePodIndex(ctx context.Context, in *CreatePodIndexRequest) (*Index, error) {
deletionProtection := toDeletionProtection(in.DeletionProtection)
metric := control.CreateIndexRequestMetric(in.Metric)

req := control.CreateIndexRequest{
Name: in.Name,
Dimension: in.Dimension,
Metric: &metric,
Name: in.Name,
Dimension: in.Dimension,
Metric: &metric,
DeletionProtection: &deletionProtection,
}

req.Spec = control.IndexSpec{
Expand Down Expand Up @@ -590,11 +594,12 @@ func (c *Client) CreatePodIndex(ctx context.Context, in *CreatePodIndexRequest)
// [region]: https://docs.pinecone.io/troubleshooting/available-cloud-regions
// [cloud provider]: https://docs.pinecone.io/troubleshooting/available-cloud-regions#regions-available-for-serverless-indexes
type CreateServerlessIndexRequest struct {
Name string
Dimension int32
Metric IndexMetric
Cloud Cloud
Region string
Name string
Dimension int32
Metric IndexMetric
DeletionProtection string
Cloud Cloud
Region string
}

// CreateServerlessIndex creates and initializes a new serverless Index via the specified Client.
Expand Down Expand Up @@ -636,11 +641,18 @@ type CreateServerlessIndexRequest struct {
// fmt.Printf("Successfully created serverless index: %s", idx.Name)
// }
func (c *Client) CreateServerlessIndex(ctx context.Context, in *CreateServerlessIndexRequest) (*Index, error) {
deletionProtection := toDeletionProtection(in.DeletionProtection)

fmt.Printf("in.Metric: %v\n", in.Metric)
fmt.Printf("control.CreateIndexRequestMetric(in.Metric): %v\n", control.CreateIndexRequestMetric(in.Metric))

metric := control.CreateIndexRequestMetric(in.Metric)

req := control.CreateIndexRequest{
Name: in.Name,
Dimension: in.Dimension,
Metric: &metric,
Name: in.Name,
Dimension: in.Dimension,
Metric: &metric,
DeletionProtection: &deletionProtection,
Spec: control.IndexSpec{
Serverless: &control.ServerlessSpec{
Cloud: control.ServerlessSpecCloud(in.Cloud),
Expand Down Expand Up @@ -754,6 +766,12 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error {
return nil
}

type ConfigureIndexParams struct {
PodType string
Replicas int32
DeletionProtection control.DeletionProtection
}

// ConfigureIndex is used to [scale a pods-based index] up or down by changing the size of the pods or the number of
// replicas.
//
Expand Down Expand Up @@ -794,12 +812,15 @@ func (c *Client) DeleteIndex(ctx context.Context, idxName string) error {
//
// [scale a pods-based index]: https://docs.pinecone.io/guides/indexes/configure-pod-based-indexes
// [app.pinecone.io]: https://app.pinecone.io
func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *string,
replicas *int32) (*Index, error) {
func (c *Client) ConfigureIndex(ctx context.Context, name string, configParams ConfigureIndexParams) (*Index, error) {

if podType == nil && replicas == nil {
return nil, fmt.Errorf("must specify either podType or replicas")
}
// if configParams.PodType == "" && configParams.Replicas == 0 && configParams.DeletionProtection == "" {
// return nil, fmt.Errorf("must specify either PodType, Replicas, or DeletionProtection")
// }

podType := pointerOrNil(configParams.PodType)
replicas := pointerOrNil(configParams.Replicas)
deletionProtection := pointerOrNil(configParams.DeletionProtection)

request := control.ConfigureIndexRequest{
Spec: &struct {
Expand All @@ -816,6 +837,7 @@ func (c *Client) ConfigureIndex(ctx context.Context, name string, podType *strin
Replicas: replicas,
},
},
DeletionProtection: deletionProtection,
}

res, err := c.restClient.ConfigureIndex(ctx, name, request)
Expand Down Expand Up @@ -1153,13 +1175,16 @@ func toIndex(idx *control.IndexModel) *Index {
Ready: idx.Status.Ready,
State: IndexStatusState(idx.Status.State),
}
deletionProtecetion := toDeletionProtection(string(derefOrDefault(idx.DeletionProtection, control.Disabled)))

return &Index{
Name: idx.Name,
Dimension: idx.Dimension,
Host: idx.Host,
Metric: IndexMetric(idx.Metric),
Spec: spec,
Status: status,
Name: idx.Name,
Dimension: idx.Dimension,
Host: idx.Host,
Metric: IndexMetric(idx.Metric),
DeletionProtection: deletionProtecetion,
Spec: spec,
Status: status,
}
}

Expand Down Expand Up @@ -1303,6 +1328,15 @@ func buildClientBaseOptions(in NewClientBaseParams) []control.ClientOption {
return clientOptions
}

func toDeletionProtection(value string) control.DeletionProtection {
switch control.DeletionProtection(value) {
case control.Enabled, control.Disabled:
return control.DeletionProtection(value)
default:
return control.Disabled
}
}

func ensureURLScheme(inputURL string) (string, error) {
parsedURL, err := url.Parse(inputURL)
if err != nil {
Expand All @@ -1324,6 +1358,14 @@ func valueOrFallback[T comparable](value, fallback T) T {
}
}

func pointerOrNil[T comparable](value T) *T {
var zero T // set to zero-value of generic type T
if value == zero {
return nil
}
return &value
}

func derefOrDefault[T any](ptr *T, defaultValue T) T {
if ptr == nil {
return defaultValue
Expand Down
65 changes: 36 additions & 29 deletions pinecone/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -482,9 +482,7 @@ func (ts *ClientTestsIntegration) TestConfigureIndexIllegalScaleDown() {
log.Fatalf("Error creating index %s: %v", name, err)
}

pods := "p1.x1" // test index originally created with "p1.x2" pods
replicas := int32(1) // could be nil, but do not want to test nil case here
_, err = ts.client.ConfigureIndex(context.Background(), name, &pods, &replicas)
_, err = ts.client.ConfigureIndex(context.Background(), name, ConfigureIndexParams{PodType: "p1.x1", Replicas: 1})
require.ErrorContainsf(ts.T(), err, "Cannot scale down", err.Error())
}

Expand All @@ -507,8 +505,7 @@ func (ts *ClientTestsIntegration) TestConfigureIndexScaleUpNoPods() {
log.Fatalf("Error creating index %s: %v", name, err)
}

replicas := int32(2)
_, err = ts.client.ConfigureIndex(context.Background(), name, nil, &replicas)
_, err = ts.client.ConfigureIndex(context.Background(), name, ConfigureIndexParams{Replicas: 2})
require.NoError(ts.T(), err)
}

Expand All @@ -531,12 +528,11 @@ func (ts *ClientTestsIntegration) TestConfigureIndexScaleUpNoReplicas() {
log.Fatalf("Error creating index %s: %v", name, err)
}

pods := "p1.x4"
_, err = ts.client.ConfigureIndex(context.Background(), name, &pods, nil)
_, err = ts.client.ConfigureIndex(context.Background(), name, ConfigureIndexParams{PodType: "p1.x4"})
require.NoError(ts.T(), err)
}

func (ts *ClientTestsIntegration) TestConfigureIndexIllegalNoPodsOrReplicas() {
func (ts *ClientTestsIntegration) TestConfigureIndexIllegalNoPodsOrReplicasOrDeletionProtection() {
name := uuid.New().String()

defer func(ts *ClientTestsIntegration, name string) {
Expand All @@ -555,8 +551,8 @@ func (ts *ClientTestsIntegration) TestConfigureIndexIllegalNoPodsOrReplicas() {
log.Fatalf("Error creating index %s: %v", name, err)
}

_, err = ts.client.ConfigureIndex(context.Background(), name, nil, nil)
require.ErrorContainsf(ts.T(), err, "must specify either podType or replicas", err.Error())
_, err = ts.client.ConfigureIndex(context.Background(), name, ConfigureIndexParams{})
require.ErrorContainsf(ts.T(), err, "must specify either PodType, Replicas, or DeletionProtection", err.Error())
}

func (ts *ClientTestsIntegration) TestConfigureIndexHitPodLimit() {
Expand All @@ -578,8 +574,7 @@ func (ts *ClientTestsIntegration) TestConfigureIndexHitPodLimit() {
log.Fatalf("Error creating index %s: %v", name, err)
}

replicas := int32(30000)
_, err = ts.client.ConfigureIndex(context.Background(), name, nil, &replicas)
_, err = ts.client.ConfigureIndex(context.Background(), name, ConfigureIndexParams{Replicas: 30000})
require.ErrorContainsf(ts.T(), err, "You've reached the max pods allowed", err.Error())
}

Expand Down Expand Up @@ -853,6 +848,9 @@ func TestEnsureURLSchemeUnit(t *testing.T) {
}

func TestToIndexUnit(t *testing.T) {
deletionProtectionEnabled := control.Enabled
deletionProtectionDisabled := control.Disabled

tests := []struct {
name string
originalInput *control.IndexModel
Expand All @@ -866,10 +864,11 @@ func TestToIndexUnit(t *testing.T) {
{
name: "pod index input",
originalInput: &control.IndexModel{
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
DeletionProtection: &deletionProtectionDisabled,
Spec: struct {
Pod *control.PodSpec `json:"pod,omitempty"`
Serverless *control.ServerlessSpec `json:"serverless,omitempty"`
Expand All @@ -894,10 +893,11 @@ func TestToIndexUnit(t *testing.T) {
},
},
expectedOutput: &Index{
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
DeletionProtection: "disabled",
Spec: &IndexSpec{
Pod: &PodSpec{
Environment: "test-environ",
Expand All @@ -917,10 +917,11 @@ func TestToIndexUnit(t *testing.T) {
{
name: "serverless index input",
originalInput: &control.IndexModel{
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
DeletionProtection: &deletionProtectionEnabled,
Spec: struct {
Pod *control.PodSpec `json:"pod,omitempty"`
Serverless *control.ServerlessSpec `json:"serverless,omitempty"`
Expand All @@ -940,10 +941,11 @@ func TestToIndexUnit(t *testing.T) {
},
},
expectedOutput: &Index{
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
Name: "testIndex",
Dimension: 128,
Host: "test-host",
Metric: "cosine",
DeletionProtection: "enabled",
Spec: &IndexSpec{
Serverless: &ServerlessSpec{
Cloud: Cloud("test-environ"),
Expand Down Expand Up @@ -1238,7 +1240,12 @@ func TestBuildClientBaseOptionsUnit(t *testing.T) {
}
}

// Helper functions:
// Helper functions

func stringPtr(s string) *string {
return &s
}

func isValidUUID(u string) bool {
_, err := uuid.Parse(u)
return err == nil
Expand Down
18 changes: 11 additions & 7 deletions pinecone/models.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package pinecone

import "google.golang.org/protobuf/types/known/structpb"
import (
"github.com/pinecone-io/go-pinecone/internal/gen/control"
"google.golang.org/protobuf/types/known/structpb"
)

// IndexMetric is the [distance metric] to be used by similarity search against a Pinecone Index.
//
Expand Down Expand Up @@ -52,12 +55,13 @@ type IndexSpec struct {

// Index is a Pinecone Index object. Can be either a pod-based or a serverless Index, depending on the IndexSpec.
type Index struct {
Name string `json:"name"`
Dimension int32 `json:"dimension"`
Host string `json:"host"`
Metric IndexMetric `json:"metric"`
Spec *IndexSpec `json:"spec,omitempty"`
Status *IndexStatus `json:"status,omitempty"`
Name string `json:"name"`
Dimension int32 `json:"dimension"`
Host string `json:"host"`
Metric IndexMetric `json:"metric"`
DeletionProtection control.DeletionProtection `json:"deletion_protection,omitempty"`
Spec *IndexSpec `json:"spec,omitempty"`
Status *IndexStatus `json:"status,omitempty"`
}

// Collection is a Pinecone [Collection object]. Only available for pod-based Indexes.
Expand Down

0 comments on commit f12a9b4

Please sign in to comment.