From 27ca6d68599b8a85c49e433c823134f6cba895d3 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 21 Apr 2021 17:20:11 -0700 Subject: [PATCH 01/58] WIP: add reservation apis Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- go.mod | 3 +-- go.sum | 4 ++-- pkg/rpc/datacatalogservice/service.go | 13 +++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 920f6ace..3e66f05a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/Selvatico/go-mocket v1.0.7 - github.com/flyteorg/flyteidl v0.18.17 + github.com/flyteorg/flyteidl v0.18.38 github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 @@ -15,5 +15,4 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 google.golang.org/grpc v1.36.0 - google.golang.org/protobuf v1.25.0 ) diff --git a/go.sum b/go.sum index 087a825d..254a9d87 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.18.15 h1:sXrlwTRaRjQsXYMNrY/S930SKdKtu4XnpNFEu8I4tn4= -github.com/flyteorg/flyteidl v0.18.15/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= +github.com/flyteorg/flyteidl v0.18.38 h1:XgAw9d2Q/UjWQyXbnZz/j4N6OVGDxr7jceden6PdCgY= +github.com/flyteorg/flyteidl v0.18.38/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= github.com/flyteorg/flytestdlib v0.3.13 h1:5ioA/q3ixlyqkFh5kDaHgmPyTP/AHtqq1K/TIbVLUzM= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index a8d1125c..e37d2020 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -23,6 +23,7 @@ type DataCatalogService struct { DatasetManager interfaces.DatasetManager ArtifactManager interfaces.ArtifactManager TagManager interfaces.TagManager + ReservationManager interfaces.ReservationManager } func (s *DataCatalogService) CreateDataset(ctx context.Context, request *catalog.CreateDatasetRequest) (*catalog.CreateDatasetResponse, error) { @@ -53,6 +54,18 @@ func (s *DataCatalogService) ListDatasets(ctx context.Context, request *catalog. return s.DatasetManager.ListDatasets(ctx, request) } +func (s *DataCatalogService) GetOrReserveArtifact(ctx context.Context, request *catalog.GetOrReserveArtifactRequest) (*catalog.GetOrReserveArtifactResponse, error) { + return s.ReservationManager.GetOrReserveArtifact(ctx, request) +} + +func (s *DataCatalogService) ExtendReservation(ctx context.Context, request *catalog.ExtendReservationRequest) (*catalog.ExtendReservationResponse, error) { + return s.ReservationManager.ExtendReservation(ctx, request) +} + +func (s *DataCatalogService) ReleaseReservation(ctx context.Context, request *catalog.ReleaseReservationRequest) (*catalog.ReleaseReservationResponse, error) { + return s.ReservationManager.ReleaseReservation(ctx, request) +} + func NewDataCatalogService() *DataCatalogService { configProvider := runtime.NewConfigurationProvider() dataCatalogConfig := configProvider.ApplicationConfiguration().GetDataCatalogConfig() From 03fce3d95a267e80f408bf4fee6fedf8b1a0a1ff Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 21 Apr 2021 17:21:27 -0700 Subject: [PATCH 02/58] add missing files Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/interfaces/reservation.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 pkg/manager/interfaces/reservation.go diff --git a/pkg/manager/interfaces/reservation.go b/pkg/manager/interfaces/reservation.go new file mode 100644 index 00000000..a44775b7 --- /dev/null +++ b/pkg/manager/interfaces/reservation.go @@ -0,0 +1,13 @@ +package interfaces + +import ( + "context" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" +) + +type ReservationManager interface { + GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) + ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) + ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) +} \ No newline at end of file From bce5b9ca6e5a52248456a9f8a34e75071319167f Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Fri, 23 Apr 2021 15:01:50 -0700 Subject: [PATCH 03/58] added create DAO Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- .../lyft/golang_support_tools/tools.go | 2 +- pkg/manager/impl/reservation_manager.go | 28 ++++++++++ pkg/manager/interfaces/reservation.go | 2 +- pkg/repositories/gormimpl/reservation.go | 47 +++++++++++++++++ pkg/repositories/gormimpl/reservation_test.go | 52 +++++++++++++++++++ pkg/repositories/handle.go | 1 + .../interfaces/reservation_repo.go | 14 +++++ pkg/repositories/models/reservation.go | 19 +++++++ pkg/rpc/datacatalogservice/service.go | 13 ++--- 9 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 pkg/manager/impl/reservation_manager.go create mode 100644 pkg/repositories/gormimpl/reservation.go create mode 100644 pkg/repositories/gormimpl/reservation_test.go create mode 100644 pkg/repositories/interfaces/reservation_repo.go create mode 100644 pkg/repositories/models/reservation.go diff --git a/boilerplate/lyft/golang_support_tools/tools.go b/boilerplate/lyft/golang_support_tools/tools.go index 4310b39d..88ff6452 100644 --- a/boilerplate/lyft/golang_support_tools/tools.go +++ b/boilerplate/lyft/golang_support_tools/tools.go @@ -3,8 +3,8 @@ package tools import ( + _ "github.com/alvaroloes/enumer" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/lyft/flytestdlib/cli/pflags" _ "github.com/vektra/mockery/cmd/mockery" - _ "github.com/alvaroloes/enumer" ) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go new file mode 100644 index 00000000..95c57739 --- /dev/null +++ b/pkg/manager/impl/reservation_manager.go @@ -0,0 +1,28 @@ +package impl + +import ( + "context" + "errors" + + "github.com/flyteorg/datacatalog/pkg/manager/interfaces" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" +) + +type reservationManager struct{} + +func NewReservationManager() interfaces.ReservationManager { + return &reservationManager{} +} + +func (r *reservationManager) GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { + return nil, errors.New("not implemented") +} + +func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { + return nil, errors.New("not implemented") +} + +func (r *reservationManager) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/manager/interfaces/reservation.go b/pkg/manager/interfaces/reservation.go index a44775b7..e89419a0 100644 --- a/pkg/manager/interfaces/reservation.go +++ b/pkg/manager/interfaces/reservation.go @@ -10,4 +10,4 @@ type ReservationManager interface { GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) -} \ No newline at end of file +} diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go new file mode 100644 index 00000000..06d662d0 --- /dev/null +++ b/pkg/repositories/gormimpl/reservation.go @@ -0,0 +1,47 @@ +package gormimpl + +import ( + "context" + "errors" + + errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" + errors3 "github.com/flyteorg/datacatalog/pkg/repositories/errors" + "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" + "github.com/flyteorg/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/jinzhu/gorm" + + "time" +) + +type reservationRepo struct { + db *gorm.DB + repoMetrics gormMetrics + errorTransformer errors2.ErrorTransformer +} + +func NewReservationRepo(db *gorm.DB, errorTransformer errors3.ErrorTransformer, scope promutils.Scope) interfaces.ReservationRepo { + return &reservationRepo{ + db: db, + errorTransformer: errorTransformer, + repoMetrics: newGormMetrics(scope), + } +} + +func (r *reservationRepo) Create(ctx context.Context, reservation models.Reservation) error { + timer := r.repoMetrics.CreateDuration.Start(ctx) + defer timer.Stop() + result := r.db.Create(reservation) + if result.Error != nil { + return r.errorTransformer.ToDataCatalogError(result.Error) + } + return nil +} + +func (r *reservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { + return models.Reservation{}, errors.New("not implemented") +} + +func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go new file mode 100644 index 00000000..433d682b --- /dev/null +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -0,0 +1,52 @@ +package gormimpl + +import ( + "context" + "database/sql/driver" + "testing" + "time" + + mocket "github.com/Selvatico/go-mocket" + "github.com/flyteorg/datacatalog/pkg/repositories/errors" + "github.com/flyteorg/datacatalog/pkg/repositories/models" + "github.com/flyteorg/datacatalog/pkg/repositories/utils" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/stretchr/testify/assert" +) + +func TestCreate(t *testing.T) { + reservation := getReservation() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + reservationCreated := false + GlobalMock.NewMock().WithQuery( + `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?,?)`).WithCallback( + func(s string, values []driver.NamedValue) { + reservationCreated = true + }, + ) + + reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + + err := reservationRepo.Create(context.Background(), reservation) + assert.Nil(t, err) + + assert.True(t, reservationCreated) +} + +func getReservation() models.Reservation { + reservation := models.Reservation{ + ReservationKey: models.ReservationKey{ + DatasetProject: "testProject", + DatasetName: "testDataset", + DatasetDomain: "testDomain", + DatasetVersion: "testVersion", + TagName: "testTag", + }, + OwnerID: "batman", + ExpireAt: time.Time{}, + } + return reservation +} diff --git a/pkg/repositories/handle.go b/pkg/repositories/handle.go index 9c142c24..d11fbd34 100644 --- a/pkg/repositories/handle.go +++ b/pkg/repositories/handle.go @@ -76,6 +76,7 @@ func (h *DBHandle) Migrate() { h.db.AutoMigrate(&models.Tag{}) h.db.AutoMigrate(&models.PartitionKey{}) h.db.AutoMigrate(&models.Partition{}) + h.db.AutoMigrate(&models.Reservation{}) } func (h *DBHandle) Close() error { diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go new file mode 100644 index 00000000..0b2ec8d5 --- /dev/null +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "context" + "time" + + "github.com/flyteorg/datacatalog/pkg/repositories/models" +) + +type ReservationRepo interface { + Create(ctx context.Context, reservation models.Reservation) error + Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) + Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time) (int, error) +} diff --git a/pkg/repositories/models/reservation.go b/pkg/repositories/models/reservation.go new file mode 100644 index 00000000..d0e580bb --- /dev/null +++ b/pkg/repositories/models/reservation.go @@ -0,0 +1,19 @@ +package models + +import "time" + +type ReservationKey struct { + DatasetProject string `gorm:"primary_key"` + DatasetName string `gorm:"primary_key"` + DatasetDomain string `gorm:"primary_key"` + DatasetVersion string `gorm:"primary_key"` + TagName string `gorm:"primary_key"` +} + +type Reservation struct { + BaseModel + ReservationKey + OwnerID string + ExpireAt time.Time + SerializedMetadata []byte +} diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index e37d2020..031a80b9 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -20,9 +20,9 @@ import ( ) type DataCatalogService struct { - DatasetManager interfaces.DatasetManager - ArtifactManager interfaces.ArtifactManager - TagManager interfaces.TagManager + DatasetManager interfaces.DatasetManager + ArtifactManager interfaces.ArtifactManager + TagManager interfaces.TagManager ReservationManager interfaces.ReservationManager } @@ -120,8 +120,9 @@ func NewDataCatalogService() *DataCatalogService { }() return &DataCatalogService{ - DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), - ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), - TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), + DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), + ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), + TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), + ReservationManager: impl.NewReservationManager(), } } From ca1b6dd831292df26f93e4f3be4a2481b7e6d223 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Fri, 23 Apr 2021 17:32:32 -0700 Subject: [PATCH 04/58] Add get dao Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 46 ++++++++- pkg/repositories/factory.go | 1 + pkg/repositories/gormimpl/reservation.go | 17 +++- pkg/repositories/gormimpl/reservation_test.go | 95 +++++++++++++++++-- pkg/repositories/interfaces/base.go | 1 + .../interfaces/reservation_repo.go | 2 +- pkg/repositories/postgres_repo.go | 6 ++ pkg/repositories/transformers/reservation.go | 16 ++++ 8 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 pkg/repositories/transformers/reservation.go diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 95c57739..8d262a6b 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -3,20 +3,60 @@ package impl import ( "context" "errors" + errors2 "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/flyteorg/datacatalog/pkg/repositories" + "github.com/flyteorg/datacatalog/pkg/repositories/transformers" "github.com/flyteorg/datacatalog/pkg/manager/interfaces" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" ) -type reservationManager struct{} +type reservationManager struct{ + repo repositories.RepositoryInterface +} func NewReservationManager() interfaces.ReservationManager { return &reservationManager{} } -func (r *reservationManager) GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { - return nil, errors.New("not implemented") +func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { + tagKey := transformers.ToTagKey(request.DatasetId, request.TagName) + tag, err := r.repo.TagRepo().Get(ctx, tagKey) + if err != nil { + if errors2.IsDoesNotExistError(err) { + // Tag does not exist yet, let's reserve a spot to work on + // generating the artifact. + r.makeReservation(ctx, request) + } + return nil, err + } + + artifact, err := transformers.FromArtifactModel(tag.Artifact) + if err != nil { + return nil, err + } + + return &datacatalog.GetOrReserveArtifactResponse{ + Value: &datacatalog.GetOrReserveArtifactResponse_Artifact{ + Artifact: artifact, + }, + }, nil +} + +func (r *reservationManager) makeReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) { + _, err := r.repo.ReservationRepo().Get(ctx, transformers.ToReservationKey( + *request.DatasetId, + request.TagName, + )) + if err != nil { + if errors2.IsDoesNotExistError(err) { + // Reservation does not exist yet, let's create one + } + } + + // Reservation already exists so there is a task already start working on it + // Let's check if the reservation is expired. } func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { diff --git a/pkg/repositories/factory.go b/pkg/repositories/factory.go index 46dacc69..ae234a18 100644 --- a/pkg/repositories/factory.go +++ b/pkg/repositories/factory.go @@ -26,6 +26,7 @@ type RepositoryInterface interface { DatasetRepo() interfaces.DatasetRepo ArtifactRepo() interfaces.ArtifactRepo TagRepo() interfaces.TagRepo + ReservationRepo() interfaces.ReservationRepo } func GetRepository(repoType RepoConfig, dbConfig config.DbConfig, scope promutils.Scope) RepositoryInterface { diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 06d662d0..68d48e44 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -39,9 +39,22 @@ func (r *reservationRepo) Create(ctx context.Context, reservation models.Reserva } func (r *reservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { - return models.Reservation{}, errors.New("not implemented") + timer := r.repoMetrics.GetDuration.Start(ctx) + defer timer.Stop() + + var reservation models.Reservation + + result := r.db.Where(&models.Reservation{ + ReservationKey: reservationKey, + }).First(&reservation) + + if result.Error != nil { + return reservation, r.errorTransformer.ToDataCatalogError(result.Error) + } + + return reservation, nil } -func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time) (int, error) { +func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, ownerID string) (int, error) { return 0, errors.New("not implemented") } diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 433d682b..df6dbdcd 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -3,6 +3,9 @@ package gormimpl import ( "context" "database/sql/driver" + apiErrors "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/jinzhu/gorm" + "google.golang.org/grpc/codes" "testing" "time" @@ -36,17 +39,93 @@ func TestCreate(t *testing.T) { assert.True(t, reservationCreated) } +func TestCreateAlreadyExists(t *testing.T) { + reservation := getReservation() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery( + `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?,?)`).WithError( + getAlreadyExistsErr(), + ) + + reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + + err := reservationRepo.Create(context.Background(), reservation) + assert.NotNil(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.AlreadyExists) +} + +func TestGet(t *testing.T) { + expectedReservation := getReservation() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery( + `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) ORDER BY "reservations"."dataset_project" ASC LIMIT 1`, + ).WithReply(getDBResponse(expectedReservation)) + + reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + reservation, err := reservationRepo.Get(context.Background(), expectedReservation.ReservationKey) + assert.Nil(t, err) + assert.Equal(t, expectedReservation.DatasetProject, reservation.DatasetProject) + assert.Equal(t, expectedReservation.DatasetDomain, reservation.DatasetDomain) + assert.Equal(t, expectedReservation.DatasetName, reservation.DatasetName) + assert.Equal(t, expectedReservation.DatasetVersion, reservation.DatasetVersion) + assert.Equal(t, expectedReservation.TagName, reservation.TagName) + assert.Equal(t, expectedReservation.ExpireAt, reservation.ExpireAt) +} + +func TestGetNotFound(t *testing.T) { + expectedReservation := getReservation() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithError(gorm.ErrRecordNotFound) + + reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + _, err := reservationRepo.Get(context.Background(), expectedReservation.ReservationKey) + assert.Error(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.NotFound) + +} + +func getDBResponse(reservation models.Reservation) []map[string]interface{} { + return []map[string]interface{}{ + { + "dataset_project": reservation.DatasetProject, + "dataset_name": reservation.DatasetName, + "dataset_domain": reservation.DatasetDomain, + "dataset_version": reservation.DatasetVersion, + "tag_name": reservation.TagName, + "owner_id": reservation.OwnerID, + "expire_at": reservation.ExpireAt, + }, + } +} + +func getReservationKey() models.ReservationKey { + return models.ReservationKey{ + DatasetProject: "testProject", + DatasetName: "testDataset", + DatasetDomain: "testDomain", + DatasetVersion: "testVersion", + TagName: "testTag", + } +} + func getReservation() models.Reservation { reservation := models.Reservation{ - ReservationKey: models.ReservationKey{ - DatasetProject: "testProject", - DatasetName: "testDataset", - DatasetDomain: "testDomain", - DatasetVersion: "testVersion", - TagName: "testTag", - }, + ReservationKey: getReservationKey(), OwnerID: "batman", - ExpireAt: time.Time{}, + ExpireAt: time.Unix(1,1), } return reservation } diff --git a/pkg/repositories/interfaces/base.go b/pkg/repositories/interfaces/base.go index 864d724d..4aad6955 100644 --- a/pkg/repositories/interfaces/base.go +++ b/pkg/repositories/interfaces/base.go @@ -4,4 +4,5 @@ type DataCatalogRepo interface { DatasetRepo() DatasetRepo ArtifactRepo() ArtifactRepo TagRepo() TagRepo + ReservationRepo() ReservationRepo } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 0b2ec8d5..a3ea5ace 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -10,5 +10,5 @@ import ( type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time) (int, error) + Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int, error) } diff --git a/pkg/repositories/postgres_repo.go b/pkg/repositories/postgres_repo.go index 7b016f67..14453c87 100644 --- a/pkg/repositories/postgres_repo.go +++ b/pkg/repositories/postgres_repo.go @@ -12,6 +12,7 @@ type PostgresRepo struct { datasetRepo interfaces.DatasetRepo artifactRepo interfaces.ArtifactRepo tagRepo interfaces.TagRepo + reservationRepo interfaces.ReservationRepo } func (dc *PostgresRepo) DatasetRepo() interfaces.DatasetRepo { @@ -26,10 +27,15 @@ func (dc *PostgresRepo) TagRepo() interfaces.TagRepo { return dc.tagRepo } +func (dc *PostgresRepo) ReservationRepo() interfaces.ReservationRepo { + return dc.reservationRepo +} + func NewPostgresRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.DataCatalogRepo { return &PostgresRepo{ datasetRepo: gormimpl.NewDatasetRepo(db, errorTransformer, scope.NewSubScope("dataset")), artifactRepo: gormimpl.NewArtifactRepo(db, errorTransformer, scope.NewSubScope("artifact")), tagRepo: gormimpl.NewTagRepo(db, errorTransformer, scope.NewSubScope("tag")), + reservationRepo: gormimpl.NewReservationRepo(db, errorTransformer, scope.NewSubScope("reservation")), } } diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go new file mode 100644 index 00000000..f0c4705d --- /dev/null +++ b/pkg/repositories/transformers/reservation.go @@ -0,0 +1,16 @@ +package transformers + +import ( + "github.com/flyteorg/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" +) + +func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.ReservationKey { + return models.ReservationKey{ + DatasetProject: datasetID.Project, + DatasetName: datasetID.Name, + DatasetDomain: datasetID.Domain, + DatasetVersion: datasetID.Version, + TagName: tagName, + } +} From 20d1568ab4c0477b03e302ce7f39fda7fe0b3381 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 27 Apr 2021 12:03:49 -0700 Subject: [PATCH 05/58] wip Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 3 ++- pkg/repositories/gormimpl/metrics.go | 3 +++ pkg/repositories/gormimpl/reservation.go | 23 +++++++++++++++---- pkg/repositories/gormimpl/reservation_test.go | 15 ++++++------ .../interfaces/reservation_repo.go | 2 +- pkg/repositories/postgres_repo.go | 12 +++++----- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 8d262a6b..5673148c 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -3,6 +3,7 @@ package impl import ( "context" "errors" + errors2 "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories" "github.com/flyteorg/datacatalog/pkg/repositories/transformers" @@ -12,7 +13,7 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" ) -type reservationManager struct{ +type reservationManager struct { repo repositories.RepositoryInterface } diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index 1c154230..d4ba332d 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -13,6 +13,7 @@ type gormMetrics struct { CreateDuration labeled.StopWatch GetDuration labeled.StopWatch ListDuration labeled.StopWatch + UpdateDuration labeled.StopWatch } func newGormMetrics(scope promutils.Scope) gormMetrics { @@ -24,5 +25,7 @@ func newGormMetrics(scope promutils.Scope) gormMetrics { "get", "Duration for retrieving an entity ", time.Millisecond, scope), ListDuration: labeled.NewStopWatch( "list", "Duration for listing entities ", time.Millisecond, scope), + UpdateDuration: labeled.NewStopWatch( + "update", "Duration for updating entities ", time.Millisecond, scope), } } diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 68d48e44..6dce6bb2 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -2,7 +2,6 @@ package gormimpl import ( "context" - "errors" errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" errors3 "github.com/flyteorg/datacatalog/pkg/repositories/errors" @@ -45,7 +44,7 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva var reservation models.Reservation result := r.db.Where(&models.Reservation{ - ReservationKey: reservationKey, + ReservationKey: reservationKey, }).First(&reservation) if result.Error != nil { @@ -55,6 +54,22 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva return reservation, nil } -func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, ownerID string) (int, error) { - return 0, errors.New("not implemented") +func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expireAt time.Time, ownerID string) (int64, error) { + timer := r.repoMetrics.UpdateDuration.Start(ctx) + defer timer.Stop() + + result := r.db.Where( + &models.Reservation{ + ReservationKey: reservationKey, + }, + ).Updates( + models.Reservation{ + OwnerID: ownerID, + ExpireAt: expireAt, + }) + if result.Error != nil { + return 0, r.errorTransformer.ToDataCatalogError(result.Error) + } + + return result.RowsAffected, nil } diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index df6dbdcd..8a43d040 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -3,11 +3,12 @@ package gormimpl import ( "context" "database/sql/driver" + "testing" + "time" + apiErrors "github.com/flyteorg/datacatalog/pkg/errors" "github.com/jinzhu/gorm" "google.golang.org/grpc/codes" - "testing" - "time" mocket "github.com/Selvatico/go-mocket" "github.com/flyteorg/datacatalog/pkg/repositories/errors" @@ -67,7 +68,7 @@ func TestGet(t *testing.T) { GlobalMock.NewMock().WithQuery( `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) ORDER BY "reservations"."dataset_project" ASC LIMIT 1`, - ).WithReply(getDBResponse(expectedReservation)) + ).WithReply(getDBResponse(expectedReservation)) reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) reservation, err := reservationRepo.Get(context.Background(), expectedReservation.ReservationKey) @@ -105,8 +106,8 @@ func getDBResponse(reservation models.Reservation) []map[string]interface{} { "dataset_domain": reservation.DatasetDomain, "dataset_version": reservation.DatasetVersion, "tag_name": reservation.TagName, - "owner_id": reservation.OwnerID, - "expire_at": reservation.ExpireAt, + "owner_id": reservation.OwnerID, + "expire_at": reservation.ExpireAt, }, } } @@ -124,8 +125,8 @@ func getReservationKey() models.ReservationKey { func getReservation() models.Reservation { reservation := models.Reservation{ ReservationKey: getReservationKey(), - OwnerID: "batman", - ExpireAt: time.Unix(1,1), + OwnerID: "batman", + ExpireAt: time.Unix(1, 1), } return reservation } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index a3ea5ace..e0403acd 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -10,5 +10,5 @@ import ( type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int, error) + Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int64, error) } diff --git a/pkg/repositories/postgres_repo.go b/pkg/repositories/postgres_repo.go index 14453c87..e7dc1cd8 100644 --- a/pkg/repositories/postgres_repo.go +++ b/pkg/repositories/postgres_repo.go @@ -9,9 +9,9 @@ import ( ) type PostgresRepo struct { - datasetRepo interfaces.DatasetRepo - artifactRepo interfaces.ArtifactRepo - tagRepo interfaces.TagRepo + datasetRepo interfaces.DatasetRepo + artifactRepo interfaces.ArtifactRepo + tagRepo interfaces.TagRepo reservationRepo interfaces.ReservationRepo } @@ -33,9 +33,9 @@ func (dc *PostgresRepo) ReservationRepo() interfaces.ReservationRepo { func NewPostgresRepo(db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.DataCatalogRepo { return &PostgresRepo{ - datasetRepo: gormimpl.NewDatasetRepo(db, errorTransformer, scope.NewSubScope("dataset")), - artifactRepo: gormimpl.NewArtifactRepo(db, errorTransformer, scope.NewSubScope("artifact")), - tagRepo: gormimpl.NewTagRepo(db, errorTransformer, scope.NewSubScope("tag")), + datasetRepo: gormimpl.NewDatasetRepo(db, errorTransformer, scope.NewSubScope("dataset")), + artifactRepo: gormimpl.NewArtifactRepo(db, errorTransformer, scope.NewSubScope("artifact")), + tagRepo: gormimpl.NewTagRepo(db, errorTransformer, scope.NewSubScope("tag")), reservationRepo: gormimpl.NewReservationRepo(db, errorTransformer, scope.NewSubScope("reservation")), } } From e80137e70b90fcdcc7291c5cb418eea299f11808 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 27 Apr 2021 12:10:42 -0700 Subject: [PATCH 06/58] fix tests Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/artifact_manager_test.go | 1 + pkg/repositories/mocks/base.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/pkg/manager/impl/artifact_manager_test.go b/pkg/manager/impl/artifact_manager_test.go index eada92dd..39b92364 100644 --- a/pkg/manager/impl/artifact_manager_test.go +++ b/pkg/manager/impl/artifact_manager_test.go @@ -92,6 +92,7 @@ func newMockDataCatalogRepo() *mocks.DataCatalogRepo { return &mocks.DataCatalogRepo{ MockDatasetRepo: &mocks.DatasetRepo{}, MockArtifactRepo: &mocks.ArtifactRepo{}, + MockReservationRepo: &mocks.ReservationRepo{}, } } diff --git a/pkg/repositories/mocks/base.go b/pkg/repositories/mocks/base.go index 827b69be..4a5c9551 100644 --- a/pkg/repositories/mocks/base.go +++ b/pkg/repositories/mocks/base.go @@ -6,6 +6,7 @@ type DataCatalogRepo struct { MockDatasetRepo *DatasetRepo MockArtifactRepo *ArtifactRepo MockTagRepo *TagRepo + MockReservationRepo *ReservationRepo } func (m *DataCatalogRepo) DatasetRepo() interfaces.DatasetRepo { @@ -19,3 +20,7 @@ func (m *DataCatalogRepo) ArtifactRepo() interfaces.ArtifactRepo { func (m *DataCatalogRepo) TagRepo() interfaces.TagRepo { return m.MockTagRepo } + +func (m *DataCatalogRepo) ReservationRepo() interfaces.ReservationRepo { + return m.MockReservationRepo +} From 02114708a4f4ca5347f008ce68a14f1479cf8541 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 27 Apr 2021 17:00:31 -0700 Subject: [PATCH 07/58] wired reservation manager Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/artifact_manager_test.go | 4 +- pkg/manager/impl/reservation_manager.go | 72 ++++++++-- .../interfaces/reservation_repo.go | 2 +- pkg/repositories/mocks/base.go | 6 +- pkg/repositories/mocks/reservation.go | 128 ++++++++++++++++++ pkg/rpc/datacatalogservice/service.go | 3 +- pkg/runtime/configs/data_catalog_config.go | 7 +- 7 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 pkg/repositories/mocks/reservation.go diff --git a/pkg/manager/impl/artifact_manager_test.go b/pkg/manager/impl/artifact_manager_test.go index 39b92364..386e15a5 100644 --- a/pkg/manager/impl/artifact_manager_test.go +++ b/pkg/manager/impl/artifact_manager_test.go @@ -90,8 +90,8 @@ func getTestArtifact() *datacatalog.Artifact { func newMockDataCatalogRepo() *mocks.DataCatalogRepo { return &mocks.DataCatalogRepo{ - MockDatasetRepo: &mocks.DatasetRepo{}, - MockArtifactRepo: &mocks.ArtifactRepo{}, + MockDatasetRepo: &mocks.DatasetRepo{}, + MockArtifactRepo: &mocks.ArtifactRepo{}, MockReservationRepo: &mocks.ReservationRepo{}, } } diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 5673148c..6a2031fb 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -3,9 +3,11 @@ package impl import ( "context" "errors" + "time" errors2 "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories" + "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/datacatalog/pkg/repositories/transformers" "github.com/flyteorg/datacatalog/pkg/manager/interfaces" @@ -14,11 +16,14 @@ import ( ) type reservationManager struct { - repo repositories.RepositoryInterface + repo repositories.RepositoryInterface + reservationTimeout time.Duration } -func NewReservationManager() interfaces.ReservationManager { - return &reservationManager{} +func NewReservationManager(reservationTimeout time.Duration) interfaces.ReservationManager { + return &reservationManager{ + reservationTimeout: reservationTimeout, + } } func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { @@ -28,7 +33,18 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * if errors2.IsDoesNotExistError(err) { // Tag does not exist yet, let's reserve a spot to work on // generating the artifact. - r.makeReservation(ctx, request) + state, err := r.makeReservation(ctx, request) + if err != nil { + return nil, err + } + + return &datacatalog.GetOrReserveArtifactResponse{ + Value: &datacatalog.GetOrReserveArtifactResponse_ReservationStatus{ + ReservationStatus: &datacatalog.ReservationStatus{ + State: state, + }, + }, + }, nil } return nil, err } @@ -45,19 +61,53 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * }, nil } -func (r *reservationManager) makeReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) { - _, err := r.repo.ReservationRepo().Get(ctx, transformers.ToReservationKey( - *request.DatasetId, - request.TagName, - )) +func (r *reservationManager) makeReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { + repo := r.repo.ReservationRepo() + reservationKey := transformers.ToReservationKey(*request.DatasetId, request.TagName) + rsv, err := repo.Get(ctx, reservationKey) + if err != nil { if errors2.IsDoesNotExistError(err) { - // Reservation does not exist yet, let's create one + // Reservation does not exist yet so let's create one + err := repo.Create(ctx, models.Reservation{ + ReservationKey: reservationKey, + OwnerID: request.OwnerId, + ExpireAt: time.Now().Add(r.reservationTimeout), + }) + + if err != nil { + return datacatalog.ReservationStatus{}, err + } + + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, + }, nil } + return datacatalog.ReservationStatus{}, err } - // Reservation already exists so there is a task already start working on it + // Reservation already exists so there is a task already working on it // Let's check if the reservation is expired. + if rsv.ExpireAt.Before(time.Now()) { + // The reservation is expired, let's try to grab the reservation + rowsAffected, err := repo.Update(ctx, reservationKey, time.Now().Add(r.reservationTimeout), request.OwnerId) + if err != nil { + return datacatalog.ReservationStatus{}, err + } + + if rowsAffected > 0 { + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, + }, nil + } + } + + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: rsv.OwnerID, + }, nil } func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index e0403acd..9de8a45d 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -10,5 +10,5 @@ import ( type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int64, error) + Update(ctx context.Context, reservationKey models.ReservationKey, expireAt time.Time, OwnerID string) (int64, error) } diff --git a/pkg/repositories/mocks/base.go b/pkg/repositories/mocks/base.go index 4a5c9551..993fe79e 100644 --- a/pkg/repositories/mocks/base.go +++ b/pkg/repositories/mocks/base.go @@ -3,9 +3,9 @@ package mocks import "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" type DataCatalogRepo struct { - MockDatasetRepo *DatasetRepo - MockArtifactRepo *ArtifactRepo - MockTagRepo *TagRepo + MockDatasetRepo *DatasetRepo + MockArtifactRepo *ArtifactRepo + MockTagRepo *TagRepo MockReservationRepo *ReservationRepo } diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go new file mode 100644 index 00000000..0f4ddf97 --- /dev/null +++ b/pkg/repositories/mocks/reservation.go @@ -0,0 +1,128 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/datacatalog/pkg/repositories/models" + + time "time" +) + +// ReservationRepo is an autogenerated mock type for the ReservationRepo type +type ReservationRepo struct { + mock.Mock +} + +type ReservationRepo_Create struct { + *mock.Call +} + +func (_m ReservationRepo_Create) Return(_a0 error) *ReservationRepo_Create { + return &ReservationRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnCreate(ctx context.Context, reservation models.Reservation) *ReservationRepo_Create { + c := _m.On("Create", ctx, reservation) + return &ReservationRepo_Create{Call: c} +} + +func (_m *ReservationRepo) OnCreateMatch(matchers ...interface{}) *ReservationRepo_Create { + c := _m.On("Create", matchers...) + return &ReservationRepo_Create{Call: c} +} + +// Create provides a mock function with given fields: ctx, reservation +func (_m *ReservationRepo) Create(ctx context.Context, reservation models.Reservation) error { + ret := _m.Called(ctx, reservation) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.Reservation) error); ok { + r0 = rf(ctx, reservation) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type ReservationRepo_Get struct { + *mock.Call +} + +func (_m ReservationRepo_Get) Return(_a0 models.Reservation, _a1 error) *ReservationRepo_Get { + return &ReservationRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ReservationRepo) OnGet(ctx context.Context, reservationKey models.ReservationKey) *ReservationRepo_Get { + c := _m.On("Get", ctx, reservationKey) + return &ReservationRepo_Get{Call: c} +} + +func (_m *ReservationRepo) OnGetMatch(matchers ...interface{}) *ReservationRepo_Get { + c := _m.On("Get", matchers...) + return &ReservationRepo_Get{Call: c} +} + +// Get provides a mock function with given fields: ctx, reservationKey +func (_m *ReservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { + ret := _m.Called(ctx, reservationKey) + + var r0 models.Reservation + if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey) models.Reservation); ok { + r0 = rf(ctx, reservationKey) + } else { + r0 = ret.Get(0).(models.Reservation) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, models.ReservationKey) error); ok { + r1 = rf(ctx, reservationKey) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type ReservationRepo_Update struct { + *mock.Call +} + +func (_m ReservationRepo_Update) Return(_a0 int64, _a1 error) *ReservationRepo_Update { + return &ReservationRepo_Update{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) *ReservationRepo_Update { + c := _m.On("Update", ctx, reservationKey, expirationDate, OwnerID) + return &ReservationRepo_Update{Call: c} +} + +func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRepo_Update { + c := _m.On("Update", matchers...) + return &ReservationRepo_Update{Call: c} +} + +// Update provides a mock function with given fields: ctx, reservationKey, expirationDate, OwnerID +func (_m *ReservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int64, error) { + ret := _m.Called(ctx, reservationKey, expirationDate, OwnerID) + + var r0 int64 + if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey, time.Time, string) int64); ok { + r0 = rf(ctx, reservationKey, expirationDate, OwnerID) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, models.ReservationKey, time.Time, string) error); ok { + r1 = rf(ctx, reservationKey, expirationDate, OwnerID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 031a80b9..722e3980 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "runtime/debug" + "time" "github.com/flyteorg/datacatalog/pkg/manager/impl" "github.com/flyteorg/datacatalog/pkg/manager/interfaces" @@ -123,6 +124,6 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(), + ReservationManager: impl.NewReservationManager(time.Second * time.Duration(dataCatalogConfig.ReservationTimeoutSec)), } } diff --git a/pkg/runtime/configs/data_catalog_config.go b/pkg/runtime/configs/data_catalog_config.go index c7160dca..0bcfaf7e 100644 --- a/pkg/runtime/configs/data_catalog_config.go +++ b/pkg/runtime/configs/data_catalog_config.go @@ -4,7 +4,8 @@ package configs // This configuration is the base configuration to start admin type DataCatalogConfig struct { - StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` - MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` - ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` + StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` + MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` + ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` + ReservationTimeoutSec int `json:"reservationTimeoutSec" pflag:",How long do we hold the reservation"` } From 3c16c6649d82147c485398dad45ca09d159f7041 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 27 Apr 2021 17:13:56 -0700 Subject: [PATCH 08/58] add todos Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 18 ++++++++---------- pkg/manager/impl/reservation_manager_test.go | 11 +++++++++++ pkg/repositories/gormimpl/reservation_test.go | 4 ++++ 3 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 pkg/manager/impl/reservation_manager_test.go diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 6a2031fb..76cae2a4 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -33,16 +33,14 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * if errors2.IsDoesNotExistError(err) { // Tag does not exist yet, let's reserve a spot to work on // generating the artifact. - state, err := r.makeReservation(ctx, request) + status, err := r.makeReservation(ctx, request) if err != nil { return nil, err } return &datacatalog.GetOrReserveArtifactResponse{ Value: &datacatalog.GetOrReserveArtifactResponse_ReservationStatus{ - ReservationStatus: &datacatalog.ReservationStatus{ - State: state, - }, + ReservationStatus: &status, }, }, nil } @@ -80,8 +78,8 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac } return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, }, nil } return datacatalog.ReservationStatus{}, err @@ -98,15 +96,15 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac if rowsAffected > 0 { return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, }, nil } } return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: rsv.OwnerID, + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: rsv.OwnerID, }, nil } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go new file mode 100644 index 00000000..0c28e7e7 --- /dev/null +++ b/pkg/manager/impl/reservation_manager_test.go @@ -0,0 +1,11 @@ +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetOrReserveArtifact(t *testing.T) { + assert.FailNow(t, "TODO") +} diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 8a43d040..a19d95b2 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -98,6 +98,10 @@ func TestGetNotFound(t *testing.T) { } +func TestUpdate(t *testing.T) { + assert.FailNow(t, "not implemented yet") +} + func getDBResponse(reservation models.Reservation) []map[string]interface{} { return []map[string]interface{}{ { From 9783b009511ddeaa6c715df420299c77b354d18b Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 28 Apr 2021 12:05:57 -0700 Subject: [PATCH 09/58] add more tests Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 4 ++- pkg/manager/impl/reservation_manager_test.go | 3 +- pkg/repositories/gormimpl/reservation.go | 3 +- pkg/repositories/gormimpl/reservation_test.go | 32 ++++++++++++++++--- .../interfaces/reservation_repo.go | 2 +- pkg/repositories/mocks/reservation.go | 18 +++++------ 6 files changed, 43 insertions(+), 19 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 76cae2a4..22fbcc17 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -89,7 +89,9 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac // Let's check if the reservation is expired. if rsv.ExpireAt.Before(time.Now()) { // The reservation is expired, let's try to grab the reservation - rowsAffected, err := repo.Update(ctx, reservationKey, time.Now().Add(r.reservationTimeout), request.OwnerId) + rowsAffected, err := repo.Update(ctx, reservationKey, + rsv.ExpireAt, + time.Now().Add(r.reservationTimeout), request.OwnerId) if err != nil { return datacatalog.ReservationStatus{}, err } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 0c28e7e7..6e221a37 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -1,9 +1,8 @@ package impl import ( - "testing" - "github.com/stretchr/testify/assert" + "testing" ) func TestGetOrReserveArtifact(t *testing.T) { diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 6dce6bb2..6d896965 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -54,13 +54,14 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva return reservation, nil } -func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expireAt time.Time, ownerID string) (int64, error) { +func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, ownerID string) (int64, error) { timer := r.repoMetrics.UpdateDuration.Start(ctx) defer timer.Stop() result := r.db.Where( &models.Reservation{ ReservationKey: reservationKey, + ExpireAt: prevExpireAt, }, ).Updates( models.Reservation{ diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index a19d95b2..f4caba32 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -3,6 +3,7 @@ package gormimpl import ( "context" "database/sql/driver" + "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "testing" "time" @@ -32,7 +33,7 @@ func TestCreate(t *testing.T) { }, ) - reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + reservationRepo := getReservationRepo(t) err := reservationRepo.Create(context.Background(), reservation) assert.Nil(t, err) @@ -51,7 +52,7 @@ func TestCreateAlreadyExists(t *testing.T) { getAlreadyExistsErr(), ) - reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + reservationRepo := getReservationRepo(t) err := reservationRepo.Create(context.Background(), reservation) assert.NotNil(t, err) @@ -70,7 +71,7 @@ func TestGet(t *testing.T) { `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) ORDER BY "reservations"."dataset_project" ASC LIMIT 1`, ).WithReply(getDBResponse(expectedReservation)) - reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + reservationRepo := getReservationRepo(t) reservation, err := reservationRepo.Get(context.Background(), expectedReservation.ReservationKey) assert.Nil(t, err) assert.Equal(t, expectedReservation.DatasetProject, reservation.DatasetProject) @@ -89,7 +90,7 @@ func TestGetNotFound(t *testing.T) { GlobalMock.NewMock().WithError(gorm.ErrRecordNotFound) - reservationRepo := NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + reservationRepo := getReservationRepo(t) _, err := reservationRepo.Get(context.Background(), expectedReservation.ReservationKey) assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) @@ -99,7 +100,28 @@ func TestGetNotFound(t *testing.T) { } func TestUpdate(t *testing.T) { - assert.FailNow(t, "not implemented yet") + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery( + `UPDATE "" SET "expire_at" = ?, "owner_id" = ? WHERE ("reservations"."dataset_project" = ?) AND ("reservations"."dataset_name" = ?) AND ("reservations"."dataset_domain" = ?) AND ("reservations"."dataset_version" = ?) AND ("reservations"."tag_name" = ?) AND ("reservations"."expire_at" = ?)`, + ).WithRowsNum(1) + + reservationRepo := getReservationRepo(t) + + reservationKey := getReservationKey() + prevExpireAt := time.Now() + expireAt := prevExpireAt.Add(time.Second*50) + ownerID := "hello" + + rows, err := reservationRepo.Update(context.Background(), reservationKey, prevExpireAt, expireAt, ownerID) + + assert.Nil(t, err) + assert.Equal(t, rows, int64(1)) +} + +func getReservationRepo(t *testing.T) interfaces.ReservationRepo { + return NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) } func getDBResponse(reservation models.Reservation) []map[string]interface{} { diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 9de8a45d..2283571c 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -10,5 +10,5 @@ import ( type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - Update(ctx context.Context, reservationKey models.ReservationKey, expireAt time.Time, OwnerID string) (int64, error) + Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) (int64, error) } diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go index 0f4ddf97..e724ffb5 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation.go @@ -96,8 +96,8 @@ func (_m ReservationRepo_Update) Return(_a0 int64, _a1 error) *ReservationRepo_U return &ReservationRepo_Update{Call: _m.Call.Return(_a0, _a1)} } -func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) *ReservationRepo_Update { - c := _m.On("Update", ctx, reservationKey, expirationDate, OwnerID) +func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) *ReservationRepo_Update { + c := _m.On("Update", ctx, reservationKey, prevExpireAt, expireAt, OwnerID) return &ReservationRepo_Update{Call: c} } @@ -106,20 +106,20 @@ func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRe return &ReservationRepo_Update{Call: c} } -// Update provides a mock function with given fields: ctx, reservationKey, expirationDate, OwnerID -func (_m *ReservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, expirationDate time.Time, OwnerID string) (int64, error) { - ret := _m.Called(ctx, reservationKey, expirationDate, OwnerID) +// Update provides a mock function with given fields: ctx, reservationKey, prevExpireAt, expireAt, OwnerID +func (_m *ReservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) (int64, error) { + ret := _m.Called(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey, time.Time, string) int64); ok { - r0 = rf(ctx, reservationKey, expirationDate, OwnerID) + if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey, time.Time, time.Time, string) int64); ok { + r0 = rf(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) } else { r0 = ret.Get(0).(int64) } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.ReservationKey, time.Time, string) error); ok { - r1 = rf(ctx, reservationKey, expirationDate, OwnerID) + if rf, ok := ret.Get(1).(func(context.Context, models.ReservationKey, time.Time, time.Time, string) error); ok { + r1 = rf(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) } else { r1 = ret.Error(1) } From 0b6135ad9bb265d9896b314b647b3844f7d73f04 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 28 Apr 2021 17:23:34 -0700 Subject: [PATCH 10/58] add more tests Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 18 +- pkg/manager/impl/reservation_manager_test.go | 224 +++++++++++++++++- pkg/repositories/gormimpl/reservation_test.go | 23 +- pkg/rpc/datacatalogservice/service.go | 2 +- 4 files changed, 248 insertions(+), 19 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 22fbcc17..0c764b13 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -15,14 +15,23 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" ) +type NowFunc func() time.Time + type reservationManager struct { repo repositories.RepositoryInterface reservationTimeout time.Duration + now NowFunc } -func NewReservationManager(reservationTimeout time.Duration) interfaces.ReservationManager { +func NewReservationManager( + repo repositories.RepositoryInterface, + reservationTimeout time.Duration, + nowFunc NowFunc, +) interfaces.ReservationManager { return &reservationManager{ + repo: repo, reservationTimeout: reservationTimeout, + now: nowFunc, } } @@ -70,7 +79,7 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac err := repo.Create(ctx, models.Reservation{ ReservationKey: reservationKey, OwnerID: request.OwnerId, - ExpireAt: time.Now().Add(r.reservationTimeout), + ExpireAt: r.now().Add(r.reservationTimeout), }) if err != nil { @@ -85,13 +94,14 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac return datacatalog.ReservationStatus{}, err } + now := r.now() // Reservation already exists so there is a task already working on it // Let's check if the reservation is expired. - if rsv.ExpireAt.Before(time.Now()) { + if rsv.ExpireAt.Before(now) { // The reservation is expired, let's try to grab the reservation rowsAffected, err := repo.Update(ctx, reservationKey, rsv.ExpireAt, - time.Now().Add(r.reservationTimeout), request.OwnerId) + now.Add(r.reservationTimeout), request.OwnerId) if err != nil { return datacatalog.ReservationStatus{}, err } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 6e221a37..0315b379 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -1,10 +1,228 @@ package impl import ( - "github.com/stretchr/testify/assert" + "context" + "testing" + "time" + + errors2 "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/flyteorg/datacatalog/pkg/repositories/mocks" + "github.com/flyteorg/datacatalog/pkg/repositories/models" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/golang/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "google.golang.org/grpc/codes" ) -func TestGetOrReserveArtifact(t *testing.T) { - assert.FailNow(t, "TODO") +var tagName = "tag" +var project = "p" +var name = "n" +var domain = "d" +var version = "v" +var datasetID = datacatalog.DatasetID{ + Project: project, + Name: name, + Domain: domain, + Version: version, +} +var timeout = time.Second * 300 +var prevOwner = "prevOwner" +var currentOwner = "currentOwner" + +func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { + serializedMetadata, err := proto.Marshal(&datacatalog.Metadata{}) + assert.Nil(t, err) + expectedArtifact := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: "123", + }, + SerializedMetadata: serializedMetadata, + } + + dcRepo := getDatacatalogRepo() + + dcRepo.MockTagRepo.On("Get", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(tagKey models.TagKey) bool { + return tagKey.DatasetProject == datasetID.Project && + tagKey.DatasetName == datasetID.Name && + tagKey.DatasetDomain == datasetID.Domain && + tagKey.DatasetVersion == datasetID.Version && + tagKey.TagName == tagName + }), + ).Return(models.Tag{ + Artifact: expectedArtifact, + }, nil) + + reservationManager := NewReservationManager(&dcRepo, timeout, time.Now) + + req := datacatalog.GetOrReserveArtifactRequest{ + DatasetId: &datasetID, + TagName: tagName, + OwnerId: currentOwner, + } + + resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + assert.Nil(t, err) + artifact := resp.GetArtifact() + assert.NotNil(t, artifact) + assert.Equal(t, expectedArtifact.ArtifactKey.ArtifactID, artifact.Id) +} + +func getDatacatalogRepo() mocks.DataCatalogRepo { + return mocks.DataCatalogRepo{ + MockReservationRepo: &mocks.ReservationRepo{}, + MockTagRepo: &mocks.TagRepo{}, + } +} + +func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { + dcRepo := getDatacatalogRepo() + + setUpTagRepoGetNotFound(&dcRepo) + + dcRepo.MockReservationRepo.On("Get", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(key models.ReservationKey) bool { + return key.DatasetProject == datasetID.Project && + key.DatasetDomain == datasetID.Domain && + key.DatasetVersion == datasetID.Version && + key.DatasetName == datasetID.Name && + key.TagName == tagName + })).Return(models.Reservation{}, errors2.NewDataCatalogErrorf(codes.NotFound, "entry not found")) + + now := time.Now() + + dcRepo.MockReservationRepo.On("Create", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservation models.Reservation) bool { + return reservation.DatasetProject == datasetID.Project && + reservation.DatasetDomain == datasetID.Domain && + reservation.DatasetName == datasetID.Name && + reservation.DatasetVersion == datasetID.Version && + reservation.TagName == tagName && + reservation.OwnerID == currentOwner && + reservation.ExpireAt == now.Add(timeout) + }), + ).Return(nil) + + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + + req := datacatalog.GetOrReserveArtifactRequest{ + DatasetId: &datasetID, + TagName: tagName, + OwnerId: currentOwner, + } + + resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + + assert.Nil(t, err) + assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) +} + +func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { + dcRepo := getDatacatalogRepo() + + setUpTagRepoGetNotFound(&dcRepo) + + now := time.Now() + prevExpireAt := now.Truncate(timeout + time.Second*10) + + setUpReservationRepoGet(&dcRepo, prevExpireAt) + + dcRepo.MockReservationRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(key models.ReservationKey) bool { + return key.DatasetProject == datasetID.Project && + key.DatasetDomain == datasetID.Domain && + key.DatasetVersion == datasetID.Version && + key.DatasetName == datasetID.Name && + key.TagName == tagName + }), + mock.MatchedBy(func(expireAt time.Time) bool { + return expireAt.Equal(prevExpireAt) + }), + mock.MatchedBy(func(expireAt time.Time) bool { + return expireAt.Equal(now.Add(timeout)) + }), + mock.MatchedBy(func(ownerID string) bool { + return ownerID == currentOwner + })).Return(int64(1), nil) + + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + + req := datacatalog.GetOrReserveArtifactRequest{ + DatasetId: &datasetID, + TagName: tagName, + OwnerId: currentOwner, + } + + resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + + assert.Nil(t, err) + assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) +} + +func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpireAt time.Time) { + dcRepo.MockReservationRepo.On("Get", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(key models.ReservationKey) bool { + return key.DatasetProject == datasetID.Project && + key.DatasetDomain == datasetID.Domain && + key.DatasetVersion == datasetID.Version && + key.DatasetName == datasetID.Name && + key.TagName == tagName + })).Return( + models.Reservation{ + ReservationKey: getReservationKey(), + OwnerID: prevOwner, + ExpireAt: prevExpireAt, + }, nil, + ) +} + +func setUpTagRepoGetNotFound(dcRepo *mocks.DataCatalogRepo) { + dcRepo.MockTagRepo.On("Get", + mock.Anything, + mock.Anything, + ).Return(models.Tag{}, errors2.NewDataCatalogErrorf(codes.NotFound, "entry not found")) +} + +func getReservationKey() models.ReservationKey { + return models.ReservationKey{ + DatasetProject: project, + DatasetName: name, + DatasetDomain: domain, + DatasetVersion: version, + TagName: tagName, + } +} + +func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { + dcRepo := getDatacatalogRepo() + + setUpTagRepoGetNotFound(&dcRepo) + + now := time.Now() + prevExpireAt := now.Add(time.Second * 10) + + setUpReservationRepoGet(&dcRepo, prevExpireAt) + + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + + req := datacatalog.GetOrReserveArtifactRequest{ + DatasetId: &datasetID, + TagName: tagName, + OwnerId: currentOwner, + } + + resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + + assert.Nil(t, err) + assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, resp.GetReservationStatus().State) } diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index f4caba32..8077c624 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -3,10 +3,11 @@ package gormimpl import ( "context" "database/sql/driver" - "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "testing" "time" + "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" + apiErrors "github.com/flyteorg/datacatalog/pkg/errors" "github.com/jinzhu/gorm" "google.golang.org/grpc/codes" @@ -20,7 +21,7 @@ import ( ) func TestCreate(t *testing.T) { - reservation := getReservation() + reservation := GetReservation() GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true @@ -42,7 +43,7 @@ func TestCreate(t *testing.T) { } func TestCreateAlreadyExists(t *testing.T) { - reservation := getReservation() + reservation := GetReservation() GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true @@ -62,7 +63,7 @@ func TestCreateAlreadyExists(t *testing.T) { } func TestGet(t *testing.T) { - expectedReservation := getReservation() + expectedReservation := GetReservation() GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true @@ -83,7 +84,7 @@ func TestGet(t *testing.T) { } func TestGetNotFound(t *testing.T) { - expectedReservation := getReservation() + expectedReservation := GetReservation() GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true @@ -105,13 +106,13 @@ func TestUpdate(t *testing.T) { GlobalMock.NewMock().WithQuery( `UPDATE "" SET "expire_at" = ?, "owner_id" = ? WHERE ("reservations"."dataset_project" = ?) AND ("reservations"."dataset_name" = ?) AND ("reservations"."dataset_domain" = ?) AND ("reservations"."dataset_version" = ?) AND ("reservations"."tag_name" = ?) AND ("reservations"."expire_at" = ?)`, - ).WithRowsNum(1) + ).WithRowsNum(1) reservationRepo := getReservationRepo(t) - reservationKey := getReservationKey() + reservationKey := GetReservationKey() prevExpireAt := time.Now() - expireAt := prevExpireAt.Add(time.Second*50) + expireAt := prevExpireAt.Add(time.Second * 50) ownerID := "hello" rows, err := reservationRepo.Update(context.Background(), reservationKey, prevExpireAt, expireAt, ownerID) @@ -138,7 +139,7 @@ func getDBResponse(reservation models.Reservation) []map[string]interface{} { } } -func getReservationKey() models.ReservationKey { +func GetReservationKey() models.ReservationKey { return models.ReservationKey{ DatasetProject: "testProject", DatasetName: "testDataset", @@ -148,9 +149,9 @@ func getReservationKey() models.ReservationKey { } } -func getReservation() models.Reservation { +func GetReservation() models.Reservation { reservation := models.Reservation{ - ReservationKey: getReservationKey(), + ReservationKey: GetReservationKey(), OwnerID: "batman", ExpireAt: time.Unix(1, 1), } diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 722e3980..cf1761d6 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -124,6 +124,6 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(time.Second * time.Duration(dataCatalogConfig.ReservationTimeoutSec)), + ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now), } } From 4457b5bd523be5eadddea5a9ced0a2b84d58656d Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 28 Apr 2021 18:09:46 -0700 Subject: [PATCH 11/58] add more logging Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 0c764b13..99823408 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -3,6 +3,9 @@ package impl import ( "context" "errors" + "github.com/flyteorg/flytestdlib/logger" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" "time" errors2 "github.com/flyteorg/datacatalog/pkg/errors" @@ -15,12 +18,22 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" ) +type reservationMetrics struct { + scope promutils.Scope + reservationAcquiredViaCreate labeled.Counter + reservationAcquiredViaUpdate labeled.Counter + reservationAlreadyInProgress labeled.Counter + makeReservationFailure labeled.Counter + getTagFailure labeled.Counter +} + type NowFunc func() time.Time type reservationManager struct { repo repositories.RepositoryInterface reservationTimeout time.Duration now NowFunc + systemMetrics reservationMetrics } func NewReservationManager( @@ -44,6 +57,7 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * // generating the artifact. status, err := r.makeReservation(ctx, request) if err != nil { + r.systemMetrics.makeReservationFailure.Inc(ctx) return nil, err } @@ -53,6 +67,8 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * }, }, nil } + logger.Errorf(ctx, "Failed retrieve tag: %+v, err: %v", tagKey, err) + r.systemMetrics.getTagFailure.Inc(ctx) return nil, err } @@ -83,9 +99,13 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac }) if err != nil { + logger.Errorf(ctx, "Failed to create reservation: %+v, err %v", reservationKey, err) + return datacatalog.ReservationStatus{}, err } + r.systemMetrics.reservationAcquiredViaCreate.Inc(ctx) + return datacatalog.ReservationStatus{ State: datacatalog.ReservationStatus_ACQUIRED, OwnerId: request.OwnerId, @@ -107,6 +127,7 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac } if rowsAffected > 0 { + r.systemMetrics.reservationAcquiredViaUpdate.Inc(ctx) return datacatalog.ReservationStatus{ State: datacatalog.ReservationStatus_ACQUIRED, OwnerId: request.OwnerId, @@ -114,6 +135,9 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac } } + logger.Debugf(ctx, "Reservation: %+v is hold by %s", reservationKey, rsv.OwnerID) + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) return datacatalog.ReservationStatus{ State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, OwnerId: rsv.OwnerID, From 60b2c8d3c4736db66d9666a44ab220a2e3d645be Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 29 Apr 2021 14:26:38 -0700 Subject: [PATCH 12/58] add logging and stats Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 31 +++++++++++++++++++- pkg/manager/impl/reservation_manager_test.go | 12 +++++--- pkg/rpc/datacatalogservice/service.go | 3 +- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 99823408..d12fb1ef 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -39,12 +39,41 @@ type reservationManager struct { func NewReservationManager( repo repositories.RepositoryInterface, reservationTimeout time.Duration, - nowFunc NowFunc, + nowFunc NowFunc, // Easier to mock time.Time for testing + reservationScope promutils.Scope, ) interfaces.ReservationManager { + systemMetrics := reservationMetrics{ + scope: reservationScope, + reservationAcquiredViaCreate: labeled.NewCounter( + "reservation_acquired_via_create", + "Number of times a reservation was acquired via create", + reservationScope), + reservationAcquiredViaUpdate: labeled.NewCounter( + "reservation_acquired_via_update", + "Number of times a reservation was acquired via update", + reservationScope), + reservationAlreadyInProgress: labeled.NewCounter( + "reservation_already_in_progress", + "Number of times we try of acquire a reservation but the reservation is in progress", + reservationScope, + ), + makeReservationFailure: labeled.NewCounter( + "make_reservation_failure", + "Number of times we failed to make reservation", + reservationScope, + ), + getTagFailure: labeled.NewCounter( + "get_tag_failure", + "Number of times we failed to get tag", + reservationScope, + ), + } + return &reservationManager{ repo: repo, reservationTimeout: reservationTimeout, now: nowFunc, + systemMetrics:systemMetrics, } } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 0315b379..83669c5f 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -2,6 +2,7 @@ package impl import ( "context" + mockScope "github.com/flyteorg/flytestdlib/promutils" "testing" "time" @@ -56,7 +57,7 @@ func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { Artifact: expectedArtifact, }, nil) - reservationManager := NewReservationManager(&dcRepo, timeout, time.Now) + reservationManager := NewReservationManager(&dcRepo, timeout, time.Now, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -108,7 +109,8 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { }), ).Return(nil) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, + mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -152,7 +154,8 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { return ownerID == currentOwner })).Return(int64(1), nil) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, + mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -212,7 +215,8 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { setUpReservationRepoGet(&dcRepo, prevExpireAt) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }) + reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, + mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index cf1761d6..614d34ba 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -124,6 +124,7 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now), + ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now, + catalogScope.NewSubScope("tag")), } } From acccd0a5585c8713e5c8a319d41eb1d7bde95ab2 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 29 Apr 2021 14:27:55 -0700 Subject: [PATCH 13/58] fix lint Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 21 ++++++++++---------- pkg/manager/impl/reservation_manager_test.go | 1 + pkg/rpc/datacatalogservice/service.go | 6 +++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index d12fb1ef..8e3ceb55 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -3,10 +3,11 @@ package impl import ( "context" "errors" + "time" + "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" "github.com/flyteorg/flytestdlib/promutils/labeled" - "time" errors2 "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories" @@ -19,7 +20,7 @@ import ( ) type reservationMetrics struct { - scope promutils.Scope + scope promutils.Scope reservationAcquiredViaCreate labeled.Counter reservationAcquiredViaUpdate labeled.Counter reservationAlreadyInProgress labeled.Counter @@ -39,11 +40,11 @@ type reservationManager struct { func NewReservationManager( repo repositories.RepositoryInterface, reservationTimeout time.Duration, - nowFunc NowFunc, // Easier to mock time.Time for testing + nowFunc NowFunc, // Easier to mock time.Time for testing reservationScope promutils.Scope, ) interfaces.ReservationManager { systemMetrics := reservationMetrics{ - scope: reservationScope, + scope: reservationScope, reservationAcquiredViaCreate: labeled.NewCounter( "reservation_acquired_via_create", "Number of times a reservation was acquired via create", @@ -56,24 +57,24 @@ func NewReservationManager( "reservation_already_in_progress", "Number of times we try of acquire a reservation but the reservation is in progress", reservationScope, - ), - makeReservationFailure: labeled.NewCounter( + ), + makeReservationFailure: labeled.NewCounter( "make_reservation_failure", "Number of times we failed to make reservation", reservationScope, - ), - getTagFailure: labeled.NewCounter( + ), + getTagFailure: labeled.NewCounter( "get_tag_failure", "Number of times we failed to get tag", reservationScope, - ), + ), } return &reservationManager{ repo: repo, reservationTimeout: reservationTimeout, now: nowFunc, - systemMetrics:systemMetrics, + systemMetrics: systemMetrics, } } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 83669c5f..fba49666 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -2,6 +2,7 @@ package impl import ( "context" + mockScope "github.com/flyteorg/flytestdlib/promutils" "testing" diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 614d34ba..dc64ba20 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -121,9 +121,9 @@ func NewDataCatalogService() *DataCatalogService { }() return &DataCatalogService{ - DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), - ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), - TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), + DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), + ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), + TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now, catalogScope.NewSubScope("tag")), } From e2c6b40ada56412a19be24d8d0ee58adbf832b06 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 29 Apr 2021 14:52:35 -0700 Subject: [PATCH 14/58] add more comments Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 5 +++++ pkg/rpc/datacatalogservice/service.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 8e3ceb55..e0a1faab 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -156,6 +156,9 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac return datacatalog.ReservationStatus{}, err } + // If rows affected is 0, possibly someone else is also + // trying to grab the reservation at the same time and + // they won. if rowsAffected > 0 { r.systemMetrics.reservationAcquiredViaUpdate.Inc(ctx) return datacatalog.ReservationStatus{ @@ -163,6 +166,8 @@ func (r *reservationManager) makeReservation(ctx context.Context, request *datac OwnerId: request.OwnerId, }, nil } + logger.Debugf(ctx, "Failed to reserve a spot: %+v, owner: %s", reservationKey, + request.OwnerId) } logger.Debugf(ctx, "Reservation: %+v is hold by %s", reservationKey, rsv.OwnerID) diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index dc64ba20..7d51a60a 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -125,6 +125,6 @@ func NewDataCatalogService() *DataCatalogService { ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now, - catalogScope.NewSubScope("tag")), + catalogScope.NewSubScope("reservation")), } } From bf3011e356871ec374844a47f7ce1ff42d260f52 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Mon, 10 May 2021 15:04:35 -0700 Subject: [PATCH 15/58] First -> Take Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/repositories/gormimpl/reservation.go | 2 +- pkg/repositories/gormimpl/reservation_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 6d896965..f5d9d808 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -45,7 +45,7 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva result := r.db.Where(&models.Reservation{ ReservationKey: reservationKey, - }).First(&reservation) + }).Take(&reservation) if result.Error != nil { return reservation, r.errorTransformer.ToDataCatalogError(result.Error) diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 8077c624..0f899327 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -69,7 +69,7 @@ func TestGet(t *testing.T) { GlobalMock.Logging = true GlobalMock.NewMock().WithQuery( - `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) ORDER BY "reservations"."dataset_project" ASC LIMIT 1`, + `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) LIMIT 1`, ).WithReply(getDBResponse(expectedReservation)) reservationRepo := getReservationRepo(t) From eb5a4e73b6a82c3e3892ff61c118c54d334234f5 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 13 May 2021 11:41:50 -0700 Subject: [PATCH 16/58] WIP: add createOrupdate API Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- go.mod | 9 +- go.sum | 138 ++++++++++++++++-- pkg/repositories/gormimpl/reservation.go | 22 ++- pkg/repositories/gormimpl/reservation_test.go | 36 ++++- .../interfaces/reservation_repo.go | 1 + 5 files changed, 188 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 3e66f05a..922339d6 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,16 @@ require ( github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 - github.com/jinzhu/gorm v1.9.11 - github.com/lib/pq v1.2.0 + github.com/jackc/pgproto3/v2 v2.0.7 // indirect + github.com/jinzhu/gorm v1.9.16 + github.com/lib/pq v1.3.0 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 + golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect + golang.org/x/text v0.3.6 // indirect google.golang.org/grpc v1.36.0 + gorm.io/driver/postgres v1.1.0 + gorm.io/gorm v1.21.9 ) diff --git a/go.sum b/go.sum index 254a9d87..297a0f71 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -71,9 +70,11 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Selvatico/go-mocket v1.0.7 h1:sXuFMnMfVL9b/Os8rGXPgbOFbr4HJm8aHsulD/uMTUk= @@ -87,6 +88,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -126,6 +128,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= @@ -137,6 +140,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -145,8 +149,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgfR0jcVVXuTsYv/erY2NnEDqwRojbxR1rBYA= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= @@ -210,16 +214,19 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -341,12 +348,72 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE= -github.com/jinzhu/gorm v1.9.11/go.mod h1:bu/pK8szGZ2puuErfU0RwyeNdsf3e6nCX/noXaVxkfw= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.8.1 h1:ySBX7Q87vOMqKU2bbmKbUvtYhauDFclYbNDYIE1/h6s= +github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.7 h1:6Pwi1b3QdY65cuv6SyVO0FgPd5J3Bl7wf/nQQjinHMA= +github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgtype v1.7.0 h1:6f4kVsW01QftE38ufBYxKciO6gyioXSC0ABIRLcZrGs= +github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/pgx/v4 v4.11.0 h1:J86tSWd3Y7nKjwT/43xZBvpi04keQWx8gNC2YkdJhZI= +github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= +github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -370,6 +437,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -378,11 +446,16 @@ github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -392,15 +465,22 @@ github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -513,14 +593,20 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= @@ -580,6 +666,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -596,25 +683,36 @@ go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -650,6 +748,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -694,6 +793,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -723,7 +824,9 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -732,6 +835,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -777,6 +881,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -795,6 +901,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -802,6 +909,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -846,6 +954,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -885,7 +995,6 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -971,6 +1080,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= @@ -993,6 +1103,14 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/driver/postgres v1.1.0 h1:afBljg7PtJ5lA6YUWluV2+xovIPhS+YiInuL3kUjrbk= +gorm.io/driver/postgres v1.1.0/go.mod h1:hXQIwafeRjJvUm+OMxcFWyswJ/vevcpPLlGocwAwuqw= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.0 h1:p5r2eGuuWxNrLFgimttWP51cut3iNuTj0dK0bMUvTgw= +gorm.io/gorm v1.21.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= +gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index f5d9d808..3ada2cb2 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -8,8 +8,8 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" - + "gorm.io/gorm" + "gorm.io/gorm/clause" "time" ) @@ -74,3 +74,21 @@ func (r *reservationRepo) Update(ctx context.Context, reservationKey models.Rese return result.RowsAffected, nil } + +func (r *reservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) (int64, error) { + + expressions := make([]clause.Expression, 0) + expressions = append(expressions, clause.Lte{Column: "expire_at", Value: now}) + + result := r.db.Clauses( + clause.OnConflict{ + Where: clause.Where{Exprs: expressions}, + UpdateAll: true, + }, + ).Create(&reservation) + if result.Error != nil { + return 0, r.errorTransformer.ToDataCatalogError(result.Error) + } + + return result.RowsAffected, nil +} \ No newline at end of file diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 0f899327..0b81f495 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -2,20 +2,23 @@ package gormimpl import ( "context" + "database/sql" "database/sql/driver" + "fmt" + + "gorm.io/driver/postgres" + "gorm.io/gorm" "testing" "time" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" apiErrors "github.com/flyteorg/datacatalog/pkg/errors" - "github.com/jinzhu/gorm" "google.golang.org/grpc/codes" mocket "github.com/Selvatico/go-mocket" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" - "github.com/flyteorg/datacatalog/pkg/repositories/utils" "github.com/flyteorg/flytestdlib/promutils" "github.com/stretchr/testify/assert" ) @@ -69,7 +72,7 @@ func TestGet(t *testing.T) { GlobalMock.Logging = true GlobalMock.NewMock().WithQuery( - `SELECT * FROM "reservations" WHERE "reservations"."deleted_at" IS NULL AND (("reservations"."dataset_project" = testProject) AND ("reservations"."dataset_name" = testDataset) AND ("reservations"."dataset_domain" = testDomain) AND ("reservations"."dataset_version" = testVersion) AND ("reservations"."tag_name" = testTag)) LIMIT 1`, + `SELECT * FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5 LIMIT 1%!!(string=testTag)!(string=testVersion)!(string=testDomain)!(string=testDataset)(EXTRA string=testProject)`, ).WithReply(getDBResponse(expectedReservation)) reservationRepo := getReservationRepo(t) @@ -121,8 +124,33 @@ func TestUpdate(t *testing.T) { assert.Equal(t, rows, int64(1)) } +func TestCreateOrUpdate(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT ("dataset_project","dataset_name","dataset_domain","dataset_version","tag_name") DO UPDATE SET "updated_at"="excluded"."updated_at","deleted_at"="excluded"."deleted_at","owner_id"="excluded"."owner_id","expire_at"="excluded"."expire_at","serialized_metadata"="excluded"."serialized_metadata"WHERE "expire_at" <= $12`, + ).WithRowsNum(1) + + reservationRepo := getReservationRepo(t) + + rows, err := reservationRepo.CreateOrUpdate(context.Background(), expectedReservation, time.Now()) + assert.Nil(t, err) + assert.Equal(t, int64(1), rows) +} + func getReservationRepo(t *testing.T) interfaces.ReservationRepo { - return NewReservationRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + mocket.Catcher.Register() + sqlDB, err := sql.Open(mocket.DriverName, "blah") + assert.Nil(t, err) + + db, err := gorm.Open(postgres.New(postgres.Config{Conn:sqlDB})) + if err != nil { + t.Fatal(fmt.Sprintf("Failed to open mock db with err %v", err)) + } + + return NewReservationRepo(db, errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) } func getDBResponse(reservation models.Reservation) []map[string]interface{} { diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 2283571c..ab70ca8d 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -11,4 +11,5 @@ type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) (int64, error) + CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) (int64, error) } From f9796331ce323bb91e9c80ed1e1152636145fa2a Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Fri, 7 May 2021 22:28:39 +0530 Subject: [PATCH 17/58] Added boilerplate automation (#41) Signed-off-by: Ketan Umare Signed-off-by: Daniel Rammer --- .github/workflows/boilerplate-automation.yml | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/boilerplate-automation.yml diff --git a/.github/workflows/boilerplate-automation.yml b/.github/workflows/boilerplate-automation.yml new file mode 100644 index 00000000..d4a76239 --- /dev/null +++ b/.github/workflows/boilerplate-automation.yml @@ -0,0 +1,37 @@ +name: Update Boilerplate Automation +on: + workflow_dispatch: + +jobs: + update-boilerplate: + name: Update Boilerplate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Update Boilerplate + run: | + make update_boilerplate + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.FLYTE_BOT_PAT }} + commit-message: Update Boilerplate + committer: Flyte-Bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: true + branch: flyte-bot-update-boilerplate + delete-branch: true + title: 'Update Boilerplate' + body: | + Update Boilerplate + - Auto-generated by [flyte-bot] + labels: | + boilerplate + team-reviewers: | + owners + maintainers + draft: false + From 5d934259ab3b5fbb1292aefa13997dbc4865609e Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sat, 8 May 2021 05:00:25 +0530 Subject: [PATCH 18/58] update boilerplate code (#40) Signed-off-by: Samhita Alla Signed-off-by: Daniel Rammer --- .github/config.yml | 15 +++++++++++++++ boilerplate/flyte/Readme.rst | 8 ++++++++ boilerplate/flyte/config.yml | 15 +++++++++++++++ boilerplate/flyte/update.sh | 14 ++++++++++++++ boilerplate/update.cfg | 3 ++- boilerplate/update.sh | 37 ++++++++++++++++++++++++++++-------- 6 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 .github/config.yml create mode 100644 boilerplate/flyte/Readme.rst create mode 100644 boilerplate/flyte/config.yml create mode 100755 boilerplate/flyte/update.sh diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 00000000..f99bcd78 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/boilerplate/flyte/Readme.rst b/boilerplate/flyte/Readme.rst new file mode 100644 index 00000000..ea187811 --- /dev/null +++ b/boilerplate/flyte/Readme.rst @@ -0,0 +1,8 @@ +Config File -- Welcome Bot +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Provides a ``config.yml`` file. + +**To Enable:** + +Add ``flyte/config.yml`` to your ``boilerplate/update.cfg`` file. \ No newline at end of file diff --git a/boilerplate/flyte/config.yml b/boilerplate/flyte/config.yml new file mode 100644 index 00000000..f99bcd78 --- /dev/null +++ b/boilerplate/flyte/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/boilerplate/flyte/update.sh b/boilerplate/flyte/update.sh new file mode 100755 index 00000000..506c3221 --- /dev/null +++ b/boilerplate/flyte/update.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Clone the config.yml file +echo " - copying ${DIR}/config.yml to the root directory." +cp ${DIR}/config.yml ${DIR}/../../.github/config.yml diff --git a/boilerplate/update.cfg b/boilerplate/update.cfg index 9a7a0ea7..f9eb7cc7 100644 --- a/boilerplate/update.cfg +++ b/boilerplate/update.cfg @@ -4,4 +4,5 @@ lyft/golangci_file lyft/golang_dockerfile lyft/golang_support_tools lyft/pull_request_template -lyft/github_workflows \ No newline at end of file +lyft/github_workflows +flyte/ diff --git a/boilerplate/update.sh b/boilerplate/update.sh index a8c05705..374e4b93 100755 --- a/boilerplate/update.sh +++ b/boilerplate/update.sh @@ -10,7 +10,9 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" OUT="$(mktemp -d)" -git clone git@github.com:lyft/boilerplate.git "${OUT}" +trap "rm -fr $OUT" EXIT + +git clone git@github.com:flyteorg/boilerplate.git "${OUT}" echo "Updating the update.sh script." cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh" @@ -18,7 +20,7 @@ echo "" CONFIG_FILE="${DIR}/update.cfg" -README="https://github.com/lyft/boilerplate/blob/master/Readme.rst" +README="https://github.com/flyteorg/boilerplate/blob/master/Readme.rst" if [ ! -f "$CONFIG_FILE" ]; then echo "$CONFIG_FILE not found." @@ -33,13 +35,34 @@ if [ -z "$REPOSITORY" ]; then exit 1 fi -while read directory; do - # TODO: Skip empty lines, whitespace only lines, and comment lines +while read directory junk; do + # Skip comment lines (which can have leading whitespace) + if [[ "$directory" == '#'* ]]; then + continue + fi + # Skip blank or whitespace-only lines + if [[ "$directory" == "" ]]; then + continue + fi + # Lines like + # valid/path other_junk + # are not acceptable, unless `other_junk` is a comment + if [[ "$junk" != "" ]] && [[ "$junk" != '#'* ]]; then + echo "Invalid config! Only one directory is allowed per line. Found '$junk'" + exit 1 + fi + + dir_path="${OUT}/boilerplate/${directory}" + # Make sure the directory exists + if ! [[ -d "$dir_path" ]]; then + echo "Invalid boilerplate directory: '$directory'" + exit 1 + fi + echo "***********************************************************************************" echo "$directory is configured in update.cfg." echo "-----------------------------------------------------------------------------------" echo "syncing files from source." - dir_path="${OUT}/boilerplate/${directory}" rm -rf "${DIR}/${directory}" mkdir -p $(dirname "${DIR}/${directory}") cp -r "$dir_path" "${DIR}/${directory}" @@ -49,6 +72,4 @@ while read directory; do fi echo "***********************************************************************************" echo "" -done < "$CONFIG_FILE" - -rm -rf "${OUT}" +done < "$CONFIG_FILE" \ No newline at end of file From 7892458e2d5077c11458b8005603ac8dc86010f6 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Wed, 19 May 2021 11:25:48 -0700 Subject: [PATCH 19/58] Upgrade gorm to v1.21.9 (#42) Signed-off-by: Daniel Rammer --- cmd/entrypoints/migrate.go | 6 ++- datacatalog_config.yaml | 1 + go.mod | 4 -- go.sum | 43 ++-------------- pkg/common/filters.go | 4 +- pkg/manager/impl/artifact_manager.go | 2 +- pkg/manager/impl/dataset_manager.go | 2 +- pkg/repositories/config/database.go | 5 +- pkg/repositories/config/postgres.go | 49 ++++++++----------- pkg/repositories/config/postgres_test.go | 14 ++++-- pkg/repositories/errors/postgres.go | 2 +- pkg/repositories/gormimpl/artifact.go | 28 ++++++----- pkg/repositories/gormimpl/artifact_test.go | 41 ++++++++-------- pkg/repositories/gormimpl/dataset.go | 22 +++++---- pkg/repositories/gormimpl/dataset_test.go | 38 +++++++------- pkg/repositories/gormimpl/list.go | 25 ++++++++-- pkg/repositories/gormimpl/list_test.go | 15 ++++-- pkg/repositories/gormimpl/tag.go | 15 +++--- pkg/repositories/gormimpl/tag_test.go | 41 +++++++++++++--- pkg/repositories/handle.go | 45 +++++++++++------ pkg/repositories/handle_test.go | 4 +- pkg/repositories/models/artifact.go | 6 +-- pkg/repositories/models/dataset.go | 2 +- pkg/repositories/models/list.go | 4 +- pkg/repositories/models/tag.go | 2 +- pkg/repositories/postgres_repo.go | 2 +- pkg/repositories/transformers/pagination.go | 6 +-- .../transformers/pagination_test.go | 4 +- pkg/repositories/utils/test_utils.go | 5 +- pkg/runtime/application_config_provider.go | 2 + 30 files changed, 245 insertions(+), 194 deletions(-) diff --git a/cmd/entrypoints/migrate.go b/cmd/entrypoints/migrate.go index 79514132..96635381 100644 --- a/cmd/entrypoints/migrate.go +++ b/cmd/entrypoints/migrate.go @@ -9,7 +9,6 @@ import ( "context" - _ "github.com/jinzhu/gorm/dialects/postgres" // Required to import database driver. "github.com/spf13/cobra" ) @@ -73,7 +72,10 @@ var migrateCmd = &cobra.Command{ logger.Infof(ctx, "Created DB connection.") // TODO: checkpoints for migrations - dbHandle.Migrate() + if err := dbHandle.Migrate(ctx); err != nil { + logger.Errorf(ctx, "Failed to migrate. err: %v", err) + panic(err) + } logger.Infof(ctx, "Ran DB migration successfully.") }, } diff --git a/datacatalog_config.yaml b/datacatalog_config.yaml index 19788f01..e0f9c3d1 100644 --- a/datacatalog_config.yaml +++ b/datacatalog_config.yaml @@ -30,3 +30,4 @@ database: host: localhost dbname: datacatalog options: "sslmode=disable" + log_level: 5 diff --git a/go.mod b/go.mod index 922339d6..525755f4 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,11 @@ require ( github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 - github.com/jackc/pgproto3/v2 v2.0.7 // indirect - github.com/jinzhu/gorm v1.9.16 github.com/lib/pq v1.3.0 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect - golang.org/x/text v0.3.6 // indirect google.golang.org/grpc v1.36.0 gorm.io/driver/postgres v1.1.0 gorm.io/gorm v1.21.9 diff --git a/go.sum b/go.sum index 297a0f71..0a3331b0 100644 --- a/go.sum +++ b/go.sum @@ -70,11 +70,11 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Selvatico/go-mocket v1.0.7 h1:sXuFMnMfVL9b/Os8rGXPgbOFbr4HJm8aHsulD/uMTUk= @@ -88,7 +88,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -128,6 +127,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -149,8 +149,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= @@ -171,8 +169,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -214,10 +210,9 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -225,8 +220,6 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -359,11 +352,11 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.8.1 h1:ySBX7Q87vOMqKU2bbmKbUvtYhauDFclYbNDYIE1/h6s= github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -376,8 +369,6 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1: github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.7 h1:6Pwi1b3QdY65cuv6SyVO0FgPd5J3Bl7wf/nQQjinHMA= -github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= @@ -387,7 +378,6 @@ github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrU github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= -github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.7.0 h1:6f4kVsW01QftE38ufBYxKciO6gyioXSC0ABIRLcZrGs= github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= @@ -396,7 +386,6 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= -github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= github.com/jackc/pgx/v4 v4.11.0 h1:J86tSWd3Y7nKjwT/43xZBvpi04keQWx8gNC2YkdJhZI= github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -404,14 +393,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= -github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -451,8 +434,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -479,8 +460,6 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -603,6 +582,7 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -694,7 +674,6 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -703,16 +682,12 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -748,7 +723,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -791,7 +765,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -881,8 +854,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1103,12 +1074,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/driver/postgres v1.1.0 h1:afBljg7PtJ5lA6YUWluV2+xovIPhS+YiInuL3kUjrbk= gorm.io/driver/postgres v1.1.0/go.mod h1:hXQIwafeRjJvUm+OMxcFWyswJ/vevcpPLlGocwAwuqw= -gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.0 h1:p5r2eGuuWxNrLFgimttWP51cut3iNuTj0dK0bMUvTgw= -gorm.io/gorm v1.21.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/common/filters.go b/pkg/common/filters.go index 58ca7916..87b81aeb 100644 --- a/pkg/common/filters.go +++ b/pkg/common/filters.go @@ -2,8 +2,8 @@ package common // Common constants and types for Filtering const ( - DefaultPageOffset = uint32(0) - MaxPageLimit = uint32(50) + DefaultPageOffset = 0 + MaxPageLimit = 50 ) // Common Entity types that can be used on any filters diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index a1f1bebc..85553d1a 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -276,7 +276,7 @@ func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalo artifact.Data = artifactDataList } - token := strconv.Itoa(int(listInput.Offset) + len(artifactsList)) + token := strconv.Itoa(listInput.Offset + len(artifactsList)) logger.Debugf(ctx, "Listed %v matching artifacts successfully", len(artifactsList)) m.systemMetrics.listSuccessCounter.Inc(ctx) diff --git a/pkg/manager/impl/dataset_manager.go b/pkg/manager/impl/dataset_manager.go index 835d0255..2607f5c3 100644 --- a/pkg/manager/impl/dataset_manager.go +++ b/pkg/manager/impl/dataset_manager.go @@ -182,7 +182,7 @@ func (dm *datasetManager) ListDatasets(ctx context.Context, request *datacatalog return nil, errors.NewCollectedErrors(codes.Internal, transformerErrs) } - token := strconv.Itoa(int(listInput.Offset) + len(datasetList)) + token := strconv.Itoa(listInput.Offset + len(datasetList)) logger.Debugf(ctx, "Listed %v matching datasets successfully", len(datasetList)) dm.systemMetrics.listSuccessCounter.Inc(ctx) diff --git a/pkg/repositories/config/database.go b/pkg/repositories/config/database.go index 7ac5930b..f6c5b26f 100644 --- a/pkg/repositories/config/database.go +++ b/pkg/repositories/config/database.go @@ -1,5 +1,7 @@ package config +import "gorm.io/gorm/logger" + //go:generate pflags DbConfigSection // This struct corresponds to the database section of in the config @@ -12,7 +14,8 @@ type DbConfigSection struct { Password string `json:"password"` PasswordPath string `json:"passwordPath"` // See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - ExtraOptions string `json:"options"` + ExtraOptions string `json:"options"` + LogLevel logger.LogLevel `json:"log_level"` } // Database config. Contains values necessary to open a database connection. diff --git a/pkg/repositories/config/postgres.go b/pkg/repositories/config/postgres.go index 7e73a694..f8449cca 100644 --- a/pkg/repositories/config/postgres.go +++ b/pkg/repositories/config/postgres.go @@ -3,30 +3,29 @@ package config import ( "fmt" + "gorm.io/gorm/logger" + "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/postgres" // Required to import database driver. + "gorm.io/driver/postgres" + "gorm.io/gorm" ) const Postgres = "postgres" // Generic interface for providing a config necessary to open a database connection. type DbConnectionConfigProvider interface { - // Returns the database type. For instance PostgreSQL or MySQL. - GetType() string - // Returns arguments specific for the database type necessary to open a database connection. - GetArgs() string - // Enables verbose logging. - WithDebugModeEnabled() - // Disables verbose logging. - WithDebugModeDisabled() - // Returns whether verbose logging is enabled or not. - IsDebug() bool + // Returns database dialector + GetDialector() gorm.Dialector + + GetDBConfig() DbConfig + + GetDSN() string } type BaseConfig struct { - IsDebug bool + LogLevel logger.LogLevel `json:"log_level"` + DisableForeignKeyConstraintWhenMigrating bool } // PostgreSQL implementation for DbConnectionConfigProvider. @@ -43,11 +42,7 @@ func NewPostgresConfigProvider(config DbConfig, scope promutils.Scope) DbConnect } } -func (p *PostgresConfigProvider) GetType() string { - return Postgres -} - -func (p *PostgresConfigProvider) GetArgs() string { +func (p *PostgresConfigProvider) GetDSN() string { if p.config.Password == "" { // Switch for development return fmt.Sprintf("host=%s port=%d dbname=%s user=%s sslmode=disable", @@ -57,25 +52,23 @@ func (p *PostgresConfigProvider) GetArgs() string { p.config.Host, p.config.Port, p.config.DbName, p.config.User, p.config.Password, p.config.ExtraOptions) } -func (p *PostgresConfigProvider) WithDebugModeEnabled() { - p.config.IsDebug = true -} - -func (p *PostgresConfigProvider) WithDebugModeDisabled() { - p.config.IsDebug = false +func (p *PostgresConfigProvider) GetDialector() gorm.Dialector { + return postgres.Open(p.GetDSN()) } -func (p *PostgresConfigProvider) IsDebug() bool { - return p.config.IsDebug +func (p *PostgresConfigProvider) GetDBConfig() DbConfig { + return p.config } // Opens a connection to the database specified in the config. // You must call CloseDbConnection at the end of your session! func OpenDbConnection(config DbConnectionConfigProvider) (*gorm.DB, error) { - db, err := gorm.Open(config.GetType(), config.GetArgs()) + db, err := gorm.Open(config.GetDialector(), &gorm.Config{ + Logger: logger.Default.LogMode(config.GetDBConfig().LogLevel), + DisableForeignKeyConstraintWhenMigrating: config.GetDBConfig().DisableForeignKeyConstraintWhenMigrating, + }) if err != nil { return nil, err } - db.LogMode(config.IsDebug()) return db, nil } diff --git a/pkg/repositories/config/postgres_test.go b/pkg/repositories/config/postgres_test.go index c79fe332..02be2cd0 100644 --- a/pkg/repositories/config/postgres_test.go +++ b/pkg/repositories/config/postgres_test.go @@ -3,6 +3,8 @@ package config import ( "testing" + "gorm.io/gorm/logger" + mockScope "github.com/flyteorg/flytestdlib/promutils" "github.com/stretchr/testify/assert" @@ -15,9 +17,15 @@ func TestConstructGormArgs(t *testing.T) { DbName: "postgres", User: "postgres", ExtraOptions: "sslmode=disable", + BaseConfig: BaseConfig{ + LogLevel: 3, + DisableForeignKeyConstraintWhenMigrating: true, + }, }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetDSN()) + assert.Equal(t, logger.LogLevel(3), postgresConfigProvider.GetDBConfig().LogLevel) + assert.Equal(t, true, postgresConfigProvider.GetDBConfig().DisableForeignKeyConstraintWhenMigrating) } func TestConstructGormArgsWithPassword(t *testing.T) { @@ -30,7 +38,7 @@ func TestConstructGormArgsWithPassword(t *testing.T) { ExtraOptions: "sslmode=enable", }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetDSN()) } func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { @@ -42,5 +50,5 @@ func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { Password: "pass", }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetDSN()) } diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index 8b663906..d68a9321 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/flyteorg/datacatalog/pkg/errors" - "github.com/jinzhu/gorm" "github.com/lib/pq" "google.golang.org/grpc/codes" + "gorm.io/gorm" ) // Postgres error codes diff --git a/pkg/repositories/gormimpl/artifact.go b/pkg/repositories/gormimpl/artifact.go index 757e248d..6b1e445a 100644 --- a/pkg/repositories/gormimpl/artifact.go +++ b/pkg/repositories/gormimpl/artifact.go @@ -3,13 +3,14 @@ package gormimpl import ( "context" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type artifactRepo struct { @@ -65,19 +66,20 @@ func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.A ) if result.Error != nil { + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Artifact{}, errors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ + Dataset: &datacatalog.DatasetID{ + Project: in.DatasetProject, + Domain: in.DatasetDomain, + Name: in.DatasetName, + Version: in.DatasetVersion, + }, + Id: in.ArtifactID, + }) + } + return models.Artifact{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Artifact{}, errors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ - Dataset: &datacatalog.DatasetID{ - Project: in.DatasetProject, - Domain: in.DatasetDomain, - Name: in.DatasetName, - Version: in.DatasetVersion, - }, - Id: in.ArtifactID, - }) - } return artifact, nil } diff --git a/pkg/repositories/gormimpl/artifact_test.go b/pkg/repositories/gormimpl/artifact_test.go index 027bb0f8..49292fb8 100644 --- a/pkg/repositories/gormimpl/artifact_test.go +++ b/pkg/repositories/gormimpl/artifact_test.go @@ -117,21 +117,22 @@ func TestCreateArtifact(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithCallback( func(s string, values []driver.NamedValue) { artifactCreated = true }, ) GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10),($11,$12,$13,$14,$15,$16,$17,$18,$19,$20) ON CONFLICT ("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name") DO UPDATE SET "dataset_project"="excluded"."dataset_project","dataset_name"="excluded"."dataset_name","dataset_domain"="excluded"."dataset_domain","dataset_version"="excluded"."dataset_version","artifact_id"="excluded"."artifact_id"`).WithCallback( func(s string, values []driver.NamedValue) { - numArtifactDataCreated++ + // Batch insert + numArtifactDataCreated += 2 }, ) GlobalMock.NewMock().WithQuery( - `INSERT INTO "partitions" ("created_at","updated_at","deleted_at","dataset_uuid","key","value","artifact_id") VALUES (?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "partitions" ("created_at","updated_at","deleted_at","dataset_uuid","key","value","artifact_id") VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT ("dataset_uuid","key","value","artifact_id") DO UPDATE SET "artifact_id"="excluded"."artifact_id"`).WithCallback( func(s string, values []driver.NamedValue) { numPartitionsCreated++ }, @@ -175,13 +176,13 @@ func TestGetArtifact(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND (("artifacts"."dataset_project" = testProject) AND ("artifacts"."dataset_name" = testName) AND ("artifacts"."dataset_domain" = testDomain) AND ("artifacts"."dataset_version" = testVersion) AND ("artifacts"."artifact_id" = 123)) ORDER BY artifacts.created_at DESC,"artifacts"."dataset_project" ASC LIMIT 1`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE "artifacts"."dataset_project" = $1 AND "artifacts"."dataset_name" = $2 AND "artifacts"."dataset_domain" = $3 AND "artifacts"."dataset_version" = $4 AND "artifacts"."artifact_id" = $5 ORDER BY artifacts.created_at DESC,"artifacts"."created_at" LIMIT 1%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) getInput := models.ArtifactKey{ DatasetProject: artifact.DatasetProject, DatasetDomain: artifact.DatasetDomain, @@ -217,13 +218,13 @@ func TestGetArtifactByID(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND (("artifacts"."artifact_id" = 123)) ORDER BY artifacts.created_at DESC,"artifacts"."dataset_project" ASC LIMIT 1`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE "artifacts"."artifact_id" = $1 ORDER BY artifacts.created_at DESC,"artifacts"."created_at" LIMIT 1%!(EXTRA string=123)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) getInput := models.ArtifactKey{ ArtifactID: artifact.ArtifactID, } @@ -265,7 +266,7 @@ func TestCreateArtifactAlreadyExists(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithError( getAlreadyExistsErr(), ) @@ -289,13 +290,13 @@ func TestListArtifactsWithPartition(t *testing.T) { expectedPartitionResponse := getDBPartitionResponse(artifact) expectedTagResponse := getDBTagResponse(artifact) GlobalMock.NewMock().WithQuery( - `SELECT "artifacts".* FROM "artifacts" JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id WHERE "artifacts"."deleted_at" IS NULL AND ((partitions0.key = val1) AND (partitions0.val = val2) AND (artifacts.dataset_uuid = test-uuid)) ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedArtifactResponse) + `SELECT "artifacts"."created_at","artifacts"."updated_at","artifacts"."deleted_at","artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id","artifacts"."dataset_uuid","artifacts"."serialized_metadata" FROM "artifacts" JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id WHERE partitions0.key = $1 AND partitions0.val = $2 AND artifacts.dataset_uuid = $3 ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10%!!(string=test-uuid)!(string=val2)(EXTRA string=val1)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ @@ -333,11 +334,11 @@ func TestListArtifactsNoPartitions(t *testing.T) { expectedPartitionResponse := make([]map[string]interface{}, 0) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND ((artifacts.dataset_uuid = test-uuid)) LIMIT 10 OFFSET 10`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE artifacts.dataset_uuid = $1 LIMIT 10 OFFSET 10%!(EXTRA string=test-uuid)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123)))`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ diff --git a/pkg/repositories/gormimpl/dataset.go b/pkg/repositories/gormimpl/dataset.go index 90d20c4d..c7b48158 100644 --- a/pkg/repositories/gormimpl/dataset.go +++ b/pkg/repositories/gormimpl/dataset.go @@ -3,14 +3,15 @@ package gormimpl import ( "context" + idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type dataSetRepo struct { @@ -51,16 +52,17 @@ func (h *dataSetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dat if result.Error != nil { logger.Debugf(ctx, "Unable to find Dataset: [%+v], err: %v", in, result.Error) + + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Dataset{}, errors.GetMissingEntityError("Dataset", &idl_datacatalog.DatasetID{ + Project: in.Project, + Domain: in.Domain, + Name: in.Name, + Version: in.Version, + }) + } return models.Dataset{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Dataset{}, errors.GetMissingEntityError("Dataset", &idl_datacatalog.DatasetID{ - Project: in.Project, - Domain: in.Domain, - Name: in.Name, - Version: in.Version, - }) - } return ds, nil } diff --git a/pkg/repositories/gormimpl/dataset_test.go b/pkg/repositories/gormimpl/dataset_test.go index 41d56b51..2f2ef8e9 100644 --- a/pkg/repositories/gormimpl/dataset_test.go +++ b/pkg/repositories/gormimpl/dataset_test.go @@ -1,9 +1,9 @@ package gormimpl import ( - "testing" - "context" + "testing" + "time" mocket "github.com/Selvatico/go-mocket" "google.golang.org/grpc/codes" @@ -40,6 +40,11 @@ func getTestDataset() models.Dataset { {Name: "key1"}, {Name: "key2"}, }, + BaseModel: models.BaseModel{ + CreatedAt: time.Unix(111, 0), + UpdatedAt: time.Unix(111, 0), + DeletedAt: nil, + }, } } @@ -82,7 +87,7 @@ func TestCreateDatasetNoPartitions(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, dataset.Project, values[3].Value) assert.EqualValues(t, dataset.Name, values[4].Value) @@ -110,7 +115,7 @@ func TestCreateDataset(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, dataset.Project, values[3].Value) assert.EqualValues(t, dataset.Name, values[4].Value) @@ -119,17 +124,14 @@ func TestCreateDataset(t *testing.T) { assert.EqualValues(t, dataset.SerializedMetadata, values[7].Value) datasetCreated = true }, - ) - - GlobalMock.NewMock().WithQuery( - `SELECT "uuid" FROM "datasets" WHERE (project = testProject) AND (name = testName) AND (domain = testDomain) AND (version = testVersion)`).WithReply([]map[string]interface{}{{"uuid": getDatasetUUID()}}) + ).WithReply([]map[string]interface{}{{"dataset_uuid": getDatasetUUID()}}) GlobalMock.NewMock().WithQuery( - `INSERT INTO "partition_keys" ("created_at","updated_at","deleted_at","dataset_uuid","name") VALUES (?,?,?,?,?)`).WithCallback( + `INSERT INTO "partition_keys" ("created_at","updated_at","deleted_at","dataset_uuid","name") VALUES ($1,$2,$3,$4,$5),($6,$7,$8,$9,$10) ON CONFLICT ("dataset_uuid","name") DO UPDATE SET "dataset_uuid"="excluded"."dataset_uuid"`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, getDatasetUUID(), values[3].Value) assert.EqualValues(t, dataset.PartitionKeys[insertKeyQueryNum].Name, values[4].Value) - insertKeyQueryNum++ + insertKeyQueryNum += 2 // batch insertion }, ) @@ -157,7 +159,7 @@ func TestGetDataset(t *testing.T) { GlobalMock.Logging = true // Only match on queries that append expected filters - GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND (("datasets"."project" = testProject) AND ("datasets"."name" = testName) AND ("datasets"."domain" = testDomain) AND ("datasets"."version" = testVersion)) ORDER BY "datasets"."project" ASC LIMIT 1`).WithReply(expectedDatasetResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."project" = $1 AND "datasets"."name" = $2 AND "datasets"."domain" = $3 AND "datasets"."version" = $4 ORDER BY "datasets"."created_at" LIMIT 1`).WithReply(expectedDatasetResponse) expectedPartitionKeyResponse := make([]map[string]interface{}, 0) samplePartitionKey := make(map[string]interface{}) @@ -165,7 +167,7 @@ func TestGetDataset(t *testing.T) { samplePartitionKey["dataset_uuid"] = getDatasetUUID() expectedPartitionKeyResponse = append(expectedPartitionKeyResponse, samplePartitionKey, samplePartitionKey) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid))) ORDER BY partition_keys.created_at ASC,"partition_keys"."dataset_uuid" ASC`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1 ORDER BY partition_keys.created_at ASC`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) assert.NoError(t, err) @@ -196,7 +198,7 @@ func TestGetDatasetWithUUID(t *testing.T) { GlobalMock.Logging = true // Only match on queries that append expected filters - GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND (("datasets"."uuid" = test-uuid)) ORDER BY "datasets"."project" ASC LIMIT 1`).WithReply(expectedResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."uuid" = $1 ORDER BY "datasets"."created_at" LIMIT 1%!(EXTRA string=test-uuid)`).WithReply(expectedResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) @@ -235,7 +237,7 @@ func TestCreateDatasetAlreadyExists(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithError( getAlreadyExistsErr(), ) @@ -256,10 +258,10 @@ func TestListDatasets(t *testing.T) { expectedDatasetDBResponse := getDBDatasetResponse(dataset) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL LIMIT 10 OFFSET 0`).WithReply(expectedDatasetDBResponse) + `SELECT * FROM "datasets" LIMIT 10`).WithReply(expectedDatasetDBResponse) expectedPartitionKeyResponse := getDBPartitionKeysResponse([]models.Dataset{dataset}) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid)))`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ Limit: 10, @@ -284,10 +286,10 @@ func TestListDatasetWithFilter(t *testing.T) { expectedDatasetDBResponse := getDBDatasetResponse(dataset) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND ((datasets.project = p) AND (datasets.domain = d)) ORDER BY datasets.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedDatasetDBResponse) + `SELECT * FROM "datasets" WHERE datasets.project = $1 AND datasets.domain = $2 ORDER BY datasets.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedDatasetDBResponse) expectedPartitionKeyResponse := getDBPartitionKeysResponse([]models.Dataset{dataset}) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid)))`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1%!(EXTRA string=test-uuid)`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ diff --git a/pkg/repositories/gormimpl/list.go b/pkg/repositories/gormimpl/list.go index 9e47b956..a231c8a1 100644 --- a/pkg/repositories/gormimpl/list.go +++ b/pkg/repositories/gormimpl/list.go @@ -3,10 +3,13 @@ package gormimpl import ( "fmt" + errors2 "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" - "github.com/jinzhu/gorm" + "google.golang.org/grpc/codes" + "gorm.io/gorm" ) const ( @@ -20,6 +23,15 @@ var entityToModel = map[common.Entity]interface{}{ common.Tag: models.Tag{}, } +func getTableName(tx *gorm.DB, model interface{}) (string, error) { + stmt := gorm.Statement{DB: tx} + + if err := stmt.Parse(model); err != nil { + return "", errors2.NewDataCatalogError(codes.InvalidArgument, err.Error()) + } + return stmt.Schema.Table, nil +} + // Apply the list query on the source model. This method will apply the necessary joins, filters and // pagination on the database for the given ListModelInputs. func applyListModelsInput(tx *gorm.DB, sourceEntity common.Entity, in models.ListModelsInput) (*gorm.DB, error) { @@ -28,14 +40,21 @@ func applyListModelsInput(tx *gorm.DB, sourceEntity common.Entity, in models.Lis return nil, errors.GetInvalidEntityError(sourceEntity) } - sourceTableName := tx.NewScope(sourceModel).TableName() + sourceTableName, err := getTableName(tx, sourceModel) + if err != nil { + return nil, err + } + for modelIndex, modelFilter := range in.ModelFilters { entity := modelFilter.Entity filterModel, ok := entityToModel[entity] if !ok { return nil, errors.GetInvalidEntityError(entity) } - tableName := tx.NewScope(filterModel).TableName() + tableName, err := getTableName(tx, filterModel) + if err != nil { + return nil, err + } tableAlias := tableName // Optionally add the join condition if the entity we need isn't the source diff --git a/pkg/repositories/gormimpl/list_test.go b/pkg/repositories/gormimpl/list_test.go index b795afe9..2220636f 100644 --- a/pkg/repositories/gormimpl/list_test.go +++ b/pkg/repositories/gormimpl/list_test.go @@ -20,13 +20,12 @@ func TestApplyFilter(t *testing.T) { validInputApply := false GlobalMock.NewMock().WithQuery( - `SELECT "artifacts".* FROM "artifacts"`).WithCallback( + `SELECT "artifacts"."created_at","artifacts"."updated_at","artifacts"."deleted_at","artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id","artifacts"."dataset_uuid","artifacts"."serialized_metadata" FROM "artifacts"`).WithCallback( func(s string, values []driver.NamedValue) { // separate the regex matching because the joins reorder on different test runs validInputApply = strings.Contains(s, `JOIN tags tags1 ON artifacts.artifact_id = tags1.artifact_id`) && strings.Contains(s, `JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id`) && - strings.Contains(s, `WHERE "artifacts"."deleted_at" IS NULL AND `+ - `((partitions0.key1 = val1) AND (partitions0.key2 = val2) AND (tags1.tag_name = special)) `+ + strings.Contains(s, `WHERE partitions0.key1 = $1 AND partitions0.key2 = $2 AND tags1.tag_name = $3 `+ `ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10`) }) @@ -67,7 +66,7 @@ func TestApplyFilterEmpty(t *testing.T) { validInputApply := false GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL LIMIT 10 OFFSET 10`).WithCallback( + `SELECT * FROM "artifacts" LIMIT 10 OFFSET 10`).WithCallback( func(s string, values []driver.NamedValue) { // separate the regex matching because the joins reorder on different test runs validInputApply = true @@ -84,3 +83,11 @@ func TestApplyFilterEmpty(t *testing.T) { tx.Find(models.Artifact{}) assert.True(t, validInputApply) } + +func TestGetTableErr(t *testing.T) { + testDB := utils.GetDbForTest(t) + + tableName, err := getTableName(testDB, "") + assert.Error(t, err) + assert.Equal(t, "", tableName) +} diff --git a/pkg/repositories/gormimpl/tag.go b/pkg/repositories/gormimpl/tag.go index 8e40de84..24c7750e 100644 --- a/pkg/repositories/gormimpl/tag.go +++ b/pkg/repositories/gormimpl/tag.go @@ -3,12 +3,13 @@ package gormimpl import ( "context" + idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type tagRepo struct { @@ -54,13 +55,13 @@ func (h *tagRepo) Get(ctx context.Context, in models.TagKey) (models.Tag, error) }) if result.Error != nil { + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Tag{}, errors.GetMissingEntityError("Tag", &idl_datacatalog.Tag{ + Name: tag.TagName, + }) + } return models.Tag{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Tag{}, errors.GetMissingEntityError("Tag", &idl_datacatalog.Tag{ - Name: tag.TagName, - }) - } return tag, nil } diff --git a/pkg/repositories/gormimpl/tag_test.go b/pkg/repositories/gormimpl/tag_test.go index 2c7526e9..c7c9f3b1 100644 --- a/pkg/repositories/gormimpl/tag_test.go +++ b/pkg/repositories/gormimpl/tag_test.go @@ -3,6 +3,8 @@ package gormimpl import ( "testing" + "gorm.io/gorm" + "context" mocket "github.com/Selvatico/go-mocket" @@ -51,7 +53,7 @@ func TestCreateTag(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithCallback( func(s string, values []driver.NamedValue) { tagCreated = true }, @@ -71,15 +73,15 @@ func TestGetTag(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND (("tags"."dataset_project" = testProject) AND ("tags"."dataset_name" = testName) AND ("tags"."dataset_domain" = testDomain) AND ("tags"."dataset_version" = testVersion) AND ("tags"."tag_name" = test-tag)) ORDER BY tags.created_at DESC,"tags"."dataset_project" ASC LIMIT 1`).WithReply(getDBTagResponse(artifact)) + `SELECT * FROM "tags" WHERE "tags"."dataset_project" = $1 AND "tags"."dataset_name" = $2 AND "tags"."dataset_domain" = $3 AND "tags"."dataset_version" = $4 AND "tags"."tag_name" = $5 ORDER BY tags.created_at DESC,"tags"."created_at" LIMIT 1%!!(string=test-tag)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBTagResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(getDBArtifactResponse(artifact)) + `SELECT * FROM "artifacts" WHERE ("artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBArtifactResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(getDBArtifactDataResponse(artifact)) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBArtifactDataResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(getDBPartitionResponse(artifact)) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(getDBPartitionResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(getDBTagResponse(artifact)) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(getDBTagResponse(artifact)) getInput := models.TagKey{ DatasetProject: artifact.DatasetProject, DatasetDomain: artifact.DatasetDomain, @@ -98,13 +100,38 @@ func TestGetTag(t *testing.T) { assert.Len(t, response.Artifact.Tags, 1) } +func TestTagNotFound(t *testing.T) { + artifact := getTestArtifact() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + // Only match on queries that append expected filters + GlobalMock.NewMock().WithQuery( + `SELECT * FROM "tags" WHERE "tags"."dataset_project" = $1 AND "tags"."dataset_name" = $2 AND "tags"."dataset_domain" = $3 AND "tags"."dataset_version" = $4 AND "tags"."tag_name" = $5 ORDER BY tags.created_at DESC,"tags"."created_at" LIMIT 1%!!(string=test-tag)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithError( + gorm.ErrRecordNotFound, + ) + getInput := models.TagKey{ + DatasetProject: artifact.DatasetProject, + DatasetDomain: artifact.DatasetDomain, + DatasetName: artifact.DatasetName, + DatasetVersion: artifact.DatasetVersion, + TagName: "test-tag", + } + + tagRepo := NewTagRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + _, err := tagRepo.Get(context.Background(), getInput) + assert.Error(t, err) + assert.Equal(t, "missing entity of type Tag with identifier ", err.Error()) +} + func TestTagAlreadyExists(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithError( getAlreadyExistsErr(), ) diff --git a/pkg/repositories/handle.go b/pkg/repositories/handle.go index d11fbd34..14c0039b 100644 --- a/pkg/repositories/handle.go +++ b/pkg/repositories/handle.go @@ -9,7 +9,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type DBHandle struct { @@ -24,6 +24,9 @@ func NewDBHandle(dbConfigValues config.DbConfig, catalogScope promutils.Scope) ( User: dbConfigValues.User, Password: dbConfigValues.Password, ExtraOptions: dbConfigValues.ExtraOptions, + BaseConfig: config.BaseConfig{ + DisableForeignKeyConstraintWhenMigrating: true, + }, } //TODO: abstract away the type of db we are connecting to @@ -65,20 +68,34 @@ func (h *DBHandle) CreateDB(dbName string) error { return nil } -func (h *DBHandle) Migrate() { - if h.db.Dialect().GetName() == config.Postgres { - logger.Infof(context.TODO(), "Creating postgres extension uuid-ossp if it does not exist") +func (h *DBHandle) Migrate(ctx context.Context) error { + if h.db.Config.Dialector.Name() == config.Postgres { + logger.Infof(ctx, "Creating postgres extension uuid-ossp if it does not exist") h.db.Exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"") } - h.db.AutoMigrate(&models.Dataset{}) - h.db.AutoMigrate(&models.Artifact{}) - h.db.AutoMigrate(&models.ArtifactData{}) - h.db.AutoMigrate(&models.Tag{}) - h.db.AutoMigrate(&models.PartitionKey{}) - h.db.AutoMigrate(&models.Partition{}) - h.db.AutoMigrate(&models.Reservation{}) -} + if err := h.db.AutoMigrate(&models.Dataset{}); err != nil { + return err + } + + if err := h.db.Debug().AutoMigrate(&models.Artifact{}); err != nil { + return err + } -func (h *DBHandle) Close() error { - return h.db.Close() + if err := h.db.AutoMigrate(&models.ArtifactData{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.Tag{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.PartitionKey{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.Partition{}); err != nil { + return err + } + + return nil } diff --git a/pkg/repositories/handle_test.go b/pkg/repositories/handle_test.go index bc3641e1..3ed0cf85 100644 --- a/pkg/repositories/handle_test.go +++ b/pkg/repositories/handle_test.go @@ -17,7 +17,7 @@ func TestCreateDB(t *testing.T) { checkExists := false GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = testDB)`).WithCallback( + `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( func(s string, values []driver.NamedValue) { checkExists = true }, @@ -51,7 +51,7 @@ func TestDBAlreadyExists(t *testing.T) { checkExists := false GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = testDB)`).WithCallback( + `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( func(s string, values []driver.NamedValue) { checkExists = true }, diff --git a/pkg/repositories/models/artifact.go b/pkg/repositories/models/artifact.go index 4856c37c..4b7fa660 100644 --- a/pkg/repositories/models/artifact.go +++ b/pkg/repositories/models/artifact.go @@ -13,9 +13,9 @@ type Artifact struct { ArtifactKey DatasetUUID string `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"` Dataset Dataset `gorm:"association_autocreate:false"` - ArtifactData []ArtifactData `gorm:"association_foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` - Partitions []Partition `gorm:"association_foreignkey:ArtifactID;foreignkey:ArtifactID"` - Tags []Tag `gorm:"association_foreignkey:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` + ArtifactData []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Partitions []Partition `gorm:"references:ArtifactID;foreignkey:ArtifactID"` + Tags []Tag `gorm:"references:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` SerializedMetadata []byte } diff --git a/pkg/repositories/models/dataset.go b/pkg/repositories/models/dataset.go index d35a96b4..95fe8d43 100644 --- a/pkg/repositories/models/dataset.go +++ b/pkg/repositories/models/dataset.go @@ -12,7 +12,7 @@ type Dataset struct { BaseModel DatasetKey SerializedMetadata []byte - PartitionKeys []PartitionKey `gorm:"association_foreignkey:UUID;foreignkey:DatasetUUID"` + PartitionKeys []PartitionKey `gorm:"references:UUID;foreignkey:DatasetUUID"` } type PartitionKey struct { diff --git a/pkg/repositories/models/list.go b/pkg/repositories/models/list.go index 26307742..234c981a 100644 --- a/pkg/repositories/models/list.go +++ b/pkg/repositories/models/list.go @@ -7,9 +7,9 @@ type ListModelsInput struct { // The filters for the list ModelFilters []ModelFilter // The number of models to list - Limit uint32 + Limit int // The token to offset results by - Offset uint32 + Offset int // Parameter to sort by SortParameter SortParameter } diff --git a/pkg/repositories/models/tag.go b/pkg/repositories/models/tag.go index 057f78c3..7b239cb3 100644 --- a/pkg/repositories/models/tag.go +++ b/pkg/repositories/models/tag.go @@ -13,5 +13,5 @@ type Tag struct { TagKey ArtifactID string DatasetUUID string `gorm:"type:uuid;index:tags_dataset_uuid_idx"` - Artifact Artifact `gorm:"association_foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` } diff --git a/pkg/repositories/postgres_repo.go b/pkg/repositories/postgres_repo.go index e7dc1cd8..c51d5db6 100644 --- a/pkg/repositories/postgres_repo.go +++ b/pkg/repositories/postgres_repo.go @@ -5,7 +5,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/gormimpl" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type PostgresRepo struct { diff --git a/pkg/repositories/transformers/pagination.go b/pkg/repositories/transformers/pagination.go index 1439cb00..0965e123 100644 --- a/pkg/repositories/transformers/pagination.go +++ b/pkg/repositories/transformers/pagination.go @@ -25,13 +25,13 @@ func ApplyPagination(paginationOpts *datacatalog.PaginationOptions, input *model if len(strings.Trim(paginationOpts.Token, " ")) == 0 { offset = common.DefaultPageOffset } else { - parsedOffset, err := strconv.ParseUint(paginationOpts.Token, 10, 32) + parsedOffset, err := strconv.ParseInt(paginationOpts.Token, 10, 32) if err != nil { return errors.NewDataCatalogErrorf(codes.InvalidArgument, "Invalid token %v", offset) } - offset = uint32(parsedOffset) + offset = int(parsedOffset) } - limit = paginationOpts.Limit + limit = int(paginationOpts.Limit) sortKey = paginationOpts.SortKey sortOrder = paginationOpts.SortOrder } diff --git a/pkg/repositories/transformers/pagination_test.go b/pkg/repositories/transformers/pagination_test.go index b05a70db..7cfa3d2f 100644 --- a/pkg/repositories/transformers/pagination_test.go +++ b/pkg/repositories/transformers/pagination_test.go @@ -33,7 +33,7 @@ func TestCorrectPagination(t *testing.T) { SortOrder: datacatalog.PaginationOptions_DESCENDING, }, listModelsInput) assert.NoError(t, err) - assert.Equal(t, uint32(50), listModelsInput.Limit) - assert.Equal(t, uint32(100), listModelsInput.Offset) + assert.Equal(t, 50, listModelsInput.Limit) + assert.Equal(t, 100, listModelsInput.Offset) assert.Equal(t, "artifacts.created_at desc", listModelsInput.SortParameter.GetDBOrderExpression("artifacts")) } diff --git a/pkg/repositories/utils/test_utils.go b/pkg/repositories/utils/test_utils.go index 127ede23..87b2382b 100644 --- a/pkg/repositories/utils/test_utils.go +++ b/pkg/repositories/utils/test_utils.go @@ -6,12 +6,13 @@ import ( "testing" mocket "github.com/Selvatico/go-mocket" - "github.com/jinzhu/gorm" + "gorm.io/driver/postgres" + "gorm.io/gorm" ) func GetDbForTest(t *testing.T) *gorm.DB { mocket.Catcher.Register() - db, err := gorm.Open(mocket.DriverName, "fake args") + db, err := gorm.Open(postgres.New(postgres.Config{DriverName: mocket.DriverName})) if err != nil { t.Fatal(fmt.Sprintf("Failed to open mock db with err %v", err)) } diff --git a/pkg/runtime/application_config_provider.go b/pkg/runtime/application_config_provider.go index eef9e93b..157ffdba 100644 --- a/pkg/runtime/application_config_provider.go +++ b/pkg/runtime/application_config_provider.go @@ -40,6 +40,7 @@ func (p *ApplicationConfigurationProvider) GetDbConfig() dbconfig.DbConfig { } password = string(passwordVal) } + return dbconfig.DbConfig{ Host: dbConfigSection.Host, Port: dbConfigSection.Port, @@ -47,6 +48,7 @@ func (p *ApplicationConfigurationProvider) GetDbConfig() dbconfig.DbConfig { User: dbConfigSection.User, Password: password, ExtraOptions: dbConfigSection.ExtraOptions, + BaseConfig: dbconfig.BaseConfig{LogLevel: dbConfigSection.LogLevel}, } } From c0fbbcaabf3c234c3e5672722e64fe25eacde201 Mon Sep 17 00:00:00 2001 From: Yuvraj <10830562+evalsocket@users.noreply.github.com> Date: Fri, 28 May 2021 01:00:15 +0530 Subject: [PATCH 20/58] Added boilerplate automation (#43) Signed-off-by: Yuvraj Signed-off-by: Daniel Rammer --- .golangci.yml | 4 +- Makefile | 5 +- boilerplate/flyte/docker_build/Makefile | 12 + .../{lyft => flyte}/docker_build/Readme.rst | 6 +- .../docker_build/docker_build.sh | 4 +- .../golang_support_tools/go.mod | 8 +- boilerplate/flyte/golang_support_tools/go.sum | 1267 +++++++++++++++++ .../golang_support_tools/tools.go | 2 +- .../golang_test_targets/Makefile | 8 +- .../golang_test_targets/Readme.rst | 6 +- .../golang_test_targets/download_tooling.sh | 6 +- .../golang_test_targets/goimports | 4 +- .../golangci_file/.golangci.yml | 4 +- .../{lyft => flyte}/golangci_file/Readme.rst | 2 +- .../{lyft => flyte}/golangci_file/update.sh | 4 +- .../pull_request_template/Readme.rst | 2 +- .../pull_request_template.md | 10 +- .../pull_request_template/update.sh | 4 +- boilerplate/lyft/docker_build/Makefile | 12 - boilerplate/lyft/github_workflows/Readme.rst | 20 - boilerplate/lyft/github_workflows/master.yml | 142 -- .../lyft/github_workflows/pull_request.yml | 103 -- boilerplate/lyft/github_workflows/update.sh | 16 - .../golang_dockerfile/Dockerfile.GoTemplate | 32 - boilerplate/lyft/golang_dockerfile/Readme.rst | 16 - boilerplate/lyft/golang_dockerfile/update.sh | 13 - boilerplate/lyft/golang_support_tools/go.sum | 558 -------- boilerplate/update.cfg | 12 +- boilerplate/update.sh | 29 +- pull_request_template.md | 10 +- 30 files changed, 1358 insertions(+), 963 deletions(-) create mode 100644 boilerplate/flyte/docker_build/Makefile rename boilerplate/{lyft => flyte}/docker_build/Readme.rst (66%) rename boilerplate/{lyft => flyte}/docker_build/docker_build.sh (91%) rename boilerplate/{lyft => flyte}/golang_support_tools/go.mod (61%) create mode 100644 boilerplate/flyte/golang_support_tools/go.sum rename boilerplate/{lyft => flyte}/golang_support_tools/tools.go (78%) rename boilerplate/{lyft => flyte}/golang_test_targets/Makefile (82%) rename boilerplate/{lyft => flyte}/golang_test_targets/Readme.rst (75%) rename boilerplate/{lyft => flyte}/golang_test_targets/download_tooling.sh (89%) rename boilerplate/{lyft => flyte}/golang_test_targets/goimports (56%) rename boilerplate/{lyft => flyte}/golangci_file/.golangci.yml (72%) rename boilerplate/{lyft => flyte}/golangci_file/Readme.rst (60%) rename boilerplate/{lyft => flyte}/golangci_file/update.sh (67%) rename boilerplate/{lyft => flyte}/pull_request_template/Readme.rst (55%) rename boilerplate/{lyft => flyte}/pull_request_template/pull_request_template.md (54%) rename boilerplate/{lyft => flyte}/pull_request_template/update.sh (61%) delete mode 100644 boilerplate/lyft/docker_build/Makefile delete mode 100644 boilerplate/lyft/github_workflows/Readme.rst delete mode 100644 boilerplate/lyft/github_workflows/master.yml delete mode 100644 boilerplate/lyft/github_workflows/pull_request.yml delete mode 100755 boilerplate/lyft/github_workflows/update.sh delete mode 100644 boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate delete mode 100644 boilerplate/lyft/golang_dockerfile/Readme.rst delete mode 100755 boilerplate/lyft/golang_dockerfile/update.sh delete mode 100644 boilerplate/lyft/golang_support_tools/go.sum diff --git a/.golangci.yml b/.golangci.yml index a414f33f..5d53f352 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst run: skip-dirs: diff --git a/Makefile b/Makefile index 78837a98..d9b244f5 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ export REPOSITORY=datacatalog -include boilerplate/lyft/docker_build/Makefile -include boilerplate/lyft/golang_test_targets/Makefile +include boilerplate/flyte/docker_build/Makefile +include boilerplate/flyte/golang_test_targets/Makefile .PHONY: update_boilerplate update_boilerplate: + @curl https://raw.githubusercontent.com/flyteorg/boilerplate/master/boilerplate/update.sh -o boilerplate/update.sh @boilerplate/update.sh .PHONY: compile diff --git a/boilerplate/flyte/docker_build/Makefile b/boilerplate/flyte/docker_build/Makefile new file mode 100644 index 00000000..e2b2b8a1 --- /dev/null +++ b/boilerplate/flyte/docker_build/Makefile @@ -0,0 +1,12 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +.PHONY: docker_build +docker_build: + IMAGE_NAME=$$REPOSITORY ./boilerplate/flyte/docker_build/docker_build.sh + +.PHONY: dockerhub_push +dockerhub_push: + IMAGE_NAME=flyteorg/$$REPOSITORY REGISTRY=docker.io ./boilerplate/flyte/docker_build/docker_build.sh diff --git a/boilerplate/lyft/docker_build/Readme.rst b/boilerplate/flyte/docker_build/Readme.rst similarity index 66% rename from boilerplate/lyft/docker_build/Readme.rst rename to boilerplate/flyte/docker_build/Readme.rst index bb6af9b4..7790b8fb 100644 --- a/boilerplate/lyft/docker_build/Readme.rst +++ b/boilerplate/flyte/docker_build/Readme.rst @@ -11,13 +11,13 @@ If git head has a git tag, the Dockerhub image will also be tagged ``: - include boilerplate/lyft/docker_build/Makefile + include boilerplate/flyte/docker_build/Makefile (this ensures the extra Make targets get included in your main Makefile) diff --git a/boilerplate/lyft/docker_build/docker_build.sh b/boilerplate/flyte/docker_build/docker_build.sh similarity index 91% rename from boilerplate/lyft/docker_build/docker_build.sh rename to boilerplate/flyte/docker_build/docker_build.sh index f504c100..a77f8a2d 100755 --- a/boilerplate/lyft/docker_build/docker_build.sh +++ b/boilerplate/flyte/docker_build/docker_build.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst set -e diff --git a/boilerplate/lyft/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod similarity index 61% rename from boilerplate/lyft/golang_support_tools/go.mod rename to boilerplate/flyte/golang_support_tools/go.mod index 6816461a..7afee48e 100644 --- a/boilerplate/lyft/golang_support_tools/go.mod +++ b/boilerplate/flyte/golang_support_tools/go.mod @@ -1,11 +1,11 @@ -module github.com/lyft/boilerplate +module github.com/flyteorg/boilerplate -go 1.13 +go 1.16 require ( github.com/alvaroloes/enumer v1.1.2 - github.com/golangci/golangci-lint v1.22.2 - github.com/lyft/flytestdlib v0.2.31 + github.com/flyteorg/flytestdlib v0.3.22 + github.com/golangci/golangci-lint v1.38.0 github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 ) diff --git a/boilerplate/flyte/golang_support_tools/go.sum b/boilerplate/flyte/golang_support_tools/go.sum new file mode 100644 index 00000000..49939b68 --- /dev/null +++ b/boilerplate/flyte/golang_support_tools/go.sum @@ -0,0 +1,1267 @@ +4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw= +4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible h1:p7blnyJSjJqf5jflHbSGhIhEpXIgIFmYZNg5uwqweso= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.17 h1:2zCdHwNgRH+St1J+ZMf66xI8aLr/5KMy+wWLH97zwYM= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.10 h1:r6fZHMaHD8B6LDCn0o5vyBFHIHrM6Ywwx7mb49lPItI= +github.com/Azure/go-autorest/autorest/adal v0.9.10/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= +github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw= +github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= +github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY= +github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/ashanbrown/forbidigo v1.1.0 h1:SJOPJyqsrVL3CvR0veFZFmIM0fXS/Kvyikqvfphd0Z4= +github.com/ashanbrown/forbidigo v1.1.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBFg8t0sG2FIxmI= +github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a h1:/U9tbJzDRof4fOR51vwzWdIBsIH6R2yU0KG1MBRM2Js= +github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.37.1 h1:BTHmuN+gzhxkvU9sac2tZvaY0gV9ihbHw+KxZOecYvY= +github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1/go.mod h1:jvdWlw8vowVGnZqSDC7yhPd7AifQeQbRDkZcQXV2nRg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bkielbasa/cyclop v1.2.0 h1:7Jmnh0yL2DjKfw28p86YTd/B4lRGcNuu12sKE35sM7A= +github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI= +github.com/bombsimon/wsl/v3 v3.2.0 h1:x3QUbwW7tPGcCNridvqmhSRthZMTALnkg5/1J+vaUas= +github.com/bombsimon/wsl/v3 v3.2.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charithe/durationcheck v0.0.6 h1:Tsy7EppNow2pDC0jN7Hsmcb6mHd71ZbI1vFissRBtc0= +github.com/charithe/durationcheck v0.0.6/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= +github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg= +github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218= +github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0 h1:qxIJwfSemSCqhG3/lEw1Rm+wYbegjuKsqy0ZqnIpL14= +github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0/go.mod h1:KfdIkmkpVY3n2sc1ykFj01uMviOiXH2HMhUCvA5FYGg= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 h1:cTavhURetDkezJCvxFggiyLeP40Mrk/TtVg2+ycw1Es= +github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= +github.com/esimonov/ifshort v1.0.1 h1:p7hlWD15c9XwvwxYg3W7f7UZHmwg7l9hC0hBiF95gd0= +github.com/esimonov/ifshort v1.0.1/go.mod h1:yZqNJUrNn20K8Q9n2CrjTKYyVEmX209Hgu+M1LBpeZE= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/flyteorg/flytestdlib v0.3.22 h1:nJEPaCdxzXBaeg2p4fdo3I3Ua09NedFRaUwuLafLEdw= +github.com/flyteorg/flytestdlib v0.3.22/go.mod h1:1XG0DwYTUm34Yrffm1Qy9Tdr/pWQypEqTq5dUxw3/cM= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fzipp/gocyclo v0.3.1 h1:A9UeX3HJSXTBzvHzhqoYVuE0eAhe+aM8XBCCwsPMZOc= +github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.5.4 h1:fPNMqImVjELN6Du7NVVuvKA4cgASNmc7e4zSYQCOnv8= +github.com/go-critic/go-critic v0.5.4/go.mod h1:cjB4YGw+n/+X8gREApej7150Uyy1Tg8If6F2XOAUXNE= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk= +github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo= +github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= +github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.38.0 h1:hgZsLRzZrjhpp44Ak+fhXNzgrbDF39ETf22a+Jd3fJQ= +github.com/golangci/golangci-lint v1.38.0/go.mod h1:Knp/sd5ATrVp7EOzWzwIIFH+c8hUfpW+oOQb8NvdZDo= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.3.5 h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo= +github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 h1:c9Mqqrm/Clj5biNaG7rABrmwUq88nHh0uABo2b/WYmc= +github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gookit/color v1.3.6/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 h1:Nb2aRlC404yz7gQIfRZxX9/MLvQiqXyiBTJtgAy6yrI= +github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254/go.mod h1:M9mZEtGIsR1oDaZagNPNG9iq9n2HrhZ17dsXk73V3Lw= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw= +github.com/gostaticanalysis/analysisutil v0.4.1 h1:/7clKqrVfiVwiBQLM0Uke4KvXnO6JcCTS7HwF2D6wG8= +github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0= +github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI= +github.com/gostaticanalysis/comment v1.4.1 h1:xHopR5L2lRz6OsjH4R2HG5wRhW9ySl3FsHIvi5pcXwc= +github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado= +github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 h1:rx8127mFPqXXsfPSo8BwnIU97MKFZc89WHAHt8PwDVY= +github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5/go.mod h1:qZEedyP/sY1lTGV1uJ3VhWZ2mqag3IkWsDHVbplHXak= +github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3Uqrmrcpk= +github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= +github.com/graymeta/stow v0.2.7 h1:b31cB1Ylw/388sYSZxnmpjT2QxC21AaQ8fRnUtE13b4= +github.com/graymeta/stow v0.2.7/go.mod h1:JAs139Zr29qfsecy7b+h9DRsWXbFbsd7LCrbCDYI84k= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jgautheron/goconst v1.4.0 h1:hp9XKUpe/MPyDamUbfsrGpe+3dnY2whNK4EtB86dvLM= +github.com/jgautheron/goconst v1.4.0/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= +github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d h1:BYDZtm80MLJpTWalkwHxNnIbO/2akQHERcfLq4TbIWE= +github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d/go.mod h1:/EZlaYCnEX24i7qdVhT9du5JrtFWYRQr67bVgR7JJC8= +github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48= +github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0 h1:exZBMUS/kB/AhxSj/9lIIxhqkCpXXdKScjFWQUTbi3M= +github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/errcheck v1.6.0 h1:YTDO4pNy7AUN/021p+JGHycQyYNIyMoenM1YDVK6RlY= +github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kulti/thelper v0.4.0 h1:2Nx7XbdbE/BYZeoip2mURKUdtHQRuy6Ug+wR7K9ywNM= +github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U= +github.com/kunwardeep/paralleltest v1.0.2 h1:/jJRv0TiqPoEy/Y8dQxCFJhD56uS/pnvtatgTZBHokU= +github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30= +github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M= +github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= +github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= +github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ= +github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= +github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 h1:pWxk9e//NbPwfxat7RXkts09K+dEBJWakUWwICVqYbA= +github.com/matoous/godox v0.0.0-20210227103229-6504466cf951/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= +github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= +github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= +github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= +github.com/mgechev/revive v1.0.3 h1:z3FL6IFFN3JKzHYHD8O1ExH9g/4lAGJ5x1+9rPZgsFg= +github.com/mgechev/revive v1.0.3/go.mod h1:POGGZagSo/0frdr7VeAifzS5Uka0d0GPiM35MsTO8nE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4= +github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= +github.com/mozilla/tls-observatory v0.0.0-20201209171846-0547674fceff/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw= +github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88 h1:o+O3Cd1HO9CTgxE3/C8p5I5Y4C0yYWbF8d4IkfOLtcQ= +github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= +github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= +github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8= +github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ= +github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw= +github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U= +github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU= +github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4= +github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/quasilyte/go-ruleguard v0.3.0 h1:A3OfpsK2ynOTbz/KMi62qWzignjGCOZVChATSf4P+A0= +github.com/quasilyte/go-ruleguard v0.3.0/go.mod h1:p2miAhLp6fERzFNbcuQ4bevXs8rgK//uCHsUDkumITg= +github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210106184943-e47d54850b18/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210115110123-c73ee1cbff1f/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc= +github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY= +github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.2.0 h1:YWfhGOrXwLGiqcC/u5EqG6YeS8nh+1fw0HEc85CVZro= +github.com/ryancurrah/gomodguard v1.2.0/go.mod h1:rNqbC4TOIdUDcVMSIpNNAzTbzXAZa6W5lnUepvuMMgQ= +github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw= +github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sanposhiho/wastedassign v0.1.3 h1:qIMpTh4NGZYRbFJ+DSpLoVn8F4SLciX2afRvXPefC7w= +github.com/sanposhiho/wastedassign v0.1.3/go.mod h1:LGpq5Hsv74QaqM47WtIsRSF/ik9kqk07kchgv66tLVE= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/securego/gosec/v2 v2.6.1 h1:+KCw+uz16FYfFyJ/A5aU6uP7mnrL+j1TbDnk1yN+8R0= +github.com/securego/gosec/v2 v2.6.1/go.mod h1:I76p3NTHBXsGhybUW+cEQ692q2Vp+A0Z6ZLzDIZy+Ao= +github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= +github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= +github.com/shirou/gopsutil/v3 v3.21.1/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU= +github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY= +github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ= +github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= +github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA= +github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b h1:HxLVTlqcHhFAz3nWUcuvpH7WuOMv8LQoCWmruLfFH2U= +github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tetafro/godot v1.4.4 h1:VAtLEoAMmopIzHVWVBrztjVWDeYm1OD/DKqhqXR4828= +github.com/tetafro/godot v1.4.4/go.mod h1:FVDd4JuKliW3UgjswZfJfHq4vAx0bD/Jd5brJjGeaz4= +github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 h1:ig99OeTyDwQWhPe2iw9lwfQVF1KB3Q4fpP3X7/2VBG8= +github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 h1:zV5mu0ESwb+WnzqVaW2z1DdbAP0S46UtjY8DHQupQP4= +github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0= +github.com/tommy-muehle/go-mnd/v2 v2.3.1 h1:a1S4+4HSXDJMgeODJH/t0EEKxcVla6Tasw+Zx9JJMog= +github.com/tommy-muehle/go-mnd/v2 v2.3.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA= +github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= +github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs= +github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok= +go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 h1:55H5j7lotzuFCEOKDsMch+fRNUQ9DgtyHOUP31FNqKc= +golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181112210238-4b1f3b6b1646/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201028025901-8cd080b735b3/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210102185154-773b96fafca2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.38.0 h1:vDyWk6eup8eQAidaZ31sNWIn8tZEL8qpbtGkBD4ytQo= +google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 h1:uLBY0yHDCj2PMQ98KWDSIDFwn9zK2zh+tgWtbvPPBjI= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.2 h1:SMdYLJl312RXuxXziCCHhRsp/tvct9cGKey0yv95tZM= +honnef.co/go/tools v0.1.2/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +k8s.io/api v0.0.0-20210217171935-8e2decd92398/go.mod h1:60tmSUpHxGPFerNHbo/ayI2lKxvtrhbxFyXuEIWJd78= +k8s.io/apimachinery v0.0.0-20210217011835-527a61b4dffe/go.mod h1:Z7ps/g0rjlTeMstYrMOUttJfT2Gg34DEaG/f2PYLCWY= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 h1:d+LBRNY3c/KGp7lDblRlUJkayx4Vla7WUTIazoGMdYo= +k8s.io/client-go v0.0.0-20210217172142-7279fc64d847/go.mod h1:q0EaghmVye2uui19vxSZ2NG6ssgUWgjudO6vrwXneSI= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.5.0 h1:8mOnjf1RmUPW6KRqQCfYSZq/K20Unmp3IhuZUhxl8KI= +k8s.io/klog/v2 v2.5.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw= +mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 h1:HT3e4Krq+IE44tiN36RvVEb6tvqeIdtsVSsxmNPqlFU= +mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7/go.mod h1:hBpJkZE8H/sb+VRFvw2+rBpHNsTBcvSpk61hr8mzXZE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/boilerplate/lyft/golang_support_tools/tools.go b/boilerplate/flyte/golang_support_tools/tools.go similarity index 78% rename from boilerplate/lyft/golang_support_tools/tools.go rename to boilerplate/flyte/golang_support_tools/tools.go index 88ff6452..71163a57 100644 --- a/boilerplate/lyft/golang_support_tools/tools.go +++ b/boilerplate/flyte/golang_support_tools/tools.go @@ -4,7 +4,7 @@ package tools import ( _ "github.com/alvaroloes/enumer" + _ "github.com/flyteorg/flytestdlib/cli/pflags" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" - _ "github.com/lyft/flytestdlib/cli/pflags" _ "github.com/vektra/mockery/cmd/mockery" ) diff --git a/boilerplate/lyft/golang_test_targets/Makefile b/boilerplate/flyte/golang_test_targets/Makefile similarity index 82% rename from boilerplate/lyft/golang_test_targets/Makefile rename to boilerplate/flyte/golang_test_targets/Makefile index 5abd2ed6..21d8b5b7 100644 --- a/boilerplate/lyft/golang_test_targets/Makefile +++ b/boilerplate/flyte/golang_test_targets/Makefile @@ -1,12 +1,12 @@ # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst .PHONY: download_tooling download_tooling: #download dependencies (including test deps) for the package - @boilerplate/lyft/golang_test_targets/download_tooling.sh + @boilerplate/flyte/golang_test_targets/download_tooling.sh .PHONY: lint lint: download_tooling #lints the package for common code smells @@ -16,7 +16,7 @@ lint: download_tooling #lints the package for common code smells # skips 'vendor' .PHONY: goimports goimports: - @boilerplate/lyft/golang_test_targets/goimports + @boilerplate/flyte/golang_test_targets/goimports .PHONY: mod_download mod_download: #download dependencies (including test deps) for the package diff --git a/boilerplate/lyft/golang_test_targets/Readme.rst b/boilerplate/flyte/golang_test_targets/Readme.rst similarity index 75% rename from boilerplate/lyft/golang_test_targets/Readme.rst rename to boilerplate/flyte/golang_test_targets/Readme.rst index 3466e30c..f9d890fd 100644 --- a/boilerplate/lyft/golang_test_targets/Readme.rst +++ b/boilerplate/flyte/golang_test_targets/Readme.rst @@ -15,17 +15,17 @@ Provides a ``test_benchmark`` target for benchmark tests. **To Enable:** -Add ``lyft/golang_test_targets`` to your ``boilerplate/update.cfg`` file. +Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file. Make sure you're using ``go mod`` for dependency management. Provide a ``.golangci`` configuration (the lint target requires it). -Add ``include boilerplate/lyft/golang_test_targets/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable +Add ``include boilerplate/flyte/golang_test_targets/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable :: REPOSITORY= - include boilerplate/lyft/golang_test_targets/Makefile + include boilerplate/flyte/golang_test_targets/Makefile (this ensures the extra make targets get included in your main Makefile) diff --git a/boilerplate/lyft/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh similarity index 89% rename from boilerplate/lyft/golang_test_targets/download_tooling.sh rename to boilerplate/flyte/golang_test_targets/download_tooling.sh index ab56c7e4..25d4ac3d 100755 --- a/boilerplate/lyft/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -7,7 +7,7 @@ # pins. To facilitate this, we're maintaining two sets of go.mod/sum files - the second one only for tooling. This is # the same approach that go 1.14 will take as well. # See: -# https://github.com/lyft/flyte/issues/129 +# https://github.com/flyteorg/flyte/issues/129 # https://github.com/golang/go/issues/30515 for some background context # https://github.com/go-modules-by-example/index/blob/5ec250b4b78114a55001bd7c9cb88f6e07270ea5/010_tools/README.md @@ -17,14 +17,14 @@ set -e # In the format of ":" or ":" if no cli tools=( "github.com/vektra/mockery/cmd/mockery" - "github.com/lyft/flytestdlib/cli/pflags" + "github.com/flyteorg/flytestdlib/cli/pflags" "github.com/golangci/golangci-lint/cmd/golangci-lint" "github.com/alvaroloes/enumer" ) tmp_dir=$(mktemp -d -t gotooling-XXX) echo "Using temp directory ${tmp_dir}" -cp -R boilerplate/lyft/golang_support_tools/* $tmp_dir +cp -R boilerplate/flyte/golang_support_tools/* $tmp_dir pushd "$tmp_dir" for tool in "${tools[@]}" diff --git a/boilerplate/lyft/golang_test_targets/goimports b/boilerplate/flyte/golang_test_targets/goimports similarity index 56% rename from boilerplate/lyft/golang_test_targets/goimports rename to boilerplate/flyte/golang_test_targets/goimports index 160525a8..ba0d6d87 100755 --- a/boilerplate/lyft/golang_test_targets/goimports +++ b/boilerplate/flyte/golang_test_targets/goimports @@ -1,8 +1,8 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*") diff --git a/boilerplate/lyft/golangci_file/.golangci.yml b/boilerplate/flyte/golangci_file/.golangci.yml similarity index 72% rename from boilerplate/lyft/golangci_file/.golangci.yml rename to boilerplate/flyte/golangci_file/.golangci.yml index a414f33f..5d53f352 100644 --- a/boilerplate/lyft/golangci_file/.golangci.yml +++ b/boilerplate/flyte/golangci_file/.golangci.yml @@ -1,7 +1,7 @@ # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst run: skip-dirs: diff --git a/boilerplate/lyft/golangci_file/Readme.rst b/boilerplate/flyte/golangci_file/Readme.rst similarity index 60% rename from boilerplate/lyft/golangci_file/Readme.rst rename to boilerplate/flyte/golangci_file/Readme.rst index ba5d2b61..e4cbd18b 100644 --- a/boilerplate/lyft/golangci_file/Readme.rst +++ b/boilerplate/flyte/golangci_file/Readme.rst @@ -5,4 +5,4 @@ Provides a ``.golangci`` file with the linters we've agreed upon. **To Enable:** -Add ``lyft/golangci_file`` to your ``boilerplate/update.cfg`` file. +Add ``flyteorg/golangci_file`` to your ``boilerplate/update.cfg`` file. diff --git a/boilerplate/lyft/golangci_file/update.sh b/boilerplate/flyte/golangci_file/update.sh similarity index 67% rename from boilerplate/lyft/golangci_file/update.sh rename to boilerplate/flyte/golangci_file/update.sh index 9e9e6c1f..ab2f85c6 100755 --- a/boilerplate/lyft/golangci_file/update.sh +++ b/boilerplate/flyte/golangci_file/update.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst set -e diff --git a/boilerplate/lyft/pull_request_template/Readme.rst b/boilerplate/flyte/pull_request_template/Readme.rst similarity index 55% rename from boilerplate/lyft/pull_request_template/Readme.rst rename to boilerplate/flyte/pull_request_template/Readme.rst index b85a4ea1..ee544372 100644 --- a/boilerplate/lyft/pull_request_template/Readme.rst +++ b/boilerplate/flyte/pull_request_template/Readme.rst @@ -5,4 +5,4 @@ Provides a Pull Request template. **To Enable:** -Add ``lyft/golang_test_targets`` to your ``boilerplate/update.cfg`` file. +Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file. diff --git a/boilerplate/lyft/pull_request_template/pull_request_template.md b/boilerplate/flyte/pull_request_template/pull_request_template.md similarity index 54% rename from boilerplate/lyft/pull_request_template/pull_request_template.md rename to boilerplate/flyte/pull_request_template/pull_request_template.md index d2becf38..d7699558 100644 --- a/boilerplate/lyft/pull_request_template/pull_request_template.md +++ b/boilerplate/flyte/pull_request_template/pull_request_template.md @@ -1,3 +1,9 @@ +## Read then delete + +- Make sure to use a concise title for the pull-request. +- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version +will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump) + # TL;DR _Please replace this text with a description of what this PR accomplishes._ @@ -18,9 +24,9 @@ _Please replace this text with a description of what this PR accomplishes._ _How did you fix the bug, make the feature etc. Link to any design docs etc_ ## Tracking Issue -https://github.com/lyft/flyte/issues/ +https://github.com/flyteorg/flyte/issues/ ## Follow-up issue _NA_ OR -_https://github.com/lyft/flyte/issues/_ +_https://github.com/flyteorg/flyte/issues/_ diff --git a/boilerplate/lyft/pull_request_template/update.sh b/boilerplate/flyte/pull_request_template/update.sh similarity index 61% rename from boilerplate/lyft/pull_request_template/update.sh rename to boilerplate/flyte/pull_request_template/update.sh index 13f0c3b5..051e9dbc 100755 --- a/boilerplate/lyft/pull_request_template/update.sh +++ b/boilerplate/flyte/pull_request_template/update.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst set -e diff --git a/boilerplate/lyft/docker_build/Makefile b/boilerplate/lyft/docker_build/Makefile deleted file mode 100644 index 4019dab8..00000000 --- a/boilerplate/lyft/docker_build/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: -# -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst - -.PHONY: docker_build -docker_build: - IMAGE_NAME=$$REPOSITORY ./boilerplate/lyft/docker_build/docker_build.sh - -.PHONY: dockerhub_push -dockerhub_push: - IMAGE_NAME=lyft/$$REPOSITORY REGISTRY=docker.io ./boilerplate/lyft/docker_build/docker_build.sh diff --git a/boilerplate/lyft/github_workflows/Readme.rst b/boilerplate/lyft/github_workflows/Readme.rst deleted file mode 100644 index d32ba7cb..00000000 --- a/boilerplate/lyft/github_workflows/Readme.rst +++ /dev/null @@ -1,20 +0,0 @@ -Golang Github Actions -~~~~~~~~~~~~~~~~~ - -Provides a two github actions workflows. - -**To Enable:** - -Add ``lyft/github_workflows`` to your ``boilerplate/update.cfg`` file. - -Add a github secret ``flytegithub_repo`` with a the name of your fork (e.g. ``my_company/flytepropeller``). - -The actions will push to 2 repos: - - 1. ``docker.pkg.github.com/lyft//operator`` - 2. ``docker.pkg.github.com/lyft//operator-stages`` : this repo is used to cache build stages to speed up iterative builds after. - -There are two workflows that get deployed: - - 1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha. - 2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest" diff --git a/boilerplate/lyft/github_workflows/master.yml b/boilerplate/lyft/github_workflows/master.yml deleted file mode 100644 index e87a40dc..00000000 --- a/boilerplate/lyft/github_workflows/master.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - # Duplicated from pull request workflow because sharing is not yet supported - build-docker: - name: Build Docker Image - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - id: load-docker-cache - name: Load Docker Cache - uses: actions/cache@v1 - with: - path: /tmp/tmp/docker-images - key: /tmp/docker-images-${{ github.event.after }} - restore-keys: | - /tmp/docker-images-${{ github.event.before }} - /tmp/docker-images-${{ github.event.pull_request.base.sha }} - - name: Prime docker cache - run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) - - name: Build dockerfile - run: | - docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . - docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . - - - name: Tag and cache docker image - run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar - - # Duplicated from pull request workflow because sharing is not yet supported - endtoend: - name: End to End tests - runs-on: ubuntu-latest - needs: [build-docker] - steps: - - name: Checkout - uses: actions/checkout@v2 - - id: load-docker-cache - name: Load Docker Cache - uses: actions/cache@v1 - with: - path: /tmp/tmp/docker-images - key: /tmp/docker-images-${{ github.event.after }} - - name: Prime docker cache - run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.5.0 - - name: End2End - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute - - bump-version: - name: Bump Version - if: github.actor != 'goreleaserbot' - runs-on: ubuntu-latest - needs: build-docker # Only to ensure it can successfully build - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump-version - uses: anothrNick/github-tag-action@1.17.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - - goreleaser: - name: Goreleaser - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.14 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} - - push-github: - name: Push to Github Registry - runs-on: ubuntu-latest - needs: bump-version - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}" - password: "${{ secrets.PUBLISH_CONTAINER }}" - image_name: ${{ github.repository }} - image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }} - push_git_tag: true - registry: ghcr.io - build_extra_args: "--compress=true" - - tests-lint: - name: Run tests and lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Unit Tests - uses: cedrickring/golang-action@1.5.2 - env: - GO111MODULE: "on" - with: - args: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: true - - name: Lint - uses: cedrickring/golang-action@1.5.2 - env: - GO111MODULE: "on" - with: - args: make install && make lint diff --git a/boilerplate/lyft/github_workflows/pull_request.yml b/boilerplate/lyft/github_workflows/pull_request.yml deleted file mode 100644 index 1f6448c8..00000000 --- a/boilerplate/lyft/github_workflows/pull_request.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Pull Request - -on: - pull_request - -jobs: - build-docker: - name: Build Docker Image - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - id: load-docker-cache - name: Load Docker Cache - uses: actions/cache@v1 - with: - path: /tmp/tmp/docker-images - key: /tmp/docker-images-${{ github.event.after }} - restore-keys: | - /tmp/docker-images-${{ github.event.before }} - /tmp/docker-images-${{ github.event.pull_request.base.sha }} - - name: Prime docker cache - run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) - - name: Build dockerfile - run: | - docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder . - docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder . - - - name: Tag and cache docker image - run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar - - endtoend: - name: End to End tests - runs-on: ubuntu-latest - needs: [build-docker] - steps: - - name: Checkout - uses: actions/checkout@v2 - - id: load-docker-cache - name: Load Docker Cache - uses: actions/cache@v1 - with: - path: /tmp/tmp/docker-images - key: /tmp/docker-images-${{ github.event.after }} - - name: Prime docker cache - run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true - - uses: engineerd/setup-kind@v0.5.0 - - name: End2End - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute - - integration: - name: Integration tests - runs-on: ubuntu-latest - needs: [build-docker] - steps: - - name: Checkout - uses: actions/checkout@v2 - - id: load-docker-cache - name: Load Docker Cache - uses: actions/cache@v1 - with: - path: /tmp/tmp/docker-images - key: /tmp/docker-images-${{ github.event.after }} - - name: Prime docker cache - run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true - - uses: engineerd/setup-kind@v0.5.0 - - name: Integration - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:builder make k8s_integration_execute - - tests-lint: - name: Run tests and lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Unit Tests - uses: cedrickring/golang-action@1.5.2 - env: - GO111MODULE: "on" - with: - args: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: true - - name: Lint - uses: cedrickring/golang-action@1.5.2 - env: - GO111MODULE: "on" - with: - args: make install && make lint diff --git a/boilerplate/lyft/github_workflows/update.sh b/boilerplate/lyft/github_workflows/update.sh deleted file mode 100755 index 1e3a0991..00000000 --- a/boilerplate/lyft/github_workflows/update.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: -# -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst - -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -mkdir -p ${DIR}/../../../.github/workflows - -echo " - generating github action workflows in root directory." -sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml -sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml diff --git a/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate b/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate deleted file mode 100644 index 214d1e40..00000000 --- a/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate +++ /dev/null @@ -1,32 +0,0 @@ -# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: -# -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst - -FROM golang:1.13.3-alpine3.10 as builder -RUN apk add git openssh-client make curl - -# COPY only the go mod files for efficient caching -COPY go.mod go.sum /go/src/github.com/lyft/{{REPOSITORY}}/ -WORKDIR /go/src/github.com/lyft/{{REPOSITORY}} - -# Pull dependencies -RUN go mod download - -# COPY the rest of the source code -COPY . /go/src/github.com/lyft/{{REPOSITORY}}/ - -# This 'linux_compile' target should compile binaries to the /artifacts directory -# The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} -RUN make linux_compile - -# update the PATH to include the /artifacts directory -ENV PATH="/artifacts:${PATH}" - -# This will eventually move to centurylink/ca-certs:latest for minimum possible image size -FROM alpine:3.10 -COPY --from=builder /artifacts /bin - -RUN apk --update add ca-certificates - -CMD ["{{REPOSITORY}}"] diff --git a/boilerplate/lyft/golang_dockerfile/Readme.rst b/boilerplate/lyft/golang_dockerfile/Readme.rst deleted file mode 100644 index f801ef98..00000000 --- a/boilerplate/lyft/golang_dockerfile/Readme.rst +++ /dev/null @@ -1,16 +0,0 @@ -Golang Dockerfile -~~~~~~~~~~~~~~~~~ - -Provides a Dockerfile that produces a small image. - -**To Enable:** - -Add ``lyft/golang_dockerfile`` to your ``boilerplate/update.cfg`` file. - -Create and configure a ``make linux_compile`` target that compiles your go binaries to the ``/artifacts`` directory :: - - .PHONY: linux_compile - linux_compile: - RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts {{ packages }} - -All binaries compiled to ``/artifacts`` will be available at ``/bin`` in your final image. diff --git a/boilerplate/lyft/golang_dockerfile/update.sh b/boilerplate/lyft/golang_dockerfile/update.sh deleted file mode 100755 index 7d846632..00000000 --- a/boilerplate/lyft/golang_dockerfile/update.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: -# -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst - -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -echo " - generating Dockerfile in root directory." -sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/Dockerfile.GoTemplate > ${DIR}/../../../Dockerfile diff --git a/boilerplate/lyft/golang_support_tools/go.sum b/boilerplate/lyft/golang_support_tools/go.sum deleted file mode 100644 index 81e04696..00000000 --- a/boilerplate/lyft/golang_support_tools/go.sum +++ /dev/null @@ -1,558 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.47.0 h1:1JUtpcY9E7+eTospEwWS2QXP3DEn7poB3E2j0jN74mM= -cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v10.2.1-beta+incompatible h1:/x4W7ZQV4PHJYnLUgKubojM8T+zlFEDdaBazAnA/QCY= -github.com/Azure/azure-sdk-for-go v10.2.1-beta+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest/autorest v0.9.2 h1:6AWuh3uWrsZJcNoCHrCF/+g4aKPCU39kaMO6/qrnK/4= -github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= -github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY= -github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.25.16 h1:k7Fy6T/uNuLX6zuayU/TJoP7yMgGcJSkZpF7QVjwYpA= -github.com/aws/aws-sdk-go v1.25.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1/go.mod h1:jvdWlw8vowVGnZqSDC7yhPd7AifQeQbRDkZcQXV2nRg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bombsimon/wsl/v2 v2.0.0 h1:+Vjcn+/T5lSrO8Bjzhk4v14Un/2UyCA1E3V5j9nwTkQ= -github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= -github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0 h1:qxIJwfSemSCqhG3/lEw1Rm+wYbegjuKsqy0ZqnIpL14= -github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0/go.mod h1:KfdIkmkpVY3n2sc1ykFj01uMviOiXH2HMhUCvA5FYGg= -github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 h1:cTavhURetDkezJCvxFggiyLeP40Mrk/TtVg2+ycw1Es= -github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structtag v1.1.0 h1:6j4mUV/ES2duvnAzKMFkN6/A5mCaNYPD3xfbAkLLOF8= -github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.8-0.20191012010759-4bf2d1fec783 h1:SmsgwFZy9pdTk/k8BZz40D3P5umP5+Ejt3hAi0paBNQ= -github.com/fsnotify/fsnotify v1.4.8-0.20191012010759-4bf2d1fec783/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-critic/go-critic v0.4.0 h1:sXD3pix0wDemuPuSlrXpJNNYXlUiKiysLrtPVQmxkzI= -github.com/go-critic/go-critic v0.4.0/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw= -github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= -github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.22.2 h1:iaihss3Tf6NvZVjun3lHimKSgofPV1+FqE/cbehoiRQ= -github.com/golangci/golangci-lint v1.22.2/go.mod h1:2Bj42k6hPQFTRxkDb7S3TQ+EsnumZXOmIYNqlQrp0FI= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= -github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/graymeta/stow v0.0.0-20190522170649-903027f87de7 h1:oIYi27Ruo2k5dxukdOisCSZrowkg70jxxuaPZck9+ic= -github.com/graymeta/stow v0.0.0-20190522170649-903027f87de7/go.mod h1:B24dekNjtWVeREK+dyMHtI22d85VzCT+sX5bVWDtjoA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/lyft/flytestdlib v0.2.31 h1:JAOSGwy/wLprhq1KR9zxekBqnKdSlAQQG1x4KQe+hlI= -github.com/lyft/flytestdlib v0.2.31/go.mod h1:/fqNXKCGChEvMzcRapVq6vDM69Vlusl+bCj7foToaUQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= -github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/ncw/swift v1.0.49-0.20190728102658-a24ef33bc9b7 h1:2FttUGr9cza0JNOUDHeVWo4wVGk92m8bralgdbAeYJY= -github.com/ncw/swift v1.0.49-0.20190728102658-a24ef33bc9b7/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU= -github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.5.0 h1:5BakdOZdtKJ1FFk6QdL8iSGrMWsXgchNJcrnarjbmJQ= -github.com/pelletier/go-toml v1.5.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/satori/uuid v1.2.0 h1:6TFY4nxn5XwBx0gDfzbEMCNT6k4N/4FNIuN8RACZ0KI= -github.com/satori/uuid v1.2.0/go.mod h1:B8HLsPLik/YNn6KKWVMDJ8nzCL8RP5WyfsnmvnAEwIU= -github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= -github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= -github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= -github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tommy-muehle/go-mnd v1.1.1 h1:4D0wuPKjOTiK2garzuPGGvm4zZ/wLYDOH8TJSABC7KU= -github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= -github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= -github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= -github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs= -github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= -github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191021144547-ec77196f6094 h1:5O4U9trLjNpuhpynaDsqwCk+Tw6seqJz1EbqbnzHrc8= -golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2 h1:nq114VpM8lsSlP+lyUbANecYHYiFcSNFtqcBlxRV+gA= -golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= -golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181112210238-4b1f3b6b1646/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113232020-e2727e816f5a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191204011308-9611592c72f6 h1:BP62y4oUl8+/CvHuvVqHIPmVRixgDl6y6a+tR7pXXIA= -golang.org/x/tools v0.0.0-20191204011308-9611592c72f6/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.11.1-0.20191020000718-bf72a15fd9e9 h1:DcDldKT1PaijNBNDNPaGtfl+LvUHR1xsw5DpipSI1CE= -google.golang.org/api v0.11.1-0.20191020000718-bf72a15fd9e9/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao= -k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 h1:OH3z6khCtxnJBAc0C5CMYWLl1CoK5R5fngX7wrwdN5c= -k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/boilerplate/update.cfg b/boilerplate/update.cfg index f9eb7cc7..9d3bd305 100644 --- a/boilerplate/update.cfg +++ b/boilerplate/update.cfg @@ -1,8 +1,6 @@ -lyft/docker_build -lyft/golang_test_targets -lyft/golangci_file -lyft/golang_dockerfile -lyft/golang_support_tools -lyft/pull_request_template -lyft/github_workflows +flyte/docker_build +flyte/golang_test_targets +flyte/golangci_file +flyte/golang_support_tools +flyte/pull_request_template flyte/ diff --git a/boilerplate/update.sh b/boilerplate/update.sh index 374e4b93..2d0fd711 100755 --- a/boilerplate/update.sh +++ b/boilerplate/update.sh @@ -1,23 +1,27 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" OUT="$(mktemp -d)" +<<<<<<< HEAD +trap 'rm -fr $OUT' EXIT + +git clone https://github.com/flyteorg/boilerplate.git "${OUT}" +======= trap "rm -fr $OUT" EXIT git clone git@github.com:flyteorg/boilerplate.git "${OUT}" +>>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 echo "Updating the update.sh script." cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh" -echo "" - CONFIG_FILE="${DIR}/update.cfg" README="https://github.com/flyteorg/boilerplate/blob/master/Readme.rst" @@ -30,12 +34,16 @@ if [ ! -f "$CONFIG_FILE" ]; then fi if [ -z "$REPOSITORY" ]; then - echo '$REPOSITORY is required to run this script' + echo "$REPOSITORY is required to run this script" echo "See $README for more details." exit 1 fi +<<<<<<< HEAD +while read -r directory junk; do +======= while read directory junk; do +>>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 # Skip comment lines (which can have leading whitespace) if [[ "$directory" == '#'* ]]; then continue @@ -63,8 +71,13 @@ while read directory junk; do echo "$directory is configured in update.cfg." echo "-----------------------------------------------------------------------------------" echo "syncing files from source." +<<<<<<< HEAD + rm -rf "${DIR:?}/${directory}" + mkdir -p "$(dirname "${DIR}"/"${directory}")" +======= rm -rf "${DIR}/${directory}" mkdir -p $(dirname "${DIR}/${directory}") +>>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 cp -r "$dir_path" "${DIR}/${directory}" if [ -f "${DIR}/${directory}/update.sh" ]; then echo "executing ${DIR}/${directory}/update.sh" @@ -72,4 +85,8 @@ while read directory junk; do fi echo "***********************************************************************************" echo "" -done < "$CONFIG_FILE" \ No newline at end of file +<<<<<<< HEAD +done < "$CONFIG_FILE" +======= +done < "$CONFIG_FILE" +>>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 diff --git a/pull_request_template.md b/pull_request_template.md index d2becf38..d7699558 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,3 +1,9 @@ +## Read then delete + +- Make sure to use a concise title for the pull-request. +- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version +will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump) + # TL;DR _Please replace this text with a description of what this PR accomplishes._ @@ -18,9 +24,9 @@ _Please replace this text with a description of what this PR accomplishes._ _How did you fix the bug, make the feature etc. Link to any design docs etc_ ## Tracking Issue -https://github.com/lyft/flyte/issues/ +https://github.com/flyteorg/flyte/issues/ ## Follow-up issue _NA_ OR -_https://github.com/lyft/flyte/issues/_ +_https://github.com/flyteorg/flyte/issues/_ From c600d56934fdaaee1be9cea46c9b1031e3883f18 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 1 Jun 2021 17:02:11 -0700 Subject: [PATCH 21/58] add more instructinos Use upsert Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 105 ++++++++---------- pkg/manager/impl/reservation_manager_test.go | 30 +++-- pkg/repositories/errors/errors.go | 7 +- pkg/repositories/gormimpl/reservation.go | 59 ++++------ pkg/repositories/gormimpl/reservation_test.go | 76 +------------ .../interfaces/reservation_repo.go | 4 +- pkg/repositories/mocks/reservation.go | 67 +++-------- 7 files changed, 106 insertions(+), 242 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index e0a1faab..f097cf03 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -11,6 +11,7 @@ import ( errors2 "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories" + repo_errors "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/datacatalog/pkg/repositories/transformers" @@ -21,10 +22,9 @@ import ( type reservationMetrics struct { scope promutils.Scope - reservationAcquiredViaCreate labeled.Counter - reservationAcquiredViaUpdate labeled.Counter + reservationAcquired labeled.Counter reservationAlreadyInProgress labeled.Counter - makeReservationFailure labeled.Counter + acquireReservationFailure labeled.Counter getTagFailure labeled.Counter } @@ -45,22 +45,18 @@ func NewReservationManager( ) interfaces.ReservationManager { systemMetrics := reservationMetrics{ scope: reservationScope, - reservationAcquiredViaCreate: labeled.NewCounter( - "reservation_acquired_via_create", - "Number of times a reservation was acquired via create", - reservationScope), - reservationAcquiredViaUpdate: labeled.NewCounter( - "reservation_acquired_via_update", - "Number of times a reservation was acquired via update", + reservationAcquired: labeled.NewCounter( + "reservation_acquired", + "Number of times a reservation was acquired", reservationScope), reservationAlreadyInProgress: labeled.NewCounter( "reservation_already_in_progress", "Number of times we try of acquire a reservation but the reservation is in progress", reservationScope, ), - makeReservationFailure: labeled.NewCounter( - "make_reservation_failure", - "Number of times we failed to make reservation", + acquireReservationFailure: labeled.NewCounter( + "acquire_reservation_failure", + "Number of times we failed to acquire reservation", reservationScope, ), getTagFailure: labeled.NewCounter( @@ -83,11 +79,11 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * tag, err := r.repo.TagRepo().Get(ctx, tagKey) if err != nil { if errors2.IsDoesNotExistError(err) { - // Tag does not exist yet, let's reserve a spot to work on + // Tag does not exist yet, let's acquire the reservation to work on // generating the artifact. - status, err := r.makeReservation(ctx, request) + status, err := r.tryAcquireReservation(ctx, request) if err != nil { - r.systemMetrics.makeReservationFailure.Inc(ctx) + r.systemMetrics.acquireReservationFailure.Inc(ctx) return nil, err } @@ -114,60 +110,57 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * }, nil } -func (r *reservationManager) makeReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { +func (r *reservationManager) tryAcquireReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.ToReservationKey(*request.DatasetId, request.TagName) rsv, err := repo.Get(ctx, reservationKey) + resvExists := true if err != nil { if errors2.IsDoesNotExistError(err) { // Reservation does not exist yet so let's create one - err := repo.Create(ctx, models.Reservation{ - ReservationKey: reservationKey, - OwnerID: request.OwnerId, - ExpireAt: r.now().Add(r.reservationTimeout), - }) - - if err != nil { - logger.Errorf(ctx, "Failed to create reservation: %+v, err %v", reservationKey, err) - - return datacatalog.ReservationStatus{}, err - } - - r.systemMetrics.reservationAcquiredViaCreate.Inc(ctx) - - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, - }, nil + resvExists = false + } else { + return datacatalog.ReservationStatus{}, err } - return datacatalog.ReservationStatus{}, err } now := r.now() - // Reservation already exists so there is a task already working on it - // Let's check if the reservation is expired. - if rsv.ExpireAt.Before(now) { - // The reservation is expired, let's try to grab the reservation - rowsAffected, err := repo.Update(ctx, reservationKey, - rsv.ExpireAt, - now.Add(r.reservationTimeout), request.OwnerId) + if !resvExists || rsv.ExpireAt.Before(now) { + // If the reservation does not exist or it is expired, + // we try to acquire the reservation + err := repo.CreateOrUpdate( + ctx, + models.Reservation{ + ReservationKey: reservationKey, + OwnerID: request.OwnerId, + ExpireAt: now.Add(r.reservationTimeout), + }, + now, + ) + if err != nil { - return datacatalog.ReservationStatus{}, err + if err.Error() == repo_errors.ReservationAlreadyInProgress { + // Looks like someone else tried to obtain the reservation + // at the same time and they won. Let's find out who won. + rsv1, err1 := repo.Get(ctx, reservationKey) + if err1 != nil { + return datacatalog.ReservationStatus{}, err1 + } + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: rsv1.OwnerID, + }, err + } } - // If rows affected is 0, possibly someone else is also - // trying to grab the reservation at the same time and - // they won. - if rowsAffected > 0 { - r.systemMetrics.reservationAcquiredViaUpdate.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, - }, nil - } - logger.Debugf(ctx, "Failed to reserve a spot: %+v, owner: %s", reservationKey, - request.OwnerId) + r.systemMetrics.reservationAcquired.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, + }, nil } logger.Debugf(ctx, "Reservation: %+v is hold by %s", reservationKey, rsv.OwnerID) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index fba49666..74582212 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -97,7 +97,7 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { now := time.Now() - dcRepo.MockReservationRepo.On("Create", + dcRepo.MockReservationRepo.On("CreateOrUpdate", mock.MatchedBy(func(ctx context.Context) bool { return true }), mock.MatchedBy(func(reservation models.Reservation) bool { return reservation.DatasetProject == datasetID.Project && @@ -108,6 +108,7 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { reservation.OwnerID == currentOwner && reservation.ExpireAt == now.Add(timeout) }), + mock.MatchedBy(func(now time.Time) bool { return true }), ).Return(nil) reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, @@ -136,24 +137,19 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { setUpReservationRepoGet(&dcRepo, prevExpireAt) - dcRepo.MockReservationRepo.On("Update", + dcRepo.MockReservationRepo.On("CreateOrUpdate", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(key models.ReservationKey) bool { - return key.DatasetProject == datasetID.Project && - key.DatasetDomain == datasetID.Domain && - key.DatasetVersion == datasetID.Version && - key.DatasetName == datasetID.Name && - key.TagName == tagName - }), - mock.MatchedBy(func(expireAt time.Time) bool { - return expireAt.Equal(prevExpireAt) - }), - mock.MatchedBy(func(expireAt time.Time) bool { - return expireAt.Equal(now.Add(timeout)) + mock.MatchedBy(func(reservation models.Reservation) bool { + return reservation.DatasetProject == datasetID.Project && + reservation.DatasetDomain == datasetID.Domain && + reservation.DatasetName == datasetID.Name && + reservation.DatasetVersion == datasetID.Version && + reservation.TagName == tagName && + reservation.OwnerID == currentOwner && + reservation.ExpireAt == now.Add(timeout) }), - mock.MatchedBy(func(ownerID string) bool { - return ownerID == currentOwner - })).Return(int64(1), nil) + mock.MatchedBy(func(now time.Time) bool { return true }), + ).Return(nil) reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, mockScope.NewTestScope()) diff --git a/pkg/repositories/errors/errors.go b/pkg/repositories/errors/errors.go index c497192e..81d66b34 100644 --- a/pkg/repositories/errors/errors.go +++ b/pkg/repositories/errors/errors.go @@ -9,9 +9,10 @@ import ( ) const ( - notFound = "missing entity of type %s with identifier %v" - invalidJoin = "cannot relate entity %s with entity %s" - invalidEntity = "no such entity %s" + notFound = "missing entity of type %s with identifier %v" + invalidJoin = "cannot relate entity %s with entity %s" + invalidEntity = "no such entity %s" + ReservationAlreadyInProgress = "reservation already in progress" ) func GetMissingEntityError(entityType string, identifier proto.Message) error { diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 3ada2cb2..ae884ed4 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -3,14 +3,17 @@ package gormimpl import ( "context" + datacatalog_error "github.com/flyteorg/datacatalog/pkg/errors" + "google.golang.org/grpc/codes" + + "time" + errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" - errors3 "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/promutils" "gorm.io/gorm" "gorm.io/gorm/clause" - "time" ) type reservationRepo struct { @@ -19,7 +22,7 @@ type reservationRepo struct { errorTransformer errors2.ErrorTransformer } -func NewReservationRepo(db *gorm.DB, errorTransformer errors3.ErrorTransformer, scope promutils.Scope) interfaces.ReservationRepo { +func NewReservationRepo(db *gorm.DB, errorTransformer errors2.ErrorTransformer, scope promutils.Scope) interfaces.ReservationRepo { return &reservationRepo{ db: db, errorTransformer: errorTransformer, @@ -27,16 +30,6 @@ func NewReservationRepo(db *gorm.DB, errorTransformer errors3.ErrorTransformer, } } -func (r *reservationRepo) Create(ctx context.Context, reservation models.Reservation) error { - timer := r.repoMetrics.CreateDuration.Start(ctx) - defer timer.Stop() - result := r.db.Create(reservation) - if result.Error != nil { - return r.errorTransformer.ToDataCatalogError(result.Error) - } - return nil -} - func (r *reservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { timer := r.repoMetrics.GetDuration.Start(ctx) defer timer.Stop() @@ -54,41 +47,27 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva return reservation, nil } -func (r *reservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, ownerID string) (int64, error) { - timer := r.repoMetrics.UpdateDuration.Start(ctx) - defer timer.Stop() - - result := r.db.Where( - &models.Reservation{ - ReservationKey: reservationKey, - ExpireAt: prevExpireAt, - }, - ).Updates( - models.Reservation{ - OwnerID: ownerID, - ExpireAt: expireAt, - }) - if result.Error != nil { - return 0, r.errorTransformer.ToDataCatalogError(result.Error) - } - - return result.RowsAffected, nil -} - -func (r *reservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) (int64, error) { +func (r *reservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error { expressions := make([]clause.Expression, 0) expressions = append(expressions, clause.Lte{Column: "expire_at", Value: now}) result := r.db.Clauses( clause.OnConflict{ - Where: clause.Where{Exprs: expressions}, + Where: clause.Where{Exprs: expressions}, UpdateAll: true, }, - ).Create(&reservation) + ).Create(&reservation) if result.Error != nil { - return 0, r.errorTransformer.ToDataCatalogError(result.Error) + return r.errorTransformer.ToDataCatalogError(result.Error) + } + + if result.RowsAffected == 0 { + return datacatalog_error.NewDataCatalogError( + codes.FailedPrecondition, + errors2.ReservationAlreadyInProgress, + ) } - return result.RowsAffected, nil -} \ No newline at end of file + return nil +} diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 0b81f495..bce0ae1c 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -3,14 +3,14 @@ package gormimpl import ( "context" "database/sql" - "database/sql/driver" "fmt" - "gorm.io/driver/postgres" - "gorm.io/gorm" "testing" "time" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" apiErrors "github.com/flyteorg/datacatalog/pkg/errors" @@ -23,48 +23,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCreate(t *testing.T) { - reservation := GetReservation() - - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - - reservationCreated := false - GlobalMock.NewMock().WithQuery( - `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?,?)`).WithCallback( - func(s string, values []driver.NamedValue) { - reservationCreated = true - }, - ) - - reservationRepo := getReservationRepo(t) - - err := reservationRepo.Create(context.Background(), reservation) - assert.Nil(t, err) - - assert.True(t, reservationCreated) -} - -func TestCreateAlreadyExists(t *testing.T) { - reservation := GetReservation() - - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - - GlobalMock.NewMock().WithQuery( - `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?,?)`).WithError( - getAlreadyExistsErr(), - ) - - reservationRepo := getReservationRepo(t) - - err := reservationRepo.Create(context.Background(), reservation) - assert.NotNil(t, err) - dcErr, ok := err.(apiErrors.DataCatalogError) - assert.True(t, ok) - assert.Equal(t, dcErr.Code(), codes.AlreadyExists) -} - func TestGet(t *testing.T) { expectedReservation := GetReservation() @@ -103,27 +61,6 @@ func TestGetNotFound(t *testing.T) { } -func TestUpdate(t *testing.T) { - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - - GlobalMock.NewMock().WithQuery( - `UPDATE "" SET "expire_at" = ?, "owner_id" = ? WHERE ("reservations"."dataset_project" = ?) AND ("reservations"."dataset_name" = ?) AND ("reservations"."dataset_domain" = ?) AND ("reservations"."dataset_version" = ?) AND ("reservations"."tag_name" = ?) AND ("reservations"."expire_at" = ?)`, - ).WithRowsNum(1) - - reservationRepo := getReservationRepo(t) - - reservationKey := GetReservationKey() - prevExpireAt := time.Now() - expireAt := prevExpireAt.Add(time.Second * 50) - ownerID := "hello" - - rows, err := reservationRepo.Update(context.Background(), reservationKey, prevExpireAt, expireAt, ownerID) - - assert.Nil(t, err) - assert.Equal(t, rows, int64(1)) -} - func TestCreateOrUpdate(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true @@ -135,9 +72,8 @@ func TestCreateOrUpdate(t *testing.T) { reservationRepo := getReservationRepo(t) - rows, err := reservationRepo.CreateOrUpdate(context.Background(), expectedReservation, time.Now()) - assert.Nil(t, err) - assert.Equal(t, int64(1), rows) + err := reservationRepo.CreateOrUpdate(context.Background(), expectedReservation, time.Now()) + assert.NoError(t, err) } func getReservationRepo(t *testing.T) interfaces.ReservationRepo { @@ -145,7 +81,7 @@ func getReservationRepo(t *testing.T) interfaces.ReservationRepo { sqlDB, err := sql.Open(mocket.DriverName, "blah") assert.Nil(t, err) - db, err := gorm.Open(postgres.New(postgres.Config{Conn:sqlDB})) + db, err := gorm.Open(postgres.New(postgres.Config{Conn: sqlDB})) if err != nil { t.Fatal(fmt.Sprintf("Failed to open mock db with err %v", err)) } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index ab70ca8d..2b951d77 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -8,8 +8,6 @@ import ( ) type ReservationRepo interface { - Create(ctx context.Context, reservation models.Reservation) error Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) (int64, error) - CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) (int64, error) + CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error } diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go index e724ffb5..7e91542a 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation.go @@ -17,31 +17,31 @@ type ReservationRepo struct { mock.Mock } -type ReservationRepo_Create struct { +type ReservationRepo_CreateOrUpdate struct { *mock.Call } -func (_m ReservationRepo_Create) Return(_a0 error) *ReservationRepo_Create { - return &ReservationRepo_Create{Call: _m.Call.Return(_a0)} +func (_m ReservationRepo_CreateOrUpdate) Return(_a0 error) *ReservationRepo_CreateOrUpdate { + return &ReservationRepo_CreateOrUpdate{Call: _m.Call.Return(_a0)} } -func (_m *ReservationRepo) OnCreate(ctx context.Context, reservation models.Reservation) *ReservationRepo_Create { - c := _m.On("Create", ctx, reservation) - return &ReservationRepo_Create{Call: c} +func (_m *ReservationRepo) OnCreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_CreateOrUpdate { + c := _m.On("CreateOrUpdate", ctx, reservation, now) + return &ReservationRepo_CreateOrUpdate{Call: c} } -func (_m *ReservationRepo) OnCreateMatch(matchers ...interface{}) *ReservationRepo_Create { - c := _m.On("Create", matchers...) - return &ReservationRepo_Create{Call: c} +func (_m *ReservationRepo) OnCreateOrUpdateMatch(matchers ...interface{}) *ReservationRepo_CreateOrUpdate { + c := _m.On("CreateOrUpdate", matchers...) + return &ReservationRepo_CreateOrUpdate{Call: c} } -// Create provides a mock function with given fields: ctx, reservation -func (_m *ReservationRepo) Create(ctx context.Context, reservation models.Reservation) error { - ret := _m.Called(ctx, reservation) +// CreateOrUpdate provides a mock function with given fields: ctx, reservation, now +func (_m *ReservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error { + ret := _m.Called(ctx, reservation, now) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.Reservation) error); ok { - r0 = rf(ctx, reservation) + if rf, ok := ret.Get(0).(func(context.Context, models.Reservation, time.Time) error); ok { + r0 = rf(ctx, reservation, now) } else { r0 = ret.Error(0) } @@ -87,42 +87,3 @@ func (_m *ReservationRepo) Get(ctx context.Context, reservationKey models.Reserv return r0, r1 } - -type ReservationRepo_Update struct { - *mock.Call -} - -func (_m ReservationRepo_Update) Return(_a0 int64, _a1 error) *ReservationRepo_Update { - return &ReservationRepo_Update{Call: _m.Call.Return(_a0, _a1)} -} - -func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) *ReservationRepo_Update { - c := _m.On("Update", ctx, reservationKey, prevExpireAt, expireAt, OwnerID) - return &ReservationRepo_Update{Call: c} -} - -func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRepo_Update { - c := _m.On("Update", matchers...) - return &ReservationRepo_Update{Call: c} -} - -// Update provides a mock function with given fields: ctx, reservationKey, prevExpireAt, expireAt, OwnerID -func (_m *ReservationRepo) Update(ctx context.Context, reservationKey models.ReservationKey, prevExpireAt time.Time, expireAt time.Time, OwnerID string) (int64, error) { - ret := _m.Called(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) - - var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey, time.Time, time.Time, string) int64); ok { - r0 = rf(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.ReservationKey, time.Time, time.Time, string) error); ok { - r1 = rf(ctx, reservationKey, prevExpireAt, expireAt, OwnerID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} From 8e1b319a04c8259c89061b7f653a987491875515 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 1 Jun 2021 17:52:24 -0700 Subject: [PATCH 22/58] add more comments Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/interfaces/reservation.go | 3 +++ pkg/repositories/interfaces/reservation_repo.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/pkg/manager/interfaces/reservation.go b/pkg/manager/interfaces/reservation.go index e89419a0..cb43ef28 100644 --- a/pkg/manager/interfaces/reservation.go +++ b/pkg/manager/interfaces/reservation.go @@ -6,6 +6,9 @@ import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" ) +// ReservationManager is the interface to handle reservation requests. +// You can find more details about the APIs in datacatalog service proto +// in flyteidl type ReservationManager interface { GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 2b951d77..4ed715db 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -7,7 +7,13 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +// Interface to interact with Reservation Table type ReservationRepo interface { + + // Get reservation Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) + + // Create the reservation. If the reservation already exists, we try to take over the + // reservation via update. CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error } From 84a7ecb12d8b2ce2d1edb6e2866fe4f0eea0f1a6 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 1 Jun 2021 18:05:54 -0700 Subject: [PATCH 23/58] refactor a bit Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 34 +++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index f097cf03..194563b6 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -50,7 +50,7 @@ func NewReservationManager( "Number of times a reservation was acquired", reservationScope), reservationAlreadyInProgress: labeled.NewCounter( - "reservation_already_in_progress", + "reservation_alrea dy_in_progress", "Number of times we try of acquire a reservation but the reservation is in progress", reservationScope, ), @@ -139,21 +139,29 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request now, ) + failedToAcquireReservation := false if err != nil { if err.Error() == repo_errors.ReservationAlreadyInProgress { - // Looks like someone else tried to obtain the reservation - // at the same time and they won. Let's find out who won. - rsv1, err1 := repo.Get(ctx, reservationKey) - if err1 != nil { - return datacatalog.ReservationStatus{}, err1 - } - - r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: rsv1.OwnerID, - }, err + + failedToAcquireReservation = true + } else { + return datacatalog.ReservationStatus{}, err + } + } + + if failedToAcquireReservation { + // Looks like someone else tried to obtain the reservation + // at the same time and they won. Let's find out who won. + rsv1, err := repo.Get(ctx, reservationKey) + if err != nil { + return datacatalog.ReservationStatus{}, err } + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: rsv1.OwnerID, + }, err } r.systemMetrics.reservationAcquired.Inc(ctx) From 6a1e6d2d79fbd72546efe9bb16d2f3a99fc3a6c2 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 1 Jun 2021 18:08:02 -0700 Subject: [PATCH 24/58] add timer Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/repositories/gormimpl/metrics.go | 6 +++--- pkg/repositories/gormimpl/reservation.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index d4ba332d..1f4450f5 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -13,7 +13,7 @@ type gormMetrics struct { CreateDuration labeled.StopWatch GetDuration labeled.StopWatch ListDuration labeled.StopWatch - UpdateDuration labeled.StopWatch + CreateOrUpdateDuration labeled.StopWatch } func newGormMetrics(scope promutils.Scope) gormMetrics { @@ -25,7 +25,7 @@ func newGormMetrics(scope promutils.Scope) gormMetrics { "get", "Duration for retrieving an entity ", time.Millisecond, scope), ListDuration: labeled.NewStopWatch( "list", "Duration for listing entities ", time.Millisecond, scope), - UpdateDuration: labeled.NewStopWatch( - "update", "Duration for updating entities ", time.Millisecond, scope), + CreateOrUpdateDuration: labeled.NewStopWatch( + "update", "Duration for creating/updating entities ", time.Millisecond, scope), } } diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index ae884ed4..fbbd2d14 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -48,6 +48,8 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva } func (r *reservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error { + timer := r.repoMetrics.CreateOrUpdateDuration.Start(ctx) + defer timer.Stop() expressions := make([]clause.Expression, 0) expressions = append(expressions, clause.Lte{Column: "expire_at", Value: now}) From 95b87163136c678993f8a6caf7d4235232f112ff Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 1 Jun 2021 18:22:01 -0700 Subject: [PATCH 25/58] fix lint / tests Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 2 +- pkg/repositories/gormimpl/metrics.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 194563b6..9e61ab42 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -50,7 +50,7 @@ func NewReservationManager( "Number of times a reservation was acquired", reservationScope), reservationAlreadyInProgress: labeled.NewCounter( - "reservation_alrea dy_in_progress", + "reservation_already_in_progress", "Number of times we try of acquire a reservation but the reservation is in progress", reservationScope, ), diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index 1f4450f5..27253011 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -9,10 +9,10 @@ import ( // Common metrics for DB CRUD operations type gormMetrics struct { - Scope promutils.Scope - CreateDuration labeled.StopWatch - GetDuration labeled.StopWatch - ListDuration labeled.StopWatch + Scope promutils.Scope + CreateDuration labeled.StopWatch + GetDuration labeled.StopWatch + ListDuration labeled.StopWatch CreateOrUpdateDuration labeled.StopWatch } From 8fa9951cbd7cfca4f68cbabcc183f2bacf163d43 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 8 Jun 2021 15:10:10 -0700 Subject: [PATCH 26/58] add comments & tests Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/repositories/models/reservation.go | 7 ++++++ .../transformers/reservation_test.go | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 pkg/repositories/transformers/reservation_test.go diff --git a/pkg/repositories/models/reservation.go b/pkg/repositories/models/reservation.go index d0e580bb..fab376dc 100644 --- a/pkg/repositories/models/reservation.go +++ b/pkg/repositories/models/reservation.go @@ -2,6 +2,7 @@ package models import "time" +// ReservationKey uniquely identifies a reservation type ReservationKey struct { DatasetProject string `gorm:"primary_key"` DatasetName string `gorm:"primary_key"` @@ -10,10 +11,16 @@ type ReservationKey struct { TagName string `gorm:"primary_key"` } +// Reservation tracks the metadata needed to allow +// task cache serialization type Reservation struct { BaseModel ReservationKey + + // Identifies who owns the reservation OwnerID string + + // When the reservation will expire ExpireAt time.Time SerializedMetadata []byte } diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go new file mode 100644 index 00000000..140a50b3 --- /dev/null +++ b/pkg/repositories/transformers/reservation_test.go @@ -0,0 +1,24 @@ +package transformers + +import ( + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/magiconair/properties/assert" + "testing" +) + + +func TestToReservationKey(t *testing.T) { + datasetID := datacatalog.DatasetID{ + Project: "p", + Name: "n", + Domain: "d", + Version: "v", + } + + reservationKey := ToReservationKey(datasetID, "t") + assert.Equal(t, datasetID.Project, reservationKey.DatasetProject) + assert.Equal(t, datasetID.Name, reservationKey.DatasetName) + assert.Equal(t, datasetID.Domain, reservationKey.DatasetDomain) + assert.Equal(t, datasetID.Version, reservationKey.DatasetVersion) + assert.Equal(t, "t", reservationKey.TagName) +} \ No newline at end of file From bf8405c21d3ea7e6ea45a02b7f454a0adb403426 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 8 Jun 2021 15:22:58 -0700 Subject: [PATCH 27/58] fix lint Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- go.mod | 1 + pkg/repositories/models/reservation.go | 2 +- pkg/repositories/transformers/reservation_test.go | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 525755f4..7b3f9698 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 github.com/lib/pq v1.3.0 + github.com/magiconair/properties v1.8.4 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 diff --git a/pkg/repositories/models/reservation.go b/pkg/repositories/models/reservation.go index fab376dc..53abf771 100644 --- a/pkg/repositories/models/reservation.go +++ b/pkg/repositories/models/reservation.go @@ -18,7 +18,7 @@ type Reservation struct { ReservationKey // Identifies who owns the reservation - OwnerID string + OwnerID string // When the reservation will expire ExpireAt time.Time diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go index 140a50b3..998c025b 100644 --- a/pkg/repositories/transformers/reservation_test.go +++ b/pkg/repositories/transformers/reservation_test.go @@ -1,18 +1,18 @@ package transformers import ( + "testing" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/magiconair/properties/assert" - "testing" ) - func TestToReservationKey(t *testing.T) { datasetID := datacatalog.DatasetID{ - Project: "p", - Name: "n", - Domain: "d", - Version: "v", + Project: "p", + Name: "n", + Domain: "d", + Version: "v", } reservationKey := ToReservationKey(datasetID, "t") @@ -21,4 +21,4 @@ func TestToReservationKey(t *testing.T) { assert.Equal(t, datasetID.Domain, reservationKey.DatasetDomain) assert.Equal(t, datasetID.Version, reservationKey.DatasetVersion) assert.Equal(t, "t", reservationKey.TagName) -} \ No newline at end of file +} From 931b1d38092195a1a557713bba58ab50a795b206 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 8 Jun 2021 15:43:56 -0700 Subject: [PATCH 28/58] add docs Signed-off-by: Chao-Han Tsai Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 6 ++++++ pkg/repositories/gormimpl/reservation.go | 1 + pkg/repositories/interfaces/reservation_repo.go | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 9e61ab42..3605a2d1 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -110,6 +110,12 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * }, nil } +// tryAcquireReservation will fetch the reservation first and only create/update +// the reservation if it does not exist or has expired. +// This is an optimization to reduce the number of writes to db. We always need +// to do a GET here because we want to know who owns the reservation +// and show it to users on the UI. However, the reservation is held by a single +// task most of the times and there is no need to do a write. func (r *reservationManager) tryAcquireReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.ToReservationKey(*request.DatasetId, request.TagName) diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index fbbd2d14..b3e9a4e4 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -22,6 +22,7 @@ type reservationRepo struct { errorTransformer errors2.ErrorTransformer } +// NewReservationRepo creates a reservationRepo func NewReservationRepo(db *gorm.DB, errorTransformer errors2.ErrorTransformer, scope promutils.Scope) interfaces.ReservationRepo { return &reservationRepo{ db: db, diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 4ed715db..57836252 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -14,6 +14,8 @@ type ReservationRepo interface { Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) // Create the reservation. If the reservation already exists, we try to take over the - // reservation via update. + // reservation via update when the reservation has expired. Note: Each reservation has its own + // expire date which is tracked in expire_at column in the reservation table. And the + // reservation expires when the date stored in expire_at column is in the past. CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error } From 571315c4c3ac9d60c801d90fb71d6caf9dd0f3e3 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Tue, 8 Jun 2021 11:49:48 -0700 Subject: [PATCH 29/58] Fix connection error handling (#45) Signed-off-by: Daniel Rammer --- cmd/entrypoints/migrate.go | 16 +++++++++++++--- go.mod | 1 + pkg/repositories/errors/postgres.go | 15 +++++++++++---- pkg/repositories/gormimpl/tag_test.go | 21 +++++++++++++++++++-- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/cmd/entrypoints/migrate.go b/cmd/entrypoints/migrate.go index 96635381..91067ce6 100644 --- a/cmd/entrypoints/migrate.go +++ b/cmd/entrypoints/migrate.go @@ -1,11 +1,15 @@ package entrypoints import ( + "reflect" + "github.com/flyteorg/datacatalog/pkg/repositories" + errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/runtime" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" - "github.com/lib/pq" + + "github.com/jackc/pgconn" "context" @@ -38,8 +42,14 @@ var migrateCmd = &cobra.Command{ if err != nil { // if db does not exist, try creating it - pqError, ok := err.(*pq.Error) - if ok && pqError.Code == pqInvalidDBCode { + cErr, ok := err.(errors2.ConnectError) + if !ok { + logger.Errorf(ctx, "Failed to cast error of type: %v, err: %v", reflect.TypeOf(err), + err) + panic(err) + } + pqError := cErr.Unwrap().(*pgconn.PgError) + if pqError.Code == pqInvalidDBCode { logger.Warningf(ctx, "Database [%v] does not exist, trying to create it now", dbName) dbConfigValues.DbName = defaultDB diff --git a/go.mod b/go.mod index 7b3f9698..4279b1ba 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/golang/protobuf v1.4.3 github.com/lib/pq v1.3.0 github.com/magiconair/properties v1.8.4 + github.com/jackc/pgconn v1.8.1 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index d68a9321..bbdd7d59 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -3,8 +3,9 @@ package errors import ( "fmt" + "github.com/jackc/pgconn" + "github.com/flyteorg/datacatalog/pkg/errors" - "github.com/lib/pq" "google.golang.org/grpc/codes" "gorm.io/gorm" ) @@ -20,7 +21,7 @@ type postgresErrorTransformer struct { const ( unexpectedType = "unexpected error type for: %v" - uniqueConstraintViolation = "value with matching %s already exists (%s)" + uniqueConstraintViolation = "value with matching already exists (%s)" defaultPgError = "failed database operation with %s" unsupportedTableOperation = "cannot query with specified table attributes: %s" ) @@ -35,13 +36,14 @@ func (p *postgresErrorTransformer) fromGormError(err error) error { } func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { - pqError, ok := err.(*pq.Error) + cErr, ok := err.(ConnectError) if !ok { return p.fromGormError(err) } + pqError := cErr.Unwrap().(*pgconn.PgError) switch pqError.Code { case uniqueConstraintViolationCode: - return errors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Constraint, pqError.Message) + return errors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) case undefinedTable: return errors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) default: @@ -52,3 +54,8 @@ func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { func NewPostgresErrorTransformer() ErrorTransformer { return &postgresErrorTransformer{} } + +type ConnectError interface { + Unwrap() error + Error() string +} diff --git a/pkg/repositories/gormimpl/tag_test.go b/pkg/repositories/gormimpl/tag_test.go index c7c9f3b1..ee33417a 100644 --- a/pkg/repositories/gormimpl/tag_test.go +++ b/pkg/repositories/gormimpl/tag_test.go @@ -3,6 +3,8 @@ package gormimpl import ( "testing" + "github.com/jackc/pgconn" + "gorm.io/gorm" "context" @@ -20,7 +22,6 @@ import ( "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/promutils" "github.com/flyteorg/flytestdlib/promutils/labeled" - "github.com/lib/pq" "google.golang.org/grpc/codes" ) @@ -28,8 +29,24 @@ func init() { labeled.SetMetricKeys(contextutils.AppNameKey) } +type pgError struct { + e error + msg string +} + +func (p *pgError) Error() string { + return p.msg +} + +func (p *pgError) Unwrap() error { + return p.e +} + func getAlreadyExistsErr() error { - return &pq.Error{Code: "23505"} + return &pgError{ + e: &pgconn.PgError{Code: "23505"}, + msg: "some error", + } } func getTestTag() models.Tag { From 9cb44720bfc42fb77961cf0fcda680ac6cf69efa Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 29 Jul 2021 09:46:56 +0530 Subject: [PATCH 30/58] Update code of conduct (#46) Signed-off-by: Ketan Umare Signed-off-by: Daniel Rammer --- CODE_OF_CONDUCT.md | 2 +- .../flyte/code_of_conduct/CODE_OF_CONDUCT.md | 2 + boilerplate/flyte/code_of_conduct/README.rst | 2 + boilerplate/flyte/code_of_conduct/update.sh | 12 ++++ boilerplate/flyte/end2end/Makefile | 12 ++++ boilerplate/flyte/end2end/end2end.sh | 34 +++++++++++ .../flyte/flyte_golang_compile/Readme.rst | 16 +++++ .../flyte_golang_compile.Template | 26 ++++++++ .../flyte/flyte_golang_compile/update.sh | 13 ++++ boilerplate/flyte/github_workflows/Readme.rst | 22 +++++++ .../boilerplate_automation.yml | 36 +++++++++++ boilerplate/flyte/github_workflows/master.yml | 31 ++++++++++ .../flyte/github_workflows/pull_request.yml | 19 ++++++ boilerplate/flyte/github_workflows/update.sh | 16 +++++ .../golang_dockerfile/Dockerfile.GoTemplate | 32 ++++++++++ .../flyte/golang_dockerfile/Readme.rst | 16 +++++ boilerplate/flyte/golang_dockerfile/update.sh | 13 ++++ boilerplate/flyte/golang_support_tools/go.mod | 3 + boilerplate/flyte/golang_support_tools/go.sum | 26 ++++++++ .../flyte/golang_support_tools/tools.go | 1 + .../golang_test_targets/download_tooling.sh | 1 + .../flyte/golang_test_targets/goimports | 2 +- boilerplate/flyte/precommit/Makefile | 9 +++ boilerplate/flyte/precommit/hooks/pre-push | 41 +++++++++++++ .../flyte/precommit/hooks/prepare-commit-msg | 16 +++++ boilerplate/flyte/precommit/update.sh | 60 +++++++++++++++++++ .../pull_request_template.md | 13 ++-- .../flyte/{ => welcome_bot}/Readme.rst | 0 .../flyte/{ => welcome_bot}/config.yml | 0 boilerplate/flyte/{ => welcome_bot}/update.sh | 7 ++- boilerplate/update.sh | 19 ------ pull_request_template.md | 13 ++-- 32 files changed, 481 insertions(+), 34 deletions(-) create mode 100644 boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md create mode 100644 boilerplate/flyte/code_of_conduct/README.rst create mode 100755 boilerplate/flyte/code_of_conduct/update.sh create mode 100644 boilerplate/flyte/end2end/Makefile create mode 100755 boilerplate/flyte/end2end/end2end.sh create mode 100644 boilerplate/flyte/flyte_golang_compile/Readme.rst create mode 100644 boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template create mode 100755 boilerplate/flyte/flyte_golang_compile/update.sh create mode 100644 boilerplate/flyte/github_workflows/Readme.rst create mode 100644 boilerplate/flyte/github_workflows/boilerplate_automation.yml create mode 100644 boilerplate/flyte/github_workflows/master.yml create mode 100644 boilerplate/flyte/github_workflows/pull_request.yml create mode 100755 boilerplate/flyte/github_workflows/update.sh create mode 100644 boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate create mode 100644 boilerplate/flyte/golang_dockerfile/Readme.rst create mode 100755 boilerplate/flyte/golang_dockerfile/update.sh create mode 100644 boilerplate/flyte/precommit/Makefile create mode 100755 boilerplate/flyte/precommit/hooks/pre-push create mode 100755 boilerplate/flyte/precommit/hooks/prepare-commit-msg create mode 100755 boilerplate/flyte/precommit/update.sh rename boilerplate/flyte/{ => welcome_bot}/Readme.rst (100%) rename boilerplate/flyte/{ => welcome_bot}/config.yml (100%) rename boilerplate/flyte/{ => welcome_bot}/update.sh (55%) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 4c3a38cc..e12139d6 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,2 +1,2 @@ -This project is governed by [Lyft's code of conduct](https://github.com/lyft/code-of-conduct). +This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). All contributors and participants agree to abide by its terms. diff --git a/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md b/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..e12139d6 --- /dev/null +++ b/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md @@ -0,0 +1,2 @@ +This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). +All contributors and participants agree to abide by its terms. diff --git a/boilerplate/flyte/code_of_conduct/README.rst b/boilerplate/flyte/code_of_conduct/README.rst new file mode 100644 index 00000000..0c9f2f1e --- /dev/null +++ b/boilerplate/flyte/code_of_conduct/README.rst @@ -0,0 +1,2 @@ +CODE OF CONDUCT +~~~~~~~~~~~~~~~ diff --git a/boilerplate/flyte/code_of_conduct/update.sh b/boilerplate/flyte/code_of_conduct/update.sh new file mode 100755 index 00000000..42f61584 --- /dev/null +++ b/boilerplate/flyte/code_of_conduct/update.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +cp ${DIR}/CODE_OF_CONDUCT.md ${DIR}/../../../CODE_OF_CONDUCT.md diff --git a/boilerplate/flyte/end2end/Makefile b/boilerplate/flyte/end2end/Makefile new file mode 100644 index 00000000..b0eb945b --- /dev/null +++ b/boilerplate/flyte/end2end/Makefile @@ -0,0 +1,12 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +.PHONY: end2end_execute +end2end_execute: + ./boilerplate/flyte/end2end/end2end.sh + +.PHONY: k8s_integration_execute +k8s_integration_execute: + echo "pass" \ No newline at end of file diff --git a/boilerplate/flyte/end2end/end2end.sh b/boilerplate/flyte/end2end/end2end.sh new file mode 100755 index 00000000..b2492de0 --- /dev/null +++ b/boilerplate/flyte/end2end/end2end.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +OUT="${DIR}/tmp" +rm -rf ${OUT} +git clone https://github.com/flyteorg/flyte.git "${OUT}" + +pushd ${OUT} + +if [ ! -z "$IMAGE" ]; +then + kind load docker-image ${IMAGE} + if [ ${IMAGE_NAME} -eq "flytepropeller" ] + then + sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/propeller/deployment.yaml + fi + + if [ ${IMAGE} -eq "flyteadmin" ] + then + sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/admindeployment/deployment.yaml + fi +fi + +make kustomize +make end2end_execute +popd diff --git a/boilerplate/flyte/flyte_golang_compile/Readme.rst b/boilerplate/flyte/flyte_golang_compile/Readme.rst new file mode 100644 index 00000000..e6b56dd1 --- /dev/null +++ b/boilerplate/flyte/flyte_golang_compile/Readme.rst @@ -0,0 +1,16 @@ +Flyte Golang Compile +~~~~~~~~~~~~~~~~~~~~ + +Common compile script for Flyte golang services. + +**To Enable:** + +Add ``flyteorg/flyte_golang_compile`` to your ``boilerplate/update.cfg`` file. + +Add the following to your Makefile + +:: + + .PHONY: compile_linux + compile_linux: + PACKAGES={{ *your packages }} OUTPUT={{ /path/to/output }} ./boilerplate/flyte/flyte_golang_compile.sh diff --git a/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template b/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template new file mode 100644 index 00000000..f587e971 --- /dev/null +++ b/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template @@ -0,0 +1,26 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +if [ -z "$PACKAGES" ]; then + echo "PACKAGES environment VAR not set" + exit 1 +fi + +if [ -z "$OUTPUT" ]; then + echo "OUTPUT environment VAR not set" + exit 1 +fi + +# get the GIT_SHA and RELEASE_SEMVER + +GIT_SHA=$(git rev-parse HEAD) +RELEASE_SEMVER=$(git describe --tags --exact-match $GIT_SHA 2>/dev/null) + +CURRENT_PKG=github.com/flyteorg/{{ REPOSITORY }} +VERSION_PKG="${CURRENT_PKG}/vendor/github.com/flyteorg/flytestdlib" + +LDFLAGS="-X ${VERSION_PKG}/version.Build=${GIT_SHA} -X ${VERSION_PKG}/version.Version=${RELEASE_SEMVER}" + +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "$LDFLAGS" -o "$OUTPUT" "$PACKAGES" diff --git a/boilerplate/flyte/flyte_golang_compile/update.sh b/boilerplate/flyte/flyte_golang_compile/update.sh new file mode 100755 index 00000000..b1e6101c --- /dev/null +++ b/boilerplate/flyte/flyte_golang_compile/update.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +echo " - generating ${DIR}/flyte_golang_compile.sh" +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/flyte_golang_compile.Template > ${DIR}/flyte_golang_compile.sh diff --git a/boilerplate/flyte/github_workflows/Readme.rst b/boilerplate/flyte/github_workflows/Readme.rst new file mode 100644 index 00000000..905ddd81 --- /dev/null +++ b/boilerplate/flyte/github_workflows/Readme.rst @@ -0,0 +1,22 @@ +Golang Github Actions +~~~~~~~~~~~~~~~~~ + +Provides a two github actions workflows. + +**To Enable:** + +Add ``flyteorg/github_workflows`` to your ``boilerplate/update.cfg`` file. + +Add a github secret ``package_name`` with the name to use for publishing (e.g. ``flytepropeller``). Typicaly, this will be the same name as the repository. + +*Note*: If you are working on a fork, include that prefix in your package name (``myfork/flytepropeller``). + +The actions will push to 2 repos: + + 1. ``docker.pkg.github.com/flyteorg//`` + 2. ``docker.pkg.github.com/flyteorg//-stages`` : this repo is used to cache build stages to speed up iterative builds after. + +There are two workflows that get deployed: + + 1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha. + 2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest" diff --git a/boilerplate/flyte/github_workflows/boilerplate_automation.yml b/boilerplate/flyte/github_workflows/boilerplate_automation.yml new file mode 100644 index 00000000..9bda305d --- /dev/null +++ b/boilerplate/flyte/github_workflows/boilerplate_automation.yml @@ -0,0 +1,36 @@ +name: Update Boilerplate Automation +on: + workflow_dispatch: +jobs: + update-boilerplate: + name: Update Boilerplate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Update Boilerplate + run: | + make update_boilerplate + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.FLYTE_BOT_PAT }} + commit-message: Update Boilerplate + committer: Flyte-Bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: true + branch: flyte-bot-update-boilerplate + delete-branch: true + title: 'Update Boilerplate' + body: | + Update Boilerplate + - Auto-generated by [flyte-bot] + labels: | + boilerplate + team-reviewers: | + owners + maintainers + draft: false + diff --git a/boilerplate/flyte/github_workflows/master.yml b/boilerplate/flyte/github_workflows/master.yml new file mode 100644 index 00000000..a8eab5af --- /dev/null +++ b/boilerplate/flyte/github_workflows/master.yml @@ -0,0 +1,31 @@ +name: Master + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: '0' + - name: Bump version and push tag + id: bump-version + uses: anothrNick/github-tag-action@1.17.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: patch + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.package_name }} + image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }} + push_git_tag: true + registry: docker.pkg.github.com + build_extra_args: "--compress=true" diff --git a/boilerplate/flyte/github_workflows/pull_request.yml b/boilerplate/flyte/github_workflows/pull_request.yml new file mode 100644 index 00000000..932400bc --- /dev/null +++ b/boilerplate/flyte/github_workflows/pull_request.yml @@ -0,0 +1,19 @@ +name: Pull Request + +on: + pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.package_name }} + image_tag: ${{ github.sha }} + push_git_tag: true + registry: docker.pkg.github.com diff --git a/boilerplate/flyte/github_workflows/update.sh b/boilerplate/flyte/github_workflows/update.sh new file mode 100755 index 00000000..d5a74a4f --- /dev/null +++ b/boilerplate/flyte/github_workflows/update.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +mkdir -p ${DIR}/../../../.github/workflows + +echo " - generating github action workflows in root directory." +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml diff --git a/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate new file mode 100644 index 00000000..2b5b775c --- /dev/null +++ b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate @@ -0,0 +1,32 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +FROM golang:1.13.3-alpine3.10 as builder +RUN apk add git openssh-client make curl + +# COPY only the go mod files for efficient caching +COPY go.mod go.sum /go/src/github.com/flyteorg/{{REPOSITORY}}/ +WORKDIR /go/src/github.com/flyteorg/{{REPOSITORY}} + +# Pull dependencies +RUN go mod download + +# COPY the rest of the source code +COPY . /go/src/github.com/flyteorg/{{REPOSITORY}}/ + +# This 'linux_compile' target should compile binaries to the /artifacts directory +# The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} +RUN make linux_compile + +# update the PATH to include the /artifacts directory +ENV PATH="/artifacts:${PATH}" + +# This will eventually move to centurylink/ca-certs:latest for minimum possible image size +FROM alpine:3.10 +COPY --from=builder /artifacts /bin + +RUN apk --update add ca-certificates + +CMD ["{{REPOSITORY}}"] diff --git a/boilerplate/flyte/golang_dockerfile/Readme.rst b/boilerplate/flyte/golang_dockerfile/Readme.rst new file mode 100644 index 00000000..dba3b34f --- /dev/null +++ b/boilerplate/flyte/golang_dockerfile/Readme.rst @@ -0,0 +1,16 @@ +Golang Dockerfile +~~~~~~~~~~~~~~~~~ + +Provides a Dockerfile that produces a small image. + +**To Enable:** + +Add ``flyteorg/golang_dockerfile`` to your ``boilerplate/update.cfg`` file. + +Create and configure a ``make linux_compile`` target that compiles your go binaries to the ``/artifacts`` directory :: + + .PHONY: linux_compile + linux_compile: + RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts {{ packages }} + +All binaries compiled to ``/artifacts`` will be available at ``/bin`` in your final image. diff --git a/boilerplate/flyte/golang_dockerfile/update.sh b/boilerplate/flyte/golang_dockerfile/update.sh new file mode 100755 index 00000000..5439bada --- /dev/null +++ b/boilerplate/flyte/golang_dockerfile/update.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +echo " - generating Dockerfile in root directory." +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/Dockerfile.GoTemplate > ${DIR}/../../../Dockerfile diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod index 7afee48e..53f64515 100644 --- a/boilerplate/flyte/golang_support_tools/go.mod +++ b/boilerplate/flyte/golang_support_tools/go.mod @@ -6,7 +6,10 @@ require ( github.com/alvaroloes/enumer v1.1.2 github.com/flyteorg/flytestdlib v0.3.22 github.com/golangci/golangci-lint v1.38.0 + github.com/pseudomuto/protoc-gen-doc v1.4.1 // indirect github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 ) replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0 + +replace github.com/pseudomuto/protoc-gen-doc => github.com/flyteorg/protoc-gen-doc v1.4.2 diff --git a/boilerplate/flyte/golang_support_tools/go.sum b/boilerplate/flyte/golang_support_tools/go.sum index 49939b68..261048f7 100644 --- a/boilerplate/flyte/golang_support_tools/go.sum +++ b/boilerplate/flyte/golang_support_tools/go.sum @@ -75,8 +75,11 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -99,6 +102,8 @@ github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cv github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY= github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -164,6 +169,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg= github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -191,6 +197,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.3.0-java h1:bV5JGEB1ouEzZa0hgVDFFiClrUEuGWRaAc/3mxR2QK0= +github.com/envoyproxy/protoc-gen-validate v0.3.0-java/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 h1:cTavhURetDkezJCvxFggiyLeP40Mrk/TtVg2+ycw1Es= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= github.com/esimonov/ifshort v1.0.1 h1:p7hlWD15c9XwvwxYg3W7f7UZHmwg7l9hC0hBiF95gd0= @@ -204,6 +212,8 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/flyteorg/flytestdlib v0.3.22 h1:nJEPaCdxzXBaeg2p4fdo3I3Ua09NedFRaUwuLafLEdw= github.com/flyteorg/flytestdlib v0.3.22/go.mod h1:1XG0DwYTUm34Yrffm1Qy9Tdr/pWQypEqTq5dUxw3/cM= +github.com/flyteorg/protoc-gen-doc v1.4.2 h1:Otw0F+RHaPQ8XlpzhLLgjsCMcrAIcMO01Zh+ALe3rrE= +github.com/flyteorg/protoc-gen-doc v1.4.2/go.mod h1:exDTOVwqpp30eV/EDPFLZy3Pwr2sn6hBC1WIYH/UbIg= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -271,6 +281,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -355,8 +366,10 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= @@ -418,9 +431,13 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -543,6 +560,8 @@ github.com/mozilla/tls-observatory v0.0.0-20201209171846-0547674fceff/go.mod h1: github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 h1:28i1IjGcx8AofiB4N3q5Yls55VEaitzuEPkFJEVgGkA= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw= github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= @@ -616,6 +635,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4= @@ -655,6 +675,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/pseudomuto/protokit v0.2.0 h1:hlnBDcy3YEDXH7kc9gV+NLaN0cDzhDvD1s7Y6FZ8RpM= +github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.3.0 h1:A3OfpsK2ynOTbz/KMi62qWzignjGCOZVChATSf4P+A0= github.com/quasilyte/go-ruleguard v0.3.0/go.mod h1:p2miAhLp6fERzFNbcuQ4bevXs8rgK//uCHsUDkumITg= @@ -737,6 +759,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -797,6 +820,7 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -905,6 +929,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1121,6 +1146,7 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/boilerplate/flyte/golang_support_tools/tools.go b/boilerplate/flyte/golang_support_tools/tools.go index 71163a57..eee691d8 100644 --- a/boilerplate/flyte/golang_support_tools/tools.go +++ b/boilerplate/flyte/golang_support_tools/tools.go @@ -7,4 +7,5 @@ import ( _ "github.com/flyteorg/flytestdlib/cli/pflags" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/vektra/mockery/cmd/mockery" + - "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index 25d4ac3d..c0ab06b0 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -20,6 +20,7 @@ tools=( "github.com/flyteorg/flytestdlib/cli/pflags" "github.com/golangci/golangci-lint/cmd/golangci-lint" "github.com/alvaroloes/enumer" + "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) tmp_dir=$(mktemp -d -t gotooling-XXX) diff --git a/boilerplate/flyte/golang_test_targets/goimports b/boilerplate/flyte/golang_test_targets/goimports index ba0d6d87..af182903 100755 --- a/boilerplate/flyte/golang_test_targets/goimports +++ b/boilerplate/flyte/golang_test_targets/goimports @@ -5,4 +5,4 @@ # # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst -goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*") +goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*" -not -path "./boilerplate/*") diff --git a/boilerplate/flyte/precommit/Makefile b/boilerplate/flyte/precommit/Makefile new file mode 100644 index 00000000..3c6f17d6 --- /dev/null +++ b/boilerplate/flyte/precommit/Makefile @@ -0,0 +1,9 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + + +.PHONY: setup-precommit +setup-precommit: #setup the precommit + @boilerplate/flyte/precommit/update.sh diff --git a/boilerplate/flyte/precommit/hooks/pre-push b/boilerplate/flyte/precommit/hooks/pre-push new file mode 100755 index 00000000..f161cfe8 --- /dev/null +++ b/boilerplate/flyte/precommit/hooks/pre-push @@ -0,0 +1,41 @@ +DUMMY_SHA=0000000000000000000000000000000000000000 + +echo "Running pre-push check; to skip this step use 'push --no-verify'" + +while read LOCAL_REF LOCAL_SHA REMOTE_REF REMOTE_SHA +do + if [ "$LOCAL_SHA" = $DUMMY_SHA ] + then + # Branch deleted. Do nothing. + exit 0 + else + if [ "$REMOTE_SHA" = $DUMMY_SHA ] + then + # New branch. Verify the last commit, since this is very likely where the new code is + # (though there is no way to know for sure). In the extremely uncommon case in which someone + # pushes more than 1 new commit to a branch, CI will enforce full checking. + RANGE="$LOCAL_SHA~1..$LOCAL_SHA" + else + # Updating branch. Verify new commits. + RANGE="$REMOTE_SHA..$LOCAL_SHA" + fi + + # Verify DCO signoff. We do this before the format checker, since it has + # some probability of failing spuriously, while this check never should. + # + # In general, we can't assume that the commits are signed off by author + # pushing, so we settle for just checking that there is a signoff at all. + SIGNED_OFF=$(git rev-list --no-merges --grep "^Signed-off-by: " "$RANGE") + NOT_SIGNED_OFF=$(git rev-list --no-merges "$RANGE" | grep -Fxv "$SIGNED_OFF") + if [ -n "$NOT_SIGNED_OFF" ] + then + echo >&2 "ERROR: The following commits do not have DCO signoff:" + while read -r commit; do + echo " $(git log --pretty=oneline --abbrev-commit -n 1 $commit)" + done <<< "$NOT_SIGNED_OFF" + exit 1 + fi + fi +done + +exit 0 diff --git a/boilerplate/flyte/precommit/hooks/prepare-commit-msg b/boilerplate/flyte/precommit/hooks/prepare-commit-msg new file mode 100755 index 00000000..8148d104 --- /dev/null +++ b/boilerplate/flyte/precommit/hooks/prepare-commit-msg @@ -0,0 +1,16 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst +# $ ln -s ../../support/hooks/prepare-commit-msg .git/hooks/prepare-commit-msg + +COMMIT_MESSAGE_FILE="$1" +AUTHOR=$(git var GIT_AUTHOR_IDENT) +SIGNOFF=$(echo $AUTHOR | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') + +# Check for DCO signoff message. If one doesn't exist, append one and then warn +# the user that you did so. +if ! $(grep -qs "^$SIGNOFF" "$COMMIT_MESSAGE_FILE") ; then + echo "\n$SIGNOFF" >> "$COMMIT_MESSAGE_FILE" + echo "Appended the following signoff to the end of the commit message:\n $SIGNOFF\n" +fi diff --git a/boilerplate/flyte/precommit/update.sh b/boilerplate/flyte/precommit/update.sh new file mode 100755 index 00000000..971c8386 --- /dev/null +++ b/boilerplate/flyte/precommit/update.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst + +set -e + +# Helper script for Automatically add DCO signoff with commit hooks +# Taken from Envoy https://gitlab.cncf.ci/envoyproxy/envoy +if [ ! "$PWD" == "$(git rev-parse --show-toplevel)" ]; then + cat >&2 <<__EOF__ +ERROR: this script must be run at the root of the envoy source tree +__EOF__ + exit 1 +fi + +# Helper functions that calculate `abspath` and `relpath`. Taken from Mesos +# commit 82b040a60561cf94dec3197ea88ae15e57bcaa97, which also carries the Apache +# V2 license, and has deployed this code successfully for some time. +abspath() { + cd "$(dirname "${1}")" + echo "${PWD}"/"$(basename "${1}")" + cd "${OLDPWD}" +} +relpath() { + local FROM TO UP + FROM="$(abspath "${1%/}")" TO="$(abspath "${2%/}"/)" + while test "${TO}" = "${TO#"${FROM}"/}" \ + -a "${TO}" != "${FROM}"; do + FROM="${FROM%/*}" UP="../${UP}" + done + TO="${UP%/}${TO#${FROM}}" + echo "${TO:-.}" +} + +# Try to find the `.git` directory, even if it's not in Flyte project root (as +# it wouldn't be if, say, this were in a submodule). The "blessed" but fairly +# new way to do this is to use `--git-common-dir`. +DOT_GIT_DIR=$(git rev-parse --git-common-dir) +if test ! -d "${DOT_GIT_DIR}"; then + # If `--git-common-dir` is not available, fall back to older way of doing it. + DOT_GIT_DIR=$(git rev-parse --git-dir) +fi + +mkdir -p ${DOT_GIT_DIR}/hooks + +HOOKS_DIR="${DOT_GIT_DIR}/hooks" +HOOKS_DIR_RELPATH=$(relpath "${HOOKS_DIR}" "${PWD}") + +if [ ! -e "${HOOKS_DIR}/prepare-commit-msg" ]; then + echo "Installing hook 'prepare-commit-msg'" + ln -s "${HOOKS_DIR_RELPATH}/boilerplate/flyte/precommit/hooks/prepare-commit-msg" "${HOOKS_DIR}/prepare-commit-msg" +fi + +if [ ! -e "${HOOKS_DIR}/pre-push" ]; then + echo "Installing hook 'pre-push'" + ln -s "${HOOKS_DIR_RELPATH}/boilerplate/flyte/precommit/hooks/pre-push" "${HOOKS_DIR}/pre-push" +fi diff --git a/boilerplate/flyte/pull_request_template/pull_request_template.md b/boilerplate/flyte/pull_request_template/pull_request_template.md index d7699558..9cdab99b 100644 --- a/boilerplate/flyte/pull_request_template/pull_request_template.md +++ b/boilerplate/flyte/pull_request_template/pull_request_template.md @@ -1,8 +1,9 @@ -## Read then delete +## _Read then delete this section_ -- Make sure to use a concise title for the pull-request. -- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version -will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump) +_- Make sure to use a concise title for the pull-request._ + +_- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version +will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump)_ # TL;DR _Please replace this text with a description of what this PR accomplishes._ @@ -24,7 +25,9 @@ _Please replace this text with a description of what this PR accomplishes._ _How did you fix the bug, make the feature etc. Link to any design docs etc_ ## Tracking Issue -https://github.com/flyteorg/flyte/issues/ +_Remove the '*fixes*' keyword if there will be multiple PRs to fix the linked issue_ + +fixes https://github.com/flyteorg/flyte/issues/ ## Follow-up issue _NA_ diff --git a/boilerplate/flyte/Readme.rst b/boilerplate/flyte/welcome_bot/Readme.rst similarity index 100% rename from boilerplate/flyte/Readme.rst rename to boilerplate/flyte/welcome_bot/Readme.rst diff --git a/boilerplate/flyte/config.yml b/boilerplate/flyte/welcome_bot/config.yml similarity index 100% rename from boilerplate/flyte/config.yml rename to boilerplate/flyte/welcome_bot/config.yml diff --git a/boilerplate/flyte/update.sh b/boilerplate/flyte/welcome_bot/update.sh similarity index 55% rename from boilerplate/flyte/update.sh rename to boilerplate/flyte/welcome_bot/update.sh index 506c3221..2db64ac3 100755 --- a/boilerplate/flyte/update.sh +++ b/boilerplate/flyte/welcome_bot/update.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. -# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # -# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst +# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst set -e @@ -11,4 +11,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # Clone the config.yml file echo " - copying ${DIR}/config.yml to the root directory." -cp ${DIR}/config.yml ${DIR}/../../.github/config.yml +cp "${DIR}"/config.yml "${DIR}"/../../../.github/config.yml + diff --git a/boilerplate/update.sh b/boilerplate/update.sh index 2d0fd711..73de4dc9 100755 --- a/boilerplate/update.sh +++ b/boilerplate/update.sh @@ -10,15 +10,9 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" OUT="$(mktemp -d)" -<<<<<<< HEAD trap 'rm -fr $OUT' EXIT git clone https://github.com/flyteorg/boilerplate.git "${OUT}" -======= -trap "rm -fr $OUT" EXIT - -git clone git@github.com:flyteorg/boilerplate.git "${OUT}" ->>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 echo "Updating the update.sh script." cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh" @@ -39,11 +33,7 @@ if [ -z "$REPOSITORY" ]; then exit 1 fi -<<<<<<< HEAD while read -r directory junk; do -======= -while read directory junk; do ->>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 # Skip comment lines (which can have leading whitespace) if [[ "$directory" == '#'* ]]; then continue @@ -71,13 +61,8 @@ while read directory junk; do echo "$directory is configured in update.cfg." echo "-----------------------------------------------------------------------------------" echo "syncing files from source." -<<<<<<< HEAD rm -rf "${DIR:?}/${directory}" mkdir -p "$(dirname "${DIR}"/"${directory}")" -======= - rm -rf "${DIR}/${directory}" - mkdir -p $(dirname "${DIR}/${directory}") ->>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 cp -r "$dir_path" "${DIR}/${directory}" if [ -f "${DIR}/${directory}/update.sh" ]; then echo "executing ${DIR}/${directory}/update.sh" @@ -85,8 +70,4 @@ while read directory junk; do fi echo "***********************************************************************************" echo "" -<<<<<<< HEAD -done < "$CONFIG_FILE" -======= done < "$CONFIG_FILE" ->>>>>>> c841290e942bbc901f235650d68347b4a03b8d28 diff --git a/pull_request_template.md b/pull_request_template.md index d7699558..9cdab99b 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,8 +1,9 @@ -## Read then delete +## _Read then delete this section_ -- Make sure to use a concise title for the pull-request. -- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version -will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump) +_- Make sure to use a concise title for the pull-request._ + +_- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version +will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump)_ # TL;DR _Please replace this text with a description of what this PR accomplishes._ @@ -24,7 +25,9 @@ _Please replace this text with a description of what this PR accomplishes._ _How did you fix the bug, make the feature etc. Link to any design docs etc_ ## Tracking Issue -https://github.com/flyteorg/flyte/issues/ +_Remove the '*fixes*' keyword if there will be multiple PRs to fix the linked issue_ + +fixes https://github.com/flyteorg/flyte/issues/ ## Follow-up issue _NA_ From 803f4cb0fe63695163becdd18f107966ecbafb6d Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 24 Aug 2021 02:52:35 -0500 Subject: [PATCH 31/58] separated ReservationManager CreateOrUpdate function into individual Create and Update functions Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 66 +++++++++++++++++-- pkg/manager/impl/reservation_manager_test.go | 34 ++++++---- pkg/repositories/gormimpl/metrics.go | 6 +- pkg/repositories/gormimpl/reservation.go | 28 ++++---- pkg/repositories/gormimpl/reservation_test.go | 28 ++++++-- .../interfaces/reservation_repo.go | 8 ++- pkg/repositories/mocks/reservation.go | 54 +++++++++++---- pkg/repositories/models/reservation.go | 2 +- 8 files changed, 175 insertions(+), 51 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 3605a2d1..deedbc90 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -119,20 +119,76 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * func (r *reservationManager) tryAcquireReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.ToReservationKey(*request.DatasetId, request.TagName) - rsv, err := repo.Get(ctx, reservationKey) + reservation, err := repo.Get(ctx, reservationKey) - resvExists := true + reservationExists := true if err != nil { if errors2.IsDoesNotExistError(err) { // Reservation does not exist yet so let's create one - resvExists = false + reservationExists = false } else { return datacatalog.ReservationStatus{}, err } } now := r.now() - if !resvExists || rsv.ExpireAt.Before(now) { + if !reservationExists { + err = repo.Create( + ctx, + models.Reservation{ + ReservationKey: reservationKey, + OwnerID: request.OwnerId, + ExpiresAt: now.Add(r.reservationTimeout), // TODO - change + }, + now, + ) + } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == request.OwnerId { + err = repo.Update( + ctx, + models.Reservation{ + ReservationKey: reservationKey, + OwnerID: request.OwnerId, + ExpiresAt: now.Add(r.reservationTimeout), // TODO - change + }, + now, + ) + } else { + logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: reservation.OwnerID, + }, nil + } + + if err != nil { + if err.Error() == repo_errors.ReservationAlreadyInProgress { + // Looks like someone else tried to obtain the reservation + // at the same time and they won. Let's find out who won. + rsv1, err := repo.Get(ctx, reservationKey) + if err != nil { + return datacatalog.ReservationStatus{}, err + } + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, + OwnerId: rsv1.OwnerID, + }, err + } else { + return datacatalog.ReservationStatus{}, err + } + } + + r.systemMetrics.reservationAcquired.Inc(ctx) + return datacatalog.ReservationStatus{ + State: datacatalog.ReservationStatus_ACQUIRED, + OwnerId: request.OwnerId, + }, nil + + // TODO - remove + /*if !resvExists || rsv.ExpireAt.Before(now) { // If the reservation does not exist or it is expired, // we try to acquire the reservation err := repo.CreateOrUpdate( @@ -183,7 +239,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request return datacatalog.ReservationStatus{ State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, OwnerId: rsv.OwnerID, - }, nil + }, nil*/ } func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 74582212..36b3354a 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -97,7 +97,7 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { now := time.Now() - dcRepo.MockReservationRepo.On("CreateOrUpdate", + dcRepo.MockReservationRepo.On("Create", mock.MatchedBy(func(ctx context.Context) bool { return true }), mock.MatchedBy(func(reservation models.Reservation) bool { return reservation.DatasetProject == datasetID.Project && @@ -106,7 +106,7 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && reservation.OwnerID == currentOwner && - reservation.ExpireAt == now.Add(timeout) + reservation.ExpiresAt == now.Add(timeout) }), mock.MatchedBy(func(now time.Time) bool { return true }), ).Return(nil) @@ -133,21 +133,29 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { setUpTagRepoGetNotFound(&dcRepo) now := time.Now() - prevExpireAt := now.Truncate(timeout + time.Second*10) + prevExpiresAt := now.Truncate(timeout + time.Second*10) - setUpReservationRepoGet(&dcRepo, prevExpireAt) + setUpReservationRepoGet(&dcRepo, prevExpiresAt) - dcRepo.MockReservationRepo.On("CreateOrUpdate", - mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(reservation models.Reservation) bool { + reservationMatchFunc := func(reservation models.Reservation) bool { return reservation.DatasetProject == datasetID.Project && reservation.DatasetDomain == datasetID.Domain && reservation.DatasetName == datasetID.Name && reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && reservation.OwnerID == currentOwner && - reservation.ExpireAt == now.Add(timeout) - }), + reservation.ExpiresAt == now.Add(timeout) + } + + dcRepo.MockReservationRepo.On("Create", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(reservationMatchFunc), + mock.MatchedBy(func(now time.Time) bool { return true }), + ).Return(nil) + + dcRepo.MockReservationRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(reservationMatchFunc), mock.MatchedBy(func(now time.Time) bool { return true }), ).Return(nil) @@ -167,7 +175,7 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) } -func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpireAt time.Time) { +func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpiresAt time.Time) { dcRepo.MockReservationRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), mock.MatchedBy(func(key models.ReservationKey) bool { @@ -180,7 +188,7 @@ func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpireAt time.Ti models.Reservation{ ReservationKey: getReservationKey(), OwnerID: prevOwner, - ExpireAt: prevExpireAt, + ExpiresAt: prevExpiresAt, }, nil, ) } @@ -208,9 +216,9 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { setUpTagRepoGetNotFound(&dcRepo) now := time.Now() - prevExpireAt := now.Add(time.Second * 10) + prevExpiresAt := now.Add(time.Second * 10) - setUpReservationRepoGet(&dcRepo, prevExpireAt) + setUpReservationRepoGet(&dcRepo, prevExpiresAt) reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, mockScope.NewTestScope()) diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index 27253011..0da57ab7 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -13,7 +13,7 @@ type gormMetrics struct { CreateDuration labeled.StopWatch GetDuration labeled.StopWatch ListDuration labeled.StopWatch - CreateOrUpdateDuration labeled.StopWatch + UpdateDuration labeled.StopWatch } func newGormMetrics(scope promutils.Scope) gormMetrics { @@ -25,7 +25,7 @@ func newGormMetrics(scope promutils.Scope) gormMetrics { "get", "Duration for retrieving an entity ", time.Millisecond, scope), ListDuration: labeled.NewStopWatch( "list", "Duration for listing entities ", time.Millisecond, scope), - CreateOrUpdateDuration: labeled.NewStopWatch( - "update", "Duration for creating/updating entities ", time.Millisecond, scope), + UpdateDuration: labeled.NewStopWatch( + "update", "Duration for updating entities ", time.Millisecond, scope), } } diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index b3e9a4e4..59a319bf 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -13,7 +13,6 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/promutils" "gorm.io/gorm" - "gorm.io/gorm/clause" ) type reservationRepo struct { @@ -48,19 +47,26 @@ func (r *reservationRepo) Get(ctx context.Context, reservationKey models.Reserva return reservation, nil } -func (r *reservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error { - timer := r.repoMetrics.CreateOrUpdateDuration.Start(ctx) +func (r *reservationRepo) Create(ctx context.Context, reservation models.Reservation, now time.Time) error { + timer := r.repoMetrics.CreateDuration.Start(ctx) defer timer.Stop() - expressions := make([]clause.Expression, 0) - expressions = append(expressions, clause.Lte{Column: "expire_at", Value: now}) + // TODO - test if we need FirstCreate and check RowsAffected + result := r.db.Create(&reservation) + if result.Error != nil { + return r.errorTransformer.ToDataCatalogError(result.Error) + } + + return nil +} + +func (r *reservationRepo) Update(ctx context.Context, reservation models.Reservation, now time.Time) error { + timer := r.repoMetrics.UpdateDuration.Start(ctx) + defer timer.Stop() - result := r.db.Clauses( - clause.OnConflict{ - Where: clause.Where{Exprs: expressions}, - UpdateAll: true, - }, - ).Create(&reservation) + result := r.db.Model(&models.Reservation{ + ReservationKey: reservation.ReservationKey, + }).Where("expires_at<=? OR owner_id=?", now, reservation.OwnerID,).Updates(reservation) if result.Error != nil { return r.errorTransformer.ToDataCatalogError(result.Error) } diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index bce0ae1c..a3975590 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -41,7 +41,7 @@ func TestGet(t *testing.T) { assert.Equal(t, expectedReservation.DatasetName, reservation.DatasetName) assert.Equal(t, expectedReservation.DatasetVersion, reservation.DatasetVersion) assert.Equal(t, expectedReservation.TagName, reservation.TagName) - assert.Equal(t, expectedReservation.ExpireAt, reservation.ExpireAt) + assert.Equal(t, expectedReservation.ExpiresAt, reservation.ExpiresAt) } func TestGetNotFound(t *testing.T) { @@ -61,18 +61,34 @@ func TestGetNotFound(t *testing.T) { } -func TestCreateOrUpdate(t *testing.T) { +func TestCreate(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true expectedReservation := GetReservation() - GlobalMock.NewMock().WithQuery( + // TODO - necessary for test of RowsAffected == 1 + /*GlobalMock.NewMock().WithQuery( `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT ("dataset_project","dataset_name","dataset_domain","dataset_version","tag_name") DO UPDATE SET "updated_at"="excluded"."updated_at","deleted_at"="excluded"."deleted_at","owner_id"="excluded"."owner_id","expire_at"="excluded"."expire_at","serialized_metadata"="excluded"."serialized_metadata"WHERE "expire_at" <= $12`, + ).WithRowsNum(1)*/ + + reservationRepo := getReservationRepo(t) + + err := reservationRepo.Create(context.Background(), expectedReservation, time.Now()) + assert.NoError(t, err) +} + +func TestUpdate(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `UPDATE "reservations" SET "updated_at"=$1,"dataset_project"=$2,"dataset_name"=$3,"dataset_domain"=$4,"dataset_version"=$5,"tag_name"=$6,"owner_id"=$7,"expires_at"=$8 WHERE (expires_at<=$9 OR owner_id=$10) AND "dataset_project" = $11 AND "dataset_name" = $12 AND "dataset_domain" = $13 AND "dataset_version" = $14 AND "tag_name" = $15`, ).WithRowsNum(1) reservationRepo := getReservationRepo(t) - err := reservationRepo.CreateOrUpdate(context.Background(), expectedReservation, time.Now()) + err := reservationRepo.Update(context.Background(), expectedReservation, time.Now()) assert.NoError(t, err) } @@ -98,7 +114,7 @@ func getDBResponse(reservation models.Reservation) []map[string]interface{} { "dataset_version": reservation.DatasetVersion, "tag_name": reservation.TagName, "owner_id": reservation.OwnerID, - "expire_at": reservation.ExpireAt, + "expires_at": reservation.ExpiresAt, }, } } @@ -117,7 +133,7 @@ func GetReservation() models.Reservation { reservation := models.Reservation{ ReservationKey: GetReservationKey(), OwnerID: "batman", - ExpireAt: time.Unix(1, 1), + ExpiresAt: time.Unix(1, 1), } return reservation } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 57836252..1d4bbed6 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -13,9 +13,15 @@ type ReservationRepo interface { // Get reservation Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) + // TODO - comment + Create(ctx context.Context, reservation models.Reservation, now time.Time) error + + // TODO - comment + Update(ctx context.Context, reservation models.Reservation, now time.Time) error + // Create the reservation. If the reservation already exists, we try to take over the // reservation via update when the reservation has expired. Note: Each reservation has its own // expire date which is tracked in expire_at column in the reservation table. And the // reservation expires when the date stored in expire_at column is in the past. - CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error + //CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error } diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go index 7e91542a..8aa000bf 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation.go @@ -17,26 +17,26 @@ type ReservationRepo struct { mock.Mock } -type ReservationRepo_CreateOrUpdate struct { +type ReservationRepo_Create struct { *mock.Call } -func (_m ReservationRepo_CreateOrUpdate) Return(_a0 error) *ReservationRepo_CreateOrUpdate { - return &ReservationRepo_CreateOrUpdate{Call: _m.Call.Return(_a0)} +func (_m ReservationRepo_Create) Return(_a0 error) *ReservationRepo_Create { + return &ReservationRepo_Create{Call: _m.Call.Return(_a0)} } -func (_m *ReservationRepo) OnCreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_CreateOrUpdate { - c := _m.On("CreateOrUpdate", ctx, reservation, now) - return &ReservationRepo_CreateOrUpdate{Call: c} +func (_m *ReservationRepo) OnCreate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_Create { + c := _m.On("Create", ctx, reservation, now) + return &ReservationRepo_Create{Call: c} } -func (_m *ReservationRepo) OnCreateOrUpdateMatch(matchers ...interface{}) *ReservationRepo_CreateOrUpdate { - c := _m.On("CreateOrUpdate", matchers...) - return &ReservationRepo_CreateOrUpdate{Call: c} +func (_m *ReservationRepo) OnCreateMatch(matchers ...interface{}) *ReservationRepo_Create { + c := _m.On("Create", matchers...) + return &ReservationRepo_Create{Call: c} } -// CreateOrUpdate provides a mock function with given fields: ctx, reservation, now -func (_m *ReservationRepo) CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error { +// Create provides a mock function with given fields: ctx, reservation, now +func (_m *ReservationRepo) Create(ctx context.Context, reservation models.Reservation, now time.Time) error { ret := _m.Called(ctx, reservation, now) var r0 error @@ -87,3 +87,35 @@ func (_m *ReservationRepo) Get(ctx context.Context, reservationKey models.Reserv return r0, r1 } + +type ReservationRepo_Update struct { + *mock.Call +} + +func (_m ReservationRepo_Update) Return(_a0 error) *ReservationRepo_Update { + return &ReservationRepo_Update{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_Update { + c := _m.On("Update", ctx, reservation, now) + return &ReservationRepo_Update{Call: c} +} + +func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRepo_Update { + c := _m.On("Update", matchers...) + return &ReservationRepo_Update{Call: c} +} + +// Update provides a mock function with given fields: ctx, reservation, now +func (_m *ReservationRepo) Update(ctx context.Context, reservation models.Reservation, now time.Time) error { + ret := _m.Called(ctx, reservation, now) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.Reservation, time.Time) error); ok { + r0 = rf(ctx, reservation, now) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/pkg/repositories/models/reservation.go b/pkg/repositories/models/reservation.go index 53abf771..366801ff 100644 --- a/pkg/repositories/models/reservation.go +++ b/pkg/repositories/models/reservation.go @@ -21,6 +21,6 @@ type Reservation struct { OwnerID string // When the reservation will expire - ExpireAt time.Time + ExpiresAt time.Time SerializedMetadata []byte } From a1fbc14cb86779057e9e1c254805afe23452b64e Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 24 Aug 2021 15:55:47 -0500 Subject: [PATCH 32/58] fixed race condition on Reservation repository Create function Signed-off-by: Daniel Rammer --- pkg/repositories/gormimpl/reservation.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 59a319bf..9fb67294 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -13,6 +13,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/promutils" "gorm.io/gorm" + "gorm.io/gorm/clause" ) type reservationRepo struct { @@ -51,12 +52,18 @@ func (r *reservationRepo) Create(ctx context.Context, reservation models.Reserva timer := r.repoMetrics.CreateDuration.Start(ctx) defer timer.Stop() - // TODO - test if we need FirstCreate and check RowsAffected - result := r.db.Create(&reservation) + result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&reservation) if result.Error != nil { return r.errorTransformer.ToDataCatalogError(result.Error) } + if result.RowsAffected == 0 { + return datacatalog_error.NewDataCatalogError( + codes.FailedPrecondition, + errors2.ReservationAlreadyInProgress, + ) + } + return nil } From 2afa4bccb4ed53eb7cf511ac56e2b97f24472ed2 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 24 Aug 2021 15:58:03 -0500 Subject: [PATCH 33/58] changed reservation expiration to use heartbeatInterval and heartbeatGracePeriodMultiplier Signed-off-by: Daniel Rammer --- datacatalog_config.yaml | 2 + pkg/manager/impl/reservation_manager.go | 91 +++++----------------- pkg/rpc/datacatalogservice/service.go | 2 +- pkg/runtime/configs/data_catalog_config.go | 9 ++- 4 files changed, 29 insertions(+), 75 deletions(-) diff --git a/datacatalog_config.yaml b/datacatalog_config.yaml index e0f9c3d1..874d9de7 100644 --- a/datacatalog_config.yaml +++ b/datacatalog_config.yaml @@ -11,6 +11,8 @@ datacatalog: storage-prefix: "metadata" metrics-scope: "datacatalog" profiler-port: 10254 + heartbeat-grace-period-multiplier: 3 + heartbeat-interval-sec: 5 storage: connection: access-key: minio diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index deedbc90..2bce7ab8 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -31,15 +31,17 @@ type reservationMetrics struct { type NowFunc func() time.Time type reservationManager struct { - repo repositories.RepositoryInterface - reservationTimeout time.Duration - now NowFunc - systemMetrics reservationMetrics + repo repositories.RepositoryInterface + heartbeatGracePeriodMultiplier time.Duration + heartbeatInterval time.Duration + now NowFunc + systemMetrics reservationMetrics } func NewReservationManager( repo repositories.RepositoryInterface, - reservationTimeout time.Duration, + heartbeatGracePeriodMultiplier time.Duration, + heartbeatInterval time.Duration, nowFunc NowFunc, // Easier to mock time.Time for testing reservationScope promutils.Scope, ) interfaces.ReservationManager { @@ -67,10 +69,11 @@ func NewReservationManager( } return &reservationManager{ - repo: repo, - reservationTimeout: reservationTimeout, - now: nowFunc, - systemMetrics: systemMetrics, + repo: repo, + heartbeatGracePeriodMultiplier: heartbeatGracePeriodMultiplier, + heartbeatInterval: heartbeatInterval, + now: nowFunc, + systemMetrics: systemMetrics, } } @@ -132,23 +135,25 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request } now := r.now() + // TODO - clean the reservation copy + var repoErr error if !reservationExists { - err = repo.Create( + repoErr = repo.Create( ctx, models.Reservation{ ReservationKey: reservationKey, OwnerID: request.OwnerId, - ExpiresAt: now.Add(r.reservationTimeout), // TODO - change + ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), }, now, ) } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == request.OwnerId { - err = repo.Update( + repoErr = repo.Update( ctx, models.Reservation{ ReservationKey: reservationKey, OwnerID: request.OwnerId, - ExpiresAt: now.Add(r.reservationTimeout), // TODO - change + ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), }, now, ) @@ -162,8 +167,8 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request }, nil } - if err != nil { - if err.Error() == repo_errors.ReservationAlreadyInProgress { + if repoErr != nil { + if repoErr.Error() == repo_errors.ReservationAlreadyInProgress { // Looks like someone else tried to obtain the reservation // at the same time and they won. Let's find out who won. rsv1, err := repo.Get(ctx, reservationKey) @@ -177,7 +182,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request OwnerId: rsv1.OwnerID, }, err } else { - return datacatalog.ReservationStatus{}, err + return datacatalog.ReservationStatus{}, repoErr } } @@ -186,60 +191,6 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request State: datacatalog.ReservationStatus_ACQUIRED, OwnerId: request.OwnerId, }, nil - - // TODO - remove - /*if !resvExists || rsv.ExpireAt.Before(now) { - // If the reservation does not exist or it is expired, - // we try to acquire the reservation - err := repo.CreateOrUpdate( - ctx, - models.Reservation{ - ReservationKey: reservationKey, - OwnerID: request.OwnerId, - ExpireAt: now.Add(r.reservationTimeout), - }, - now, - ) - - failedToAcquireReservation := false - if err != nil { - if err.Error() == repo_errors.ReservationAlreadyInProgress { - - failedToAcquireReservation = true - } else { - return datacatalog.ReservationStatus{}, err - } - } - - if failedToAcquireReservation { - // Looks like someone else tried to obtain the reservation - // at the same time and they won. Let's find out who won. - rsv1, err := repo.Get(ctx, reservationKey) - if err != nil { - return datacatalog.ReservationStatus{}, err - } - - r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: rsv1.OwnerID, - }, err - } - - r.systemMetrics.reservationAcquired.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, - }, nil - } - - logger.Debugf(ctx, "Reservation: %+v is hold by %s", reservationKey, rsv.OwnerID) - - r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: rsv.OwnerID, - }, nil*/ } func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 7d51a60a..59116fad 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -124,7 +124,7 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(repos, time.Second*time.Duration(dataCatalogConfig.ReservationTimeoutSec), time.Now, + ReservationManager: impl.NewReservationManager(repos, time.Duration(dataCatalogConfig.HeartbeatGracePeriodMultiplier), time.Second*time.Duration(dataCatalogConfig.HeartbeatIntervalSec), time.Now, catalogScope.NewSubScope("reservation")), } } diff --git a/pkg/runtime/configs/data_catalog_config.go b/pkg/runtime/configs/data_catalog_config.go index 0bcfaf7e..a1c331b6 100644 --- a/pkg/runtime/configs/data_catalog_config.go +++ b/pkg/runtime/configs/data_catalog_config.go @@ -4,8 +4,9 @@ package configs // This configuration is the base configuration to start admin type DataCatalogConfig struct { - StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` - MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` - ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` - ReservationTimeoutSec int `json:"reservationTimeoutSec" pflag:",How long do we hold the reservation"` + StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` + MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` + ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` + HeartbeatGracePeriodMultiplier int `json:"heartbeat-grace-period-multiplier" pflag:",Number of heartbeats before a reservation expires without an extension."` + HeartbeatIntervalSec int `json:"heartbeat-interval-sec" pflag:",Recommended reservation extension heartbeat interval."` } From c72a6b698fe30dae70340030b82c1ce36f0577d2 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 24 Aug 2021 18:12:52 -0500 Subject: [PATCH 34/58] fixed lint and unit test errors Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 33 ++++++--------- pkg/manager/impl/reservation_manager_test.go | 43 +++++++++----------- pkg/repositories/gormimpl/metrics.go | 10 ++--- pkg/repositories/gormimpl/reservation.go | 2 +- 4 files changed, 38 insertions(+), 50 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 2bce7ab8..46da4383 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -135,28 +135,19 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request } now := r.now() - // TODO - clean the reservation copy + newReservation := models.Reservation{ + ReservationKey: reservationKey, + OwnerID: request.OwnerId, + ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), + } + + // Conditional upsert on reservation. Race conditions are handled + // within the reservation repository Create and Update function calls. var repoErr error if !reservationExists { - repoErr = repo.Create( - ctx, - models.Reservation{ - ReservationKey: reservationKey, - OwnerID: request.OwnerId, - ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), - }, - now, - ) + repoErr = repo.Create(ctx, newReservation, now) } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == request.OwnerId { - repoErr = repo.Update( - ctx, - models.Reservation{ - ReservationKey: reservationKey, - OwnerID: request.OwnerId, - ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), - }, - now, - ) + repoErr = repo.Update(ctx, newReservation, now) } else { logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) @@ -181,9 +172,9 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, OwnerId: rsv1.OwnerID, }, err - } else { - return datacatalog.ReservationStatus{}, repoErr } + + return datacatalog.ReservationStatus{}, repoErr } r.systemMetrics.reservationAcquired.Inc(ctx) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 36b3354a..49161441 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -29,7 +29,8 @@ var datasetID = datacatalog.DatasetID{ Domain: domain, Version: version, } -var timeout = time.Second * 300 +var heartbeatInterval = time.Second * 5 +var heartbeatGracePeriodMultiplier = time.Second * 3 var prevOwner = "prevOwner" var currentOwner = "currentOwner" @@ -58,7 +59,8 @@ func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { Artifact: expectedArtifact, }, nil) - reservationManager := NewReservationManager(&dcRepo, timeout, time.Now, mockScope.NewTestScope()) + reservationManager := NewReservationManager(&dcRepo, heartbeatGracePeriodMultiplier, + heartbeatInterval, time.Now, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -106,13 +108,14 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && reservation.OwnerID == currentOwner && - reservation.ExpiresAt == now.Add(timeout) + reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) }), mock.MatchedBy(func(now time.Time) bool { return true }), ).Return(nil) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, - mockScope.NewTestScope()) + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -133,34 +136,27 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { setUpTagRepoGetNotFound(&dcRepo) now := time.Now() - prevExpiresAt := now.Truncate(timeout + time.Second*10) + prevExpiresAt := now.Add(time.Second * 10 * time.Duration(-1)) setUpReservationRepoGet(&dcRepo, prevExpiresAt) - reservationMatchFunc := func(reservation models.Reservation) bool { + dcRepo.MockReservationRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservation models.Reservation) bool { return reservation.DatasetProject == datasetID.Project && reservation.DatasetDomain == datasetID.Domain && reservation.DatasetName == datasetID.Name && reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && reservation.OwnerID == currentOwner && - reservation.ExpiresAt == now.Add(timeout) - } - - dcRepo.MockReservationRepo.On("Create", - mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(reservationMatchFunc), - mock.MatchedBy(func(now time.Time) bool { return true }), - ).Return(nil) - - dcRepo.MockReservationRepo.On("Update", - mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(reservationMatchFunc), + reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) + }), mock.MatchedBy(func(now time.Time) bool { return true }), ).Return(nil) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, - mockScope.NewTestScope()) + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, @@ -220,8 +216,9 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { setUpReservationRepoGet(&dcRepo, prevExpiresAt) - reservationManager := NewReservationManager(&dcRepo, timeout, func() time.Time { return now }, - mockScope.NewTestScope()) + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ DatasetId: &datasetID, diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index 0da57ab7..d4ba332d 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -9,11 +9,11 @@ import ( // Common metrics for DB CRUD operations type gormMetrics struct { - Scope promutils.Scope - CreateDuration labeled.StopWatch - GetDuration labeled.StopWatch - ListDuration labeled.StopWatch - UpdateDuration labeled.StopWatch + Scope promutils.Scope + CreateDuration labeled.StopWatch + GetDuration labeled.StopWatch + ListDuration labeled.StopWatch + UpdateDuration labeled.StopWatch } func newGormMetrics(scope promutils.Scope) gormMetrics { diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 9fb67294..680f3d2d 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -73,7 +73,7 @@ func (r *reservationRepo) Update(ctx context.Context, reservation models.Reserva result := r.db.Model(&models.Reservation{ ReservationKey: reservation.ReservationKey, - }).Where("expires_at<=? OR owner_id=?", now, reservation.OwnerID,).Updates(reservation) + }).Where("expires_at<=? OR owner_id=?", now, reservation.OwnerID).Updates(reservation) if result.Error != nil { return r.errorTransformer.ToDataCatalogError(result.Error) } From 1bc36e43451976d3b40636bd066e50a660c66e9d Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 12:23:02 -0500 Subject: [PATCH 35/58] added unit tests for extending reservation and update failure Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager_test.go | 41 +++++++++++++++++++ pkg/repositories/gormimpl/reservation_test.go | 23 +++++++++-- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 49161441..c4ca69aa 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -206,6 +206,47 @@ func getReservationKey() models.ReservationKey { } } +func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { + dcRepo := getDatacatalogRepo() + + setUpTagRepoGetNotFound(&dcRepo) + + now := time.Now() + prevExpiresAt := now.Add(time.Second * 10) + + setUpReservationRepoGet(&dcRepo, prevExpiresAt) + + dcRepo.MockReservationRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservation models.Reservation) bool { + return reservation.DatasetProject == datasetID.Project && + reservation.DatasetDomain == datasetID.Domain && + reservation.DatasetName == datasetID.Name && + reservation.DatasetVersion == datasetID.Version && + reservation.TagName == tagName && + reservation.OwnerID == prevOwner && + reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) + }), + mock.MatchedBy(func(now time.Time) bool { return true }), + ).Return(nil) + + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) + + req := datacatalog.GetOrReserveArtifactRequest{ + DatasetId: &datasetID, + TagName: tagName, + OwnerId: prevOwner, + } + + resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + + assert.Nil(t, err) + assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) +} + func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { dcRepo := getDatacatalogRepo() diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index a3975590..c09ecaf8 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -66,10 +66,9 @@ func TestCreate(t *testing.T) { GlobalMock.Logging = true expectedReservation := GetReservation() - // TODO - necessary for test of RowsAffected == 1 - /*GlobalMock.NewMock().WithQuery( - `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expire_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT ("dataset_project","dataset_name","dataset_domain","dataset_version","tag_name") DO UPDATE SET "updated_at"="excluded"."updated_at","deleted_at"="excluded"."deleted_at","owner_id"="excluded"."owner_id","expire_at"="excluded"."expire_at","serialized_metadata"="excluded"."serialized_metadata"WHERE "expire_at" <= $12`, - ).WithRowsNum(1)*/ + GlobalMock.NewMock().WithQuery( + `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expires_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT DO NOTHING`, + ).WithRowsNum(1) reservationRepo := getReservationRepo(t) @@ -92,6 +91,22 @@ func TestUpdate(t *testing.T) { assert.NoError(t, err) } +func TestUpdateFailure(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `UPDATE "reservations" SET "updated_at"=$1,"dataset_project"=$2,"dataset_name"=$3,"dataset_domain"=$4,"dataset_version"=$5,"tag_name"=$6,"owner_id"=$7,"expires_at"=$8 WHERE (expires_at<=$9 OR owner_id=$10) AND "dataset_project" = $11 AND "dataset_name" = $12 AND "dataset_domain" = $13 AND "dataset_version" = $14 AND "tag_name" = $15`, + ).WithRowsNum(0) + + reservationRepo := getReservationRepo(t) + + err := reservationRepo.Update(context.Background(), expectedReservation, time.Now()) + assert.Error(t, err) + assert.Equal(t, "reservation already in progress", err.Error()) +} + func getReservationRepo(t *testing.T) interfaces.ReservationRepo { mocket.Catcher.Register() sqlDB, err := sql.Open(mocket.DriverName, "blah") From 7bc155ce9a939d55a19822153d815bfec3c86cb2 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 12:24:46 -0500 Subject: [PATCH 36/58] removed ExtendReservation API mocks Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 4 ---- pkg/manager/interfaces/reservation.go | 1 - pkg/rpc/datacatalogservice/service.go | 4 ---- 3 files changed, 9 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 46da4383..65480fed 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -184,10 +184,6 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request }, nil } -func (r *reservationManager) ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) { - return nil, errors.New("not implemented") -} - func (r *reservationManager) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { return nil, errors.New("not implemented") } diff --git a/pkg/manager/interfaces/reservation.go b/pkg/manager/interfaces/reservation.go index cb43ef28..5ddf81f9 100644 --- a/pkg/manager/interfaces/reservation.go +++ b/pkg/manager/interfaces/reservation.go @@ -11,6 +11,5 @@ import ( // in flyteidl type ReservationManager interface { GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) - ExtendReservation(context.Context, *datacatalog.ExtendReservationRequest) (*datacatalog.ExtendReservationResponse, error) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) } diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 59116fad..88081055 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -59,10 +59,6 @@ func (s *DataCatalogService) GetOrReserveArtifact(ctx context.Context, request * return s.ReservationManager.GetOrReserveArtifact(ctx, request) } -func (s *DataCatalogService) ExtendReservation(ctx context.Context, request *catalog.ExtendReservationRequest) (*catalog.ExtendReservationResponse, error) { - return s.ReservationManager.ExtendReservation(ctx, request) -} - func (s *DataCatalogService) ReleaseReservation(ctx context.Context, request *catalog.ReleaseReservationRequest) (*catalog.ReleaseReservationResponse, error) { return s.ReservationManager.ReleaseReservation(ctx, request) } From 080ff5ce527e27464461d908a971bc3929d1d45d Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 13:11:36 -0500 Subject: [PATCH 37/58] added ExpiresAt and HeartbeatInterval fiedls to ReservationStatus return based on new API Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 37 +++++++++++++------- pkg/repositories/transformers/reservation.go | 22 ++++++++++++ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 65480fed..ef4f9465 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -151,11 +151,15 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request } else { logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) + reservationStatus, err := transformers.CreateReservationStatus( + reservation, r.heartbeatInterval, + datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) + if err != nil { + return reservationStatus, err + } + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: reservation.OwnerID, - }, nil + return reservationStatus, nil } if repoErr != nil { @@ -167,21 +171,28 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request return datacatalog.ReservationStatus{}, err } + reservationStatus, err := transformers.CreateReservationStatus( + rsv1, r.heartbeatInterval, + datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) + if err != nil { + return reservationStatus, err + } + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, - OwnerId: rsv1.OwnerID, - }, err + return reservationStatus, nil } return datacatalog.ReservationStatus{}, repoErr } - r.systemMetrics.reservationAcquired.Inc(ctx) - return datacatalog.ReservationStatus{ - State: datacatalog.ReservationStatus_ACQUIRED, - OwnerId: request.OwnerId, - }, nil + reservationStatus, err := transformers.CreateReservationStatus(newReservation, + r.heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) + if err != nil { + return reservationStatus, err + } + + r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) + return reservationStatus, nil } func (r *reservationManager) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go index f0c4705d..8ed89cde 100644 --- a/pkg/repositories/transformers/reservation.go +++ b/pkg/repositories/transformers/reservation.go @@ -1,8 +1,15 @@ package transformers import ( + "time" + + "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + + "github.com/golang/protobuf/ptypes" + + "google.golang.org/grpc/codes" ) func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.ReservationKey { @@ -14,3 +21,18 @@ func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.Re TagName: tagName, } } + +func CreateReservationStatus(reservation models.Reservation, heartbeatInterval time.Duration, state datacatalog.ReservationStatus_State) (datacatalog.ReservationStatus, error) { + expiresAtPb, err := ptypes.TimestampProto(reservation.CreatedAt) + if err != nil { + return datacatalog.ReservationStatus{}, errors.NewDataCatalogErrorf(codes.Internal, "failed to serialize expires at time") + } + + heartbeatIntervalPb := ptypes.DurationProto(heartbeatInterval) + return datacatalog.ReservationStatus{ + State: state, + ExpiresAt: expiresAtPb, + HeartbeatInterval: heartbeatIntervalPb, + OwnerId: reservation.OwnerID, + }, nil +} From 62fcc7f9a7717730d4fa3d8975ef87a729377772 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 13:57:31 -0500 Subject: [PATCH 38/58] implemented ReleaseReservation Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 48 ++++++++++---- pkg/repositories/gormimpl/metrics.go | 3 + pkg/repositories/gormimpl/reservation.go | 65 ++++++++++++++----- pkg/repositories/gormimpl/reservation_test.go | 30 ++++----- .../interfaces/reservation_repo.go | 19 +++--- pkg/repositories/mocks/reservation.go | 32 +++++++++ pkg/repositories/transformers/reservation.go | 15 ++++- 7 files changed, 157 insertions(+), 55 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index ef4f9465..03c801ae 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -2,14 +2,13 @@ package impl import ( "context" - "errors" "time" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" "github.com/flyteorg/flytestdlib/promutils/labeled" - errors2 "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/flyteorg/datacatalog/pkg/errors" "github.com/flyteorg/datacatalog/pkg/repositories" repo_errors "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" @@ -23,9 +22,11 @@ import ( type reservationMetrics struct { scope promutils.Scope reservationAcquired labeled.Counter + reservationReleased labeled.Counter reservationAlreadyInProgress labeled.Counter acquireReservationFailure labeled.Counter getTagFailure labeled.Counter + releaseReservationFailure labeled.Counter } type NowFunc func() time.Time @@ -51,6 +52,10 @@ func NewReservationManager( "reservation_acquired", "Number of times a reservation was acquired", reservationScope), + reservationReleased: labeled.NewCounter( + "reservation_released", + "Number of times a reservation was released", + reservationScope), reservationAlreadyInProgress: labeled.NewCounter( "reservation_already_in_progress", "Number of times we try of acquire a reservation but the reservation is in progress", @@ -66,6 +71,11 @@ func NewReservationManager( "Number of times we failed to get tag", reservationScope, ), + releaseReservationFailure: labeled.NewCounter( + "release_reservation_failure", + "Number of times we failed to release a reservation", + reservationScope, + ), } return &reservationManager{ @@ -78,13 +88,14 @@ func NewReservationManager( } func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { - tagKey := transformers.ToTagKey(request.DatasetId, request.TagName) + reservationId := request.ReservationId + tagKey := transformers.ToTagKey(reservationId.DatasetId, reservationId.TagName) tag, err := r.repo.TagRepo().Get(ctx, tagKey) if err != nil { - if errors2.IsDoesNotExistError(err) { + if errors.IsDoesNotExistError(err) { // Tag does not exist yet, let's acquire the reservation to work on // generating the artifact. - status, err := r.tryAcquireReservation(ctx, request) + status, err := r.tryAcquireReservation(ctx, reservationId, request.OwnerId) if err != nil { r.systemMetrics.acquireReservationFailure.Inc(ctx) return nil, err @@ -119,14 +130,14 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * // to do a GET here because we want to know who owns the reservation // and show it to users on the UI. However, the reservation is held by a single // task most of the times and there is no need to do a write. -func (r *reservationManager) tryAcquireReservation(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (datacatalog.ReservationStatus, error) { +func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationId *datacatalog.ReservationID, ownerId string) (datacatalog.ReservationStatus, error) { repo := r.repo.ReservationRepo() - reservationKey := transformers.ToReservationKey(*request.DatasetId, request.TagName) + reservationKey := transformers.FromReservationID(reservationId) reservation, err := repo.Get(ctx, reservationKey) reservationExists := true if err != nil { - if errors2.IsDoesNotExistError(err) { + if errors.IsDoesNotExistError(err) { // Reservation does not exist yet so let's create one reservationExists = false } else { @@ -137,7 +148,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request now := r.now() newReservation := models.Reservation{ ReservationKey: reservationKey, - OwnerID: request.OwnerId, + OwnerID: ownerId, ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), } @@ -146,7 +157,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request var repoErr error if !reservationExists { repoErr = repo.Create(ctx, newReservation, now) - } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == request.OwnerId { + } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == ownerId { repoErr = repo.Update(ctx, newReservation, now) } else { logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) @@ -185,6 +196,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request return datacatalog.ReservationStatus{}, repoErr } + // Reservation has been acquired or extended without error reservationStatus, err := transformers.CreateReservationStatus(newReservation, r.heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) if err != nil { @@ -195,6 +207,18 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, request return reservationStatus, nil } -func (r *reservationManager) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { - return nil, errors.New("not implemented") +func (r *reservationManager) ReleaseReservation(ctx context.Context, request *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { + repo := r.repo.ReservationRepo() + reservationKey := transformers.FromReservationID(request.ReservationId) + + err := repo.Delete(ctx, reservationKey) + if err != nil { + logger.Errorf(ctx, "Failed to release reservation: %+v, err: %v", reservationKey, err) + + r.systemMetrics.releaseReservationFailure.Inc(ctx) + return nil, err + } + + r.systemMetrics.reservationReleased.Inc(ctx) + return &datacatalog.ReleaseReservationResponse{}, nil } diff --git a/pkg/repositories/gormimpl/metrics.go b/pkg/repositories/gormimpl/metrics.go index d4ba332d..42fccd3e 100644 --- a/pkg/repositories/gormimpl/metrics.go +++ b/pkg/repositories/gormimpl/metrics.go @@ -11,6 +11,7 @@ import ( type gormMetrics struct { Scope promutils.Scope CreateDuration labeled.StopWatch + DeleteDuration labeled.StopWatch GetDuration labeled.StopWatch ListDuration labeled.StopWatch UpdateDuration labeled.StopWatch @@ -21,6 +22,8 @@ func newGormMetrics(scope promutils.Scope) gormMetrics { Scope: scope, CreateDuration: labeled.NewStopWatch( "create", "Duration for creating a new entity", time.Millisecond, scope), + DeleteDuration: labeled.NewStopWatch( + "delete", "Duration for deleting a new entity", time.Millisecond, scope), GetDuration: labeled.NewStopWatch( "get", "Duration for retrieving an entity ", time.Millisecond, scope), ListDuration: labeled.NewStopWatch( diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 680f3d2d..53bae84f 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -8,6 +8,8 @@ import ( "time" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" @@ -31,23 +33,6 @@ func NewReservationRepo(db *gorm.DB, errorTransformer errors2.ErrorTransformer, } } -func (r *reservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { - timer := r.repoMetrics.GetDuration.Start(ctx) - defer timer.Stop() - - var reservation models.Reservation - - result := r.db.Where(&models.Reservation{ - ReservationKey: reservationKey, - }).Take(&reservation) - - if result.Error != nil { - return reservation, r.errorTransformer.ToDataCatalogError(result.Error) - } - - return reservation, nil -} - func (r *reservationRepo) Create(ctx context.Context, reservation models.Reservation, now time.Time) error { timer := r.repoMetrics.CreateDuration.Start(ctx) defer timer.Stop() @@ -67,6 +52,52 @@ func (r *reservationRepo) Create(ctx context.Context, reservation models.Reserva return nil } +func (r *reservationRepo) Delete(ctx context.Context, reservationKey models.ReservationKey) error { + timer := r.repoMetrics.DeleteDuration.Start(ctx) + defer timer.Stop() + + var reservation models.Reservation + + result := r.db.Where(&models.Reservation{ + ReservationKey: reservationKey, + }).Delete(&reservation) + if result.Error != nil { + return r.errorTransformer.ToDataCatalogError(result.Error) + } + + if result.RowsAffected == 0 { + return errors2.GetMissingEntityError("Reservation", + &datacatalog.ReservationID { + DatasetId: &datacatalog.DatasetID{ + Project: reservationKey.DatasetProject, + Domain: reservationKey.DatasetDomain, + Name: reservationKey.DatasetName, + Version: reservationKey.DatasetVersion, + }, + TagName: reservationKey.TagName, + }) + } + + return nil +} + +func (r *reservationRepo) Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) { + timer := r.repoMetrics.GetDuration.Start(ctx) + defer timer.Stop() + + var reservation models.Reservation + + result := r.db.Where(&models.Reservation{ + ReservationKey: reservationKey, + }).Take(&reservation) + + if result.Error != nil { + return reservation, r.errorTransformer.ToDataCatalogError(result.Error) + } + + return reservation, nil +} + func (r *reservationRepo) Update(ctx context.Context, reservation models.Reservation, now time.Time) error { timer := r.repoMetrics.UpdateDuration.Start(ctx) defer timer.Stop() diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index c09ecaf8..e1ec3c46 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -23,6 +23,21 @@ import ( "github.com/stretchr/testify/assert" ) +func TestCreate(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expires_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT DO NOTHING`, + ).WithRowsNum(1) + + reservationRepo := getReservationRepo(t) + + err := reservationRepo.Create(context.Background(), expectedReservation, time.Now()) + assert.NoError(t, err) +} + func TestGet(t *testing.T) { expectedReservation := GetReservation() @@ -61,21 +76,6 @@ func TestGetNotFound(t *testing.T) { } -func TestCreate(t *testing.T) { - GlobalMock := mocket.Catcher.Reset() - GlobalMock.Logging = true - expectedReservation := GetReservation() - - GlobalMock.NewMock().WithQuery( - `INSERT INTO "reservations" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","owner_id","expires_at","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ON CONFLICT DO NOTHING`, - ).WithRowsNum(1) - - reservationRepo := getReservationRepo(t) - - err := reservationRepo.Create(context.Background(), expectedReservation, time.Now()) - assert.NoError(t, err) -} - func TestUpdate(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 1d4bbed6..0592f3af 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -10,18 +10,17 @@ import ( // Interface to interact with Reservation Table type ReservationRepo interface { + // Create a new reservation if the reservation does not already exist + Create(ctx context.Context, reservation models.Reservation, now time.Time) error + + // Delete a reservation if it exists + Delete(ctx context.Context, reservation models.ReservationKey) error + // Get reservation Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) - // TODO - comment - Create(ctx context.Context, reservation models.Reservation, now time.Time) error - - // TODO - comment + // Update an existing reservation. If called by the current owner, we update the + // expiresAt timestamp. If called by a new owner and the current reservation has + // expired, we attempt to take over the reservation. Update(ctx context.Context, reservation models.Reservation, now time.Time) error - - // Create the reservation. If the reservation already exists, we try to take over the - // reservation via update when the reservation has expired. Note: Each reservation has its own - // expire date which is tracked in expire_at column in the reservation table. And the - // reservation expires when the date stored in expire_at column is in the past. - //CreateOrUpdate(ctx context.Context, reservation models.Reservation, now time.Time) error } diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go index 8aa000bf..c8f0782c 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation.go @@ -49,6 +49,38 @@ func (_m *ReservationRepo) Create(ctx context.Context, reservation models.Reserv return r0 } +type ReservationRepo_Delete struct { + *mock.Call +} + +func (_m ReservationRepo_Delete) Return(_a0 error) *ReservationRepo_Delete { + return &ReservationRepo_Delete{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnDelete(ctx context.Context, reservation models.ReservationKey) *ReservationRepo_Delete { + c := _m.On("Delete", ctx, reservation) + return &ReservationRepo_Delete{Call: c} +} + +func (_m *ReservationRepo) OnDeleteMatch(matchers ...interface{}) *ReservationRepo_Delete { + c := _m.On("Delete", matchers...) + return &ReservationRepo_Delete{Call: c} +} + +// Delete provides a mock function with given fields: ctx, reservation +func (_m *ReservationRepo) Delete(ctx context.Context, reservation models.ReservationKey) error { + ret := _m.Called(ctx, reservation) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey) error); ok { + r0 = rf(ctx, reservation) + } else { + r0 = ret.Error(0) + } + + return r0 +} + type ReservationRepo_Get struct { *mock.Call } diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go index 8ed89cde..d27c9b45 100644 --- a/pkg/repositories/transformers/reservation.go +++ b/pkg/repositories/transformers/reservation.go @@ -12,7 +12,8 @@ import ( "google.golang.org/grpc/codes" ) -func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.ReservationKey { +// TODO - remove +/*func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.ReservationKey { return models.ReservationKey{ DatasetProject: datasetID.Project, DatasetName: datasetID.Name, @@ -20,6 +21,18 @@ func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.Re DatasetVersion: datasetID.Version, TagName: tagName, } +}*/ + +func FromReservationID(reservationID *datacatalog.ReservationID) models.ReservationKey { + datasetID := reservationID.DatasetId + + return models.ReservationKey{ + DatasetProject: datasetID.Project, + DatasetDomain: datasetID.Domain, + DatasetName: datasetID.Name, + DatasetVersion: datasetID.Version, + TagName: reservationID.TagName, + } } func CreateReservationStatus(reservation models.Reservation, heartbeatInterval time.Duration, state datacatalog.ReservationStatus_State) (datacatalog.ReservationStatus, error) { From f5efc5bab9e019aa3acc9a87cdefb3b4cd2a7d07 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 17:29:01 -0500 Subject: [PATCH 39/58] added unit tests for reservation transformer Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 6 +- pkg/manager/impl/reservation_manager_test.go | 25 ++++---- pkg/repositories/transformers/reservation.go | 26 ++++---- .../transformers/reservation_test.go | 60 +++++++++++++++---- 4 files changed, 75 insertions(+), 42 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 03c801ae..f6f94543 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -163,7 +163,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) reservationStatus, err := transformers.CreateReservationStatus( - reservation, r.heartbeatInterval, + &reservation, r.heartbeatInterval, datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) if err != nil { return reservationStatus, err @@ -183,7 +183,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat } reservationStatus, err := transformers.CreateReservationStatus( - rsv1, r.heartbeatInterval, + &rsv1, r.heartbeatInterval, datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) if err != nil { return reservationStatus, err @@ -197,7 +197,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat } // Reservation has been acquired or extended without error - reservationStatus, err := transformers.CreateReservationStatus(newReservation, + reservationStatus, err := transformers.CreateReservationStatus(&newReservation, r.heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) if err != nil { return reservationStatus, err diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index c4ca69aa..d14181fc 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -29,6 +29,10 @@ var datasetID = datacatalog.DatasetID{ Domain: domain, Version: version, } +var reservationID = datacatalog.ReservationID{ + DatasetId: &datasetID, + TagName: tagName, +} var heartbeatInterval = time.Second * 5 var heartbeatGracePeriodMultiplier = time.Second * 3 var prevOwner = "prevOwner" @@ -63,8 +67,7 @@ func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { heartbeatInterval, time.Now, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ - DatasetId: &datasetID, - TagName: tagName, + ReservationId: &reservationID, OwnerId: currentOwner, } @@ -118,9 +121,8 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ - DatasetId: &datasetID, - TagName: tagName, - OwnerId: currentOwner, + ReservationId: &reservationID, + OwnerId: currentOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) @@ -159,9 +161,8 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ - DatasetId: &datasetID, - TagName: tagName, - OwnerId: currentOwner, + ReservationId: &reservationID, + OwnerId: currentOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) @@ -235,8 +236,7 @@ func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ - DatasetId: &datasetID, - TagName: tagName, + ReservationId: &reservationID, OwnerId: prevOwner, } @@ -262,9 +262,8 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.GetOrReserveArtifactRequest{ - DatasetId: &datasetID, - TagName: tagName, - OwnerId: currentOwner, + ReservationId: &reservationID, + OwnerId: currentOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go index d27c9b45..71a471a8 100644 --- a/pkg/repositories/transformers/reservation.go +++ b/pkg/repositories/transformers/reservation.go @@ -12,17 +12,6 @@ import ( "google.golang.org/grpc/codes" ) -// TODO - remove -/*func ToReservationKey(datasetID datacatalog.DatasetID, tagName string) models.ReservationKey { - return models.ReservationKey{ - DatasetProject: datasetID.Project, - DatasetName: datasetID.Name, - DatasetDomain: datasetID.Domain, - DatasetVersion: datasetID.Version, - TagName: tagName, - } -}*/ - func FromReservationID(reservationID *datacatalog.ReservationID) models.ReservationKey { datasetID := reservationID.DatasetId @@ -35,17 +24,26 @@ func FromReservationID(reservationID *datacatalog.ReservationID) models.Reservat } } -func CreateReservationStatus(reservation models.Reservation, heartbeatInterval time.Duration, state datacatalog.ReservationStatus_State) (datacatalog.ReservationStatus, error) { - expiresAtPb, err := ptypes.TimestampProto(reservation.CreatedAt) +func CreateReservationStatus(reservation *models.Reservation, heartbeatInterval time.Duration, state datacatalog.ReservationStatus_State) (datacatalog.ReservationStatus, error) { + expiresAtPb, err := ptypes.TimestampProto(reservation.ExpiresAt) if err != nil { return datacatalog.ReservationStatus{}, errors.NewDataCatalogErrorf(codes.Internal, "failed to serialize expires at time") } heartbeatIntervalPb := ptypes.DurationProto(heartbeatInterval) return datacatalog.ReservationStatus{ + ReservationId: &datacatalog.ReservationID { + DatasetId: &datacatalog.DatasetID { + Project: reservation.DatasetProject, + Domain: reservation.DatasetDomain, + Name: reservation.DatasetName, + Version: reservation.DatasetVersion, + }, + TagName: reservation.TagName, + }, + OwnerId: reservation.OwnerID, State: state, ExpiresAt: expiresAtPb, HeartbeatInterval: heartbeatIntervalPb, - OwnerId: reservation.OwnerID, }, nil } diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go index 998c025b..496a1487 100644 --- a/pkg/repositories/transformers/reservation_test.go +++ b/pkg/repositories/transformers/reservation_test.go @@ -2,23 +2,59 @@ package transformers import ( "testing" + "time" + "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/magiconair/properties/assert" ) -func TestToReservationKey(t *testing.T) { - datasetID := datacatalog.DatasetID{ - Project: "p", - Name: "n", - Domain: "d", - Version: "v", +func TestFromReservationID(t *testing.T) { + reservationID := datacatalog.ReservationID{ + DatasetId: &datacatalog.DatasetID{ + Project: "p", + Name: "n", + Domain: "d", + Version: "v", + }, + TagName: "t", } - reservationKey := ToReservationKey(datasetID, "t") - assert.Equal(t, datasetID.Project, reservationKey.DatasetProject) - assert.Equal(t, datasetID.Name, reservationKey.DatasetName) - assert.Equal(t, datasetID.Domain, reservationKey.DatasetDomain) - assert.Equal(t, datasetID.Version, reservationKey.DatasetVersion) - assert.Equal(t, "t", reservationKey.TagName) + reservationKey := FromReservationID(&reservationID) + assert.Equal(t, reservationKey.DatasetProject, reservationID.DatasetId.Project) + assert.Equal(t, reservationKey.DatasetName, reservationID.DatasetId.Name) + assert.Equal(t, reservationKey.DatasetDomain, reservationID.DatasetId.Domain) + assert.Equal(t, reservationKey.DatasetVersion, reservationID.DatasetId.Version) + assert.Equal(t, reservationKey.TagName, reservationID.TagName) +} + +func TestCreateReservationStatus(t *testing.T) { + now := time.Now() + heartbeatInterval := time.Duration(time.Second * 5) + reservation := models.Reservation { + ReservationKey: models.ReservationKey { + DatasetProject: "p", + DatasetName: "n", + DatasetDomain: "d", + DatasetVersion: "v", + TagName: "t", + }, + OwnerID: "o", + ExpiresAt: now, + } + + reservationStatus, err := CreateReservationStatus(&reservation, heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) + + assert.Equal(t, err, nil) + assert.Equal(t, reservationStatus.OwnerId, reservation.OwnerID) + assert.Equal(t, reservationStatus.ExpiresAt.AsTime(), reservation.ExpiresAt.UTC()) + + reservationID := reservationStatus.ReservationId + assert.Equal(t, reservationID.TagName, reservation.TagName) + + datasetID := reservationID.DatasetId + assert.Equal(t, datasetID.Project, reservation.DatasetProject) + assert.Equal(t, datasetID.Name, reservation.DatasetName) + assert.Equal(t, datasetID.Domain, reservation.DatasetDomain) + assert.Equal(t, datasetID.Version, reservation.DatasetVersion) } From 89f41cd8ff7bd8983cfdc4b865d819976495172c Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 17:43:53 -0500 Subject: [PATCH 40/58] fixed lint errors Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 14 +++++++------- pkg/manager/impl/reservation_manager_test.go | 4 ++-- pkg/repositories/gormimpl/reservation.go | 4 ++-- pkg/repositories/transformers/reservation.go | 4 ++-- pkg/repositories/transformers/reservation_test.go | 11 ++++++----- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index f6f94543..8c234393 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -88,14 +88,14 @@ func NewReservationManager( } func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { - reservationId := request.ReservationId - tagKey := transformers.ToTagKey(reservationId.DatasetId, reservationId.TagName) + reservationID := request.ReservationId + tagKey := transformers.ToTagKey(reservationID.DatasetId, reservationID.TagName) tag, err := r.repo.TagRepo().Get(ctx, tagKey) if err != nil { if errors.IsDoesNotExistError(err) { // Tag does not exist yet, let's acquire the reservation to work on // generating the artifact. - status, err := r.tryAcquireReservation(ctx, reservationId, request.OwnerId) + status, err := r.tryAcquireReservation(ctx, reservationID, request.OwnerId) if err != nil { r.systemMetrics.acquireReservationFailure.Inc(ctx) return nil, err @@ -130,9 +130,9 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * // to do a GET here because we want to know who owns the reservation // and show it to users on the UI. However, the reservation is held by a single // task most of the times and there is no need to do a write. -func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationId *datacatalog.ReservationID, ownerId string) (datacatalog.ReservationStatus, error) { +func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationID *datacatalog.ReservationID, ownerID string) (datacatalog.ReservationStatus, error) { repo := r.repo.ReservationRepo() - reservationKey := transformers.FromReservationID(reservationId) + reservationKey := transformers.FromReservationID(reservationID) reservation, err := repo.Get(ctx, reservationKey) reservationExists := true @@ -148,7 +148,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat now := r.now() newReservation := models.Reservation{ ReservationKey: reservationKey, - OwnerID: ownerId, + OwnerID: ownerID, ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), } @@ -157,7 +157,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat var repoErr error if !reservationExists { repoErr = repo.Create(ctx, newReservation, now) - } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == ownerId { + } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == ownerID { repoErr = repo.Update(ctx, newReservation, now) } else { logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index d14181fc..7a8f958d 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -68,7 +68,7 @@ func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { req := datacatalog.GetOrReserveArtifactRequest{ ReservationId: &reservationID, - OwnerId: currentOwner, + OwnerId: currentOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) @@ -237,7 +237,7 @@ func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { req := datacatalog.GetOrReserveArtifactRequest{ ReservationId: &reservationID, - OwnerId: prevOwner, + OwnerId: prevOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 53bae84f..31f9f516 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -67,14 +67,14 @@ func (r *reservationRepo) Delete(ctx context.Context, reservationKey models.Rese if result.RowsAffected == 0 { return errors2.GetMissingEntityError("Reservation", - &datacatalog.ReservationID { + &datacatalog.ReservationID{ DatasetId: &datacatalog.DatasetID{ Project: reservationKey.DatasetProject, Domain: reservationKey.DatasetDomain, Name: reservationKey.DatasetName, Version: reservationKey.DatasetVersion, }, - TagName: reservationKey.TagName, + TagName: reservationKey.TagName, }) } diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go index 71a471a8..770d5754 100644 --- a/pkg/repositories/transformers/reservation.go +++ b/pkg/repositories/transformers/reservation.go @@ -32,8 +32,8 @@ func CreateReservationStatus(reservation *models.Reservation, heartbeatInterval heartbeatIntervalPb := ptypes.DurationProto(heartbeatInterval) return datacatalog.ReservationStatus{ - ReservationId: &datacatalog.ReservationID { - DatasetId: &datacatalog.DatasetID { + ReservationId: &datacatalog.ReservationID{ + DatasetId: &datacatalog.DatasetID{ Project: reservation.DatasetProject, Domain: reservation.DatasetDomain, Name: reservation.DatasetName, diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go index 496a1487..87ff6bc2 100644 --- a/pkg/repositories/transformers/reservation_test.go +++ b/pkg/repositories/transformers/reservation_test.go @@ -30,24 +30,25 @@ func TestFromReservationID(t *testing.T) { func TestCreateReservationStatus(t *testing.T) { now := time.Now() - heartbeatInterval := time.Duration(time.Second * 5) - reservation := models.Reservation { - ReservationKey: models.ReservationKey { + heartbeatInterval := time.Second * 5 + reservation := models.Reservation{ + ReservationKey: models.ReservationKey{ DatasetProject: "p", DatasetName: "n", DatasetDomain: "d", DatasetVersion: "v", TagName: "t", }, - OwnerID: "o", + OwnerID: "o", ExpiresAt: now, } reservationStatus, err := CreateReservationStatus(&reservation, heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) assert.Equal(t, err, nil) - assert.Equal(t, reservationStatus.OwnerId, reservation.OwnerID) assert.Equal(t, reservationStatus.ExpiresAt.AsTime(), reservation.ExpiresAt.UTC()) + assert.Equal(t, reservationStatus.HeartbeatInterval.AsDuration(), heartbeatInterval) + assert.Equal(t, reservationStatus.OwnerId, reservation.OwnerID) reservationID := reservationStatus.ReservationId assert.Equal(t, reservationID.TagName, reservation.TagName) From 81e46da35668bc4258ac3548c1434536ab465cb7 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 25 Aug 2021 23:21:36 -0500 Subject: [PATCH 41/58] implemented unit tests for ReleaseReservation API call Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager_test.go | 170 ++++++++++++------ pkg/repositories/gormimpl/reservation_test.go | 31 ++++ 2 files changed, 149 insertions(+), 52 deletions(-) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 7a8f958d..f72825e8 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -9,6 +9,7 @@ import ( "time" errors2 "github.com/flyteorg/datacatalog/pkg/errors" + errors3 "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/mocks" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" @@ -78,13 +79,6 @@ func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { assert.Equal(t, expectedArtifact.ArtifactKey.ArtifactID, artifact.Id) } -func getDatacatalogRepo() mocks.DataCatalogRepo { - return mocks.DataCatalogRepo{ - MockReservationRepo: &mocks.ReservationRepo{}, - MockTagRepo: &mocks.TagRepo{}, - } -} - func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { dcRepo := getDatacatalogRepo() @@ -132,13 +126,13 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) } -func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { +func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) now := time.Now() - prevExpiresAt := now.Add(time.Second * 10 * time.Duration(-1)) + prevExpiresAt := now.Add(time.Second * 10) setUpReservationRepoGet(&dcRepo, prevExpiresAt) @@ -150,7 +144,7 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { reservation.DatasetName == datasetID.Name && reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && - reservation.OwnerID == currentOwner && + reservation.OwnerID == prevOwner && reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) }), mock.MatchedBy(func(now time.Time) bool { return true }), @@ -162,58 +156,23 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { req := datacatalog.GetOrReserveArtifactRequest{ ReservationId: &reservationID, - OwnerId: currentOwner, + OwnerId: prevOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) } -func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpiresAt time.Time) { - dcRepo.MockReservationRepo.On("Get", - mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(key models.ReservationKey) bool { - return key.DatasetProject == datasetID.Project && - key.DatasetDomain == datasetID.Domain && - key.DatasetVersion == datasetID.Version && - key.DatasetName == datasetID.Name && - key.TagName == tagName - })).Return( - models.Reservation{ - ReservationKey: getReservationKey(), - OwnerID: prevOwner, - ExpiresAt: prevExpiresAt, - }, nil, - ) -} - -func setUpTagRepoGetNotFound(dcRepo *mocks.DataCatalogRepo) { - dcRepo.MockTagRepo.On("Get", - mock.Anything, - mock.Anything, - ).Return(models.Tag{}, errors2.NewDataCatalogErrorf(codes.NotFound, "entry not found")) -} - -func getReservationKey() models.ReservationKey { - return models.ReservationKey{ - DatasetProject: project, - DatasetName: name, - DatasetDomain: domain, - DatasetVersion: version, - TagName: tagName, - } -} - -func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { +func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) now := time.Now() - prevExpiresAt := now.Add(time.Second * 10) + prevExpiresAt := now.Add(time.Second * 10 * time.Duration(-1)) setUpReservationRepoGet(&dcRepo, prevExpiresAt) @@ -225,7 +184,7 @@ func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { reservation.DatasetName == datasetID.Name && reservation.DatasetVersion == datasetID.Version && reservation.TagName == tagName && - reservation.OwnerID == prevOwner && + reservation.OwnerID == currentOwner && reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) }), mock.MatchedBy(func(now time.Time) bool { return true }), @@ -237,13 +196,13 @@ func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { req := datacatalog.GetOrReserveArtifactRequest{ ReservationId: &reservationID, - OwnerId: prevOwner, + OwnerId: currentOwner, } resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) + assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) } @@ -272,3 +231,110 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) assert.Equal(t, datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, resp.GetReservationStatus().State) } + +func TestReleaseReservation(t *testing.T) { + dcRepo := getDatacatalogRepo() + + now := time.Now() + + dcRepo.MockReservationRepo.On("Delete", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservationKey models.ReservationKey) bool { + return reservationKey.DatasetProject == datasetID.Project && + reservationKey.DatasetDomain == datasetID.Domain && + reservationKey.DatasetName == datasetID.Name && + reservationKey.DatasetVersion == datasetID.Version && + reservationKey.TagName == tagName + }), + ).Return(nil) + + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) + + req := datacatalog.ReleaseReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + } + + _, err := reservationManager.ReleaseReservation(context.Background(), &req) + + assert.Nil(t, err) +} + +func TestReleaseReservation_Failure(t *testing.T) { + dcRepo := getDatacatalogRepo() + + now := time.Now() + reservationErr := errors3.GetMissingEntityError("Reservation", + &datacatalog.ReservationID{ + DatasetId: &datasetID, + TagName: tagName, + }) + + dcRepo.MockReservationRepo.On("Delete", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservationKey models.ReservationKey) bool { + return reservationKey.DatasetProject == datasetID.Project && + reservationKey.DatasetDomain == datasetID.Domain && + reservationKey.DatasetName == datasetID.Name && + reservationKey.DatasetVersion == datasetID.Version && + reservationKey.TagName == tagName + }), + ).Return(reservationErr) + + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, heartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) + + req := datacatalog.ReleaseReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + } + + _, err := reservationManager.ReleaseReservation(context.Background(), &req) + + assert.Equal(t, err, reservationErr) +} + +func getDatacatalogRepo() mocks.DataCatalogRepo { + return mocks.DataCatalogRepo{ + MockReservationRepo: &mocks.ReservationRepo{}, + MockTagRepo: &mocks.TagRepo{}, + } +} + +func setUpReservationRepoGet(dcRepo *mocks.DataCatalogRepo, prevExpiresAt time.Time) { + dcRepo.MockReservationRepo.On("Get", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(key models.ReservationKey) bool { + return key.DatasetProject == datasetID.Project && + key.DatasetDomain == datasetID.Domain && + key.DatasetVersion == datasetID.Version && + key.DatasetName == datasetID.Name && + key.TagName == tagName + })).Return( + models.Reservation{ + ReservationKey: getReservationKey(), + OwnerID: prevOwner, + ExpiresAt: prevExpiresAt, + }, nil, + ) +} + +func setUpTagRepoGetNotFound(dcRepo *mocks.DataCatalogRepo) { + dcRepo.MockTagRepo.On("Get", + mock.Anything, + mock.Anything, + ).Return(models.Tag{}, errors2.NewDataCatalogErrorf(codes.NotFound, "entry not found")) +} + +func getReservationKey() models.ReservationKey { + return models.ReservationKey{ + DatasetProject: project, + DatasetName: name, + DatasetDomain: domain, + DatasetVersion: version, + TagName: tagName, + } +} diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index e1ec3c46..584b6797 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -38,6 +38,37 @@ func TestCreate(t *testing.T) { assert.NoError(t, err) } +func TestDelete(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5`, + ).WithRowsNum(1) + + reservationRepo := getReservationRepo(t) + + err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey) + assert.NoError(t, err) +} + +func TestDeleteFailure(t *testing.T) { + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + expectedReservation := GetReservation() + + GlobalMock.NewMock().WithQuery( + `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5`, + ).WithRowsNum(0) + + reservationRepo := getReservationRepo(t) + + err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey) + assert.Error(t, err) + assert.Equal(t, "missing entity of type Reservation with identifier dataset_id: tag_name:\"testTag\" ", err.Error()) +} + func TestGet(t *testing.T) { expectedReservation := GetReservation() From e65570a76d29c357e596272631e6f28d2c7b3dc8 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 20 Sep 2021 17:54:37 -0500 Subject: [PATCH 42/58] updated reservation API to only work with reservations - not actual artifacts. Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 85 ++++++-------------- pkg/manager/interfaces/reservation.go | 2 +- pkg/repositories/errors/errors.go | 8 +- pkg/repositories/gormimpl/reservation.go | 10 +-- pkg/repositories/transformers/reservation.go | 9 +-- pkg/rpc/datacatalogservice/service.go | 4 +- 6 files changed, 38 insertions(+), 80 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 8c234393..788a8e3b 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -25,7 +25,6 @@ type reservationMetrics struct { reservationReleased labeled.Counter reservationAlreadyInProgress labeled.Counter acquireReservationFailure labeled.Counter - getTagFailure labeled.Counter releaseReservationFailure labeled.Counter } @@ -66,11 +65,6 @@ func NewReservationManager( "Number of times we failed to acquire reservation", reservationScope, ), - getTagFailure: labeled.NewCounter( - "get_tag_failure", - "Number of times we failed to get tag", - reservationScope, - ), releaseReservationFailure: labeled.NewCounter( "release_reservation_failure", "Number of times we failed to release a reservation", @@ -87,40 +81,16 @@ func NewReservationManager( } } -func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) { +func (r *reservationManager) GetOrExtendReservation(ctx context.Context, request *datacatalog.GetOrExtendReservationRequest) (*datacatalog.GetOrExtendReservationResponse, error) { reservationID := request.ReservationId - tagKey := transformers.ToTagKey(reservationID.DatasetId, reservationID.TagName) - tag, err := r.repo.TagRepo().Get(ctx, tagKey) - if err != nil { - if errors.IsDoesNotExistError(err) { - // Tag does not exist yet, let's acquire the reservation to work on - // generating the artifact. - status, err := r.tryAcquireReservation(ctx, reservationID, request.OwnerId) - if err != nil { - r.systemMetrics.acquireReservationFailure.Inc(ctx) - return nil, err - } - - return &datacatalog.GetOrReserveArtifactResponse{ - Value: &datacatalog.GetOrReserveArtifactResponse_ReservationStatus{ - ReservationStatus: &status, - }, - }, nil - } - logger.Errorf(ctx, "Failed retrieve tag: %+v, err: %v", tagKey, err) - r.systemMetrics.getTagFailure.Inc(ctx) - return nil, err - } - - artifact, err := transformers.FromArtifactModel(tag.Artifact) + reservation, err := r.tryAcquireReservation(ctx, reservationID, request.OwnerId) if err != nil { + r.systemMetrics.acquireReservationFailure.Inc(ctx) return nil, err } - return &datacatalog.GetOrReserveArtifactResponse{ - Value: &datacatalog.GetOrReserveArtifactResponse_Artifact{ - Artifact: artifact, - }, + return &datacatalog.GetOrExtendReservationResponse{ + Reservation: &reservation, }, nil } @@ -130,10 +100,10 @@ func (r *reservationManager) GetOrReserveArtifact(ctx context.Context, request * // to do a GET here because we want to know who owns the reservation // and show it to users on the UI. However, the reservation is held by a single // task most of the times and there is no need to do a write. -func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationID *datacatalog.ReservationID, ownerID string) (datacatalog.ReservationStatus, error) { +func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationID *datacatalog.ReservationID, ownerID string) (datacatalog.Reservation, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.FromReservationID(reservationID) - reservation, err := repo.Get(ctx, reservationKey) + repoReservation, err := repo.Get(ctx, reservationKey) reservationExists := true if err != nil { @@ -141,12 +111,12 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat // Reservation does not exist yet so let's create one reservationExists = false } else { - return datacatalog.ReservationStatus{}, err + return datacatalog.Reservation{}, err } } now := r.now() - newReservation := models.Reservation{ + newRepoReservation := models.Reservation{ ReservationKey: reservationKey, OwnerID: ownerID, ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), @@ -156,55 +126,50 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat // within the reservation repository Create and Update function calls. var repoErr error if !reservationExists { - repoErr = repo.Create(ctx, newReservation, now) - } else if reservation.ExpiresAt.Before(now) || reservation.OwnerID == ownerID { - repoErr = repo.Update(ctx, newReservation, now) + repoErr = repo.Create(ctx, newRepoReservation, now) + } else if repoReservation.ExpiresAt.Before(now) || repoReservation.OwnerID == ownerID { + repoErr = repo.Update(ctx, newRepoReservation, now) } else { - logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, reservation.OwnerID) + logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, repoReservation.OwnerID) - reservationStatus, err := transformers.CreateReservationStatus( - &reservation, r.heartbeatInterval, - datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) + reservation, err := transformers.CreateReservation(&repoReservation, r.heartbeatInterval) if err != nil { - return reservationStatus, err + return reservation, err } r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return reservationStatus, nil + return reservation, nil } if repoErr != nil { - if repoErr.Error() == repo_errors.ReservationAlreadyInProgress { + if repoErr.Error() == repo_errors.AlreadyExists { // Looks like someone else tried to obtain the reservation // at the same time and they won. Let's find out who won. rsv1, err := repo.Get(ctx, reservationKey) if err != nil { - return datacatalog.ReservationStatus{}, err + return datacatalog.Reservation{}, err } - reservationStatus, err := transformers.CreateReservationStatus( - &rsv1, r.heartbeatInterval, - datacatalog.ReservationStatus_ALREADY_IN_PROGRESS) + reservation, err := transformers.CreateReservation(&rsv1, r.heartbeatInterval) if err != nil { - return reservationStatus, err + return reservation, err } r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return reservationStatus, nil + return reservation, nil } - return datacatalog.ReservationStatus{}, repoErr + return datacatalog.Reservation{}, repoErr } // Reservation has been acquired or extended without error - reservationStatus, err := transformers.CreateReservationStatus(&newReservation, - r.heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) + reservation, err := transformers.CreateReservation(&newRepoReservation, r.heartbeatInterval) if err != nil { - return reservationStatus, err + return reservation, err } r.systemMetrics.reservationAlreadyInProgress.Inc(ctx) - return reservationStatus, nil + return reservation, nil } func (r *reservationManager) ReleaseReservation(ctx context.Context, request *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { diff --git a/pkg/manager/interfaces/reservation.go b/pkg/manager/interfaces/reservation.go index 5ddf81f9..42d95b8c 100644 --- a/pkg/manager/interfaces/reservation.go +++ b/pkg/manager/interfaces/reservation.go @@ -10,6 +10,6 @@ import ( // You can find more details about the APIs in datacatalog service proto // in flyteidl type ReservationManager interface { - GetOrReserveArtifact(context.Context, *datacatalog.GetOrReserveArtifactRequest) (*datacatalog.GetOrReserveArtifactResponse, error) + GetOrExtendReservation(context.Context, *datacatalog.GetOrExtendReservationRequest) (*datacatalog.GetOrExtendReservationResponse, error) ReleaseReservation(context.Context, *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) } diff --git a/pkg/repositories/errors/errors.go b/pkg/repositories/errors/errors.go index 81d66b34..1c2a29fc 100644 --- a/pkg/repositories/errors/errors.go +++ b/pkg/repositories/errors/errors.go @@ -9,10 +9,10 @@ import ( ) const ( - notFound = "missing entity of type %s with identifier %v" - invalidJoin = "cannot relate entity %s with entity %s" - invalidEntity = "no such entity %s" - ReservationAlreadyInProgress = "reservation already in progress" + AlreadyExists = "entity already exists" + notFound = "missing entity of type %s with identifier %v" + invalidJoin = "cannot relate entity %s with entity %s" + invalidEntity = "no such entity %s" ) func GetMissingEntityError(entityType string, identifier proto.Message) error { diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 31f9f516..23be4576 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -43,10 +43,7 @@ func (r *reservationRepo) Create(ctx context.Context, reservation models.Reserva } if result.RowsAffected == 0 { - return datacatalog_error.NewDataCatalogError( - codes.FailedPrecondition, - errors2.ReservationAlreadyInProgress, - ) + return datacatalog_error.NewDataCatalogError(codes.FailedPrecondition, errors2.AlreadyExists) } return nil @@ -110,10 +107,7 @@ func (r *reservationRepo) Update(ctx context.Context, reservation models.Reserva } if result.RowsAffected == 0 { - return datacatalog_error.NewDataCatalogError( - codes.FailedPrecondition, - errors2.ReservationAlreadyInProgress, - ) + return datacatalog_error.NewDataCatalogError(codes.FailedPrecondition, errors2.AlreadyExists) } return nil diff --git a/pkg/repositories/transformers/reservation.go b/pkg/repositories/transformers/reservation.go index 770d5754..50e9133b 100644 --- a/pkg/repositories/transformers/reservation.go +++ b/pkg/repositories/transformers/reservation.go @@ -24,14 +24,14 @@ func FromReservationID(reservationID *datacatalog.ReservationID) models.Reservat } } -func CreateReservationStatus(reservation *models.Reservation, heartbeatInterval time.Duration, state datacatalog.ReservationStatus_State) (datacatalog.ReservationStatus, error) { +func CreateReservation(reservation *models.Reservation, heartbeatInterval time.Duration) (datacatalog.Reservation, error) { expiresAtPb, err := ptypes.TimestampProto(reservation.ExpiresAt) if err != nil { - return datacatalog.ReservationStatus{}, errors.NewDataCatalogErrorf(codes.Internal, "failed to serialize expires at time") + return datacatalog.Reservation{}, errors.NewDataCatalogErrorf(codes.Internal, "failed to serialize expires at time") } heartbeatIntervalPb := ptypes.DurationProto(heartbeatInterval) - return datacatalog.ReservationStatus{ + return datacatalog.Reservation{ ReservationId: &datacatalog.ReservationID{ DatasetId: &datacatalog.DatasetID{ Project: reservation.DatasetProject, @@ -42,8 +42,7 @@ func CreateReservationStatus(reservation *models.Reservation, heartbeatInterval TagName: reservation.TagName, }, OwnerId: reservation.OwnerID, - State: state, - ExpiresAt: expiresAtPb, HeartbeatInterval: heartbeatIntervalPb, + ExpiresAt: expiresAtPb, }, nil } diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 88081055..58706d99 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -55,8 +55,8 @@ func (s *DataCatalogService) ListDatasets(ctx context.Context, request *catalog. return s.DatasetManager.ListDatasets(ctx, request) } -func (s *DataCatalogService) GetOrReserveArtifact(ctx context.Context, request *catalog.GetOrReserveArtifactRequest) (*catalog.GetOrReserveArtifactResponse, error) { - return s.ReservationManager.GetOrReserveArtifact(ctx, request) +func (s *DataCatalogService) GetOrExtendReservation(ctx context.Context, request *catalog.GetOrExtendReservationRequest) (*catalog.GetOrExtendReservationResponse, error) { + return s.ReservationManager.GetOrExtendReservation(ctx, request) } func (s *DataCatalogService) ReleaseReservation(ctx context.Context, request *catalog.ReleaseReservationRequest) (*catalog.ReleaseReservationResponse, error) { From 06fcf9006f9ac5db445ecb5891d643012b566a6c Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 1 Sep 2021 16:40:34 -0700 Subject: [PATCH 43/58] Fix error type check to detect uniqueConstraintViolation Signed-off-by: Haytham Abuelfutuh Signed-off-by: Daniel Rammer --- pkg/manager/impl/artifact_manager.go | 2 +- pkg/repositories/errors/postgres.go | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index 85553d1a..7ac97d46 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -161,7 +161,7 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request *datacatalog. if err != nil { if errors.IsDoesNotExistError(err) { - logger.Warnf(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) + logger.Infof(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) m.systemMetrics.doesNotExistCounter.Inc(ctx) } else { logger.Errorf(ctx, "Unable to retrieve Artifact by tag %v, err: %v", request.GetTagName(), err) diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index bbdd7d59..257fedf9 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -2,6 +2,9 @@ package errors import ( "fmt" + "reflect" + + "github.com/flyteorg/flytestdlib/logger" "github.com/jackc/pgconn" @@ -22,7 +25,7 @@ type postgresErrorTransformer struct { const ( unexpectedType = "unexpected error type for: %v" uniqueConstraintViolation = "value with matching already exists (%s)" - defaultPgError = "failed database operation with %s" + defaultPgError = "failed database operation with code [%s] and msg [%s]" unsupportedTableOperation = "cannot query with specified table attributes: %s" ) @@ -36,18 +39,20 @@ func (p *postgresErrorTransformer) fromGormError(err error) error { } func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { - cErr, ok := err.(ConnectError) + pqError, ok := err.(*pgconn.PgError) if !ok { + logger.InfofNoCtx("Unable to cast to pgconn.PgError. Error type: [%v]", + reflect.TypeOf(err)) return p.fromGormError(err) } - pqError := cErr.Unwrap().(*pgconn.PgError) + switch pqError.Code { case uniqueConstraintViolationCode: return errors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) case undefinedTable: return errors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) default: - return errors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Message)) + return errors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Code, pqError.Message)) } } From c21349cb1b19ad2d85631720835f784bd332da85 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 1 Sep 2021 16:46:43 -0700 Subject: [PATCH 44/58] Revert "Fix error type check to detect uniqueConstraintViolation" This reverts commit 2ba07757a31fe47d1c49141c6cd6d6e499e5f60c. Signed-off-by: Daniel Rammer --- go.sum | 3 +++ pkg/manager/impl/artifact_manager.go | 2 +- pkg/repositories/errors/postgres.go | 13 ++++--------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/go.sum b/go.sum index 0a3331b0..a8ea6f61 100644 --- a/go.sum +++ b/go.sum @@ -434,6 +434,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -685,6 +686,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -765,6 +767,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index 7ac97d46..85553d1a 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -161,7 +161,7 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request *datacatalog. if err != nil { if errors.IsDoesNotExistError(err) { - logger.Infof(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) + logger.Warnf(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) m.systemMetrics.doesNotExistCounter.Inc(ctx) } else { logger.Errorf(ctx, "Unable to retrieve Artifact by tag %v, err: %v", request.GetTagName(), err) diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index 257fedf9..bbdd7d59 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -2,9 +2,6 @@ package errors import ( "fmt" - "reflect" - - "github.com/flyteorg/flytestdlib/logger" "github.com/jackc/pgconn" @@ -25,7 +22,7 @@ type postgresErrorTransformer struct { const ( unexpectedType = "unexpected error type for: %v" uniqueConstraintViolation = "value with matching already exists (%s)" - defaultPgError = "failed database operation with code [%s] and msg [%s]" + defaultPgError = "failed database operation with %s" unsupportedTableOperation = "cannot query with specified table attributes: %s" ) @@ -39,20 +36,18 @@ func (p *postgresErrorTransformer) fromGormError(err error) error { } func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { - pqError, ok := err.(*pgconn.PgError) + cErr, ok := err.(ConnectError) if !ok { - logger.InfofNoCtx("Unable to cast to pgconn.PgError. Error type: [%v]", - reflect.TypeOf(err)) return p.fromGormError(err) } - + pqError := cErr.Unwrap().(*pgconn.PgError) switch pqError.Code { case uniqueConstraintViolationCode: return errors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) case undefinedTable: return errors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) default: - return errors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Code, pqError.Message)) + return errors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Message)) } } From 2c968ee2045cf837dfe5c58561d40a185d23fca3 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 1 Sep 2021 17:37:07 -0700 Subject: [PATCH 45/58] Fix gorm wrong error type cast (#48) * Fix gorm wrong error type cast Signed-off-by: Haytham Abuelfutuh * Fix unit tests Signed-off-by: Haytham Abuelfutuh * Support unwrapping errors to detect connection problems Signed-off-by: Haytham Abuelfutuh * cleanup Signed-off-by: Haytham Abuelfutuh Signed-off-by: Daniel Rammer --- go.sum | 3 --- pkg/manager/impl/artifact_manager.go | 2 +- pkg/repositories/errors/postgres.go | 28 +++++++++++++++------- pkg/repositories/gormimpl/artifact_test.go | 2 +- pkg/repositories/gormimpl/dataset_test.go | 2 +- pkg/repositories/gormimpl/tag_test.go | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/go.sum b/go.sum index a8ea6f61..0a3331b0 100644 --- a/go.sum +++ b/go.sum @@ -434,7 +434,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -686,7 +685,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -767,7 +765,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index 85553d1a..7ac97d46 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -161,7 +161,7 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request *datacatalog. if err != nil { if errors.IsDoesNotExistError(err) { - logger.Warnf(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) + logger.Infof(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) m.systemMetrics.doesNotExistCounter.Inc(ctx) } else { logger.Errorf(ctx, "Unable to retrieve Artifact by tag %v, err: %v", request.GetTagName(), err) diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index bbdd7d59..39cdddcc 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -1,11 +1,15 @@ package errors import ( + "errors" "fmt" + "reflect" + + "github.com/flyteorg/flytestdlib/logger" "github.com/jackc/pgconn" - "github.com/flyteorg/datacatalog/pkg/errors" + catalogErrors "github.com/flyteorg/datacatalog/pkg/errors" "google.golang.org/grpc/codes" "gorm.io/gorm" ) @@ -22,32 +26,38 @@ type postgresErrorTransformer struct { const ( unexpectedType = "unexpected error type for: %v" uniqueConstraintViolation = "value with matching already exists (%s)" - defaultPgError = "failed database operation with %s" + defaultPgError = "failed database operation with code [%s] and msg [%s]" unsupportedTableOperation = "cannot query with specified table attributes: %s" ) func (p *postgresErrorTransformer) fromGormError(err error) error { switch err.Error() { case gorm.ErrRecordNotFound.Error(): - return errors.NewDataCatalogErrorf(codes.NotFound, "entry not found") + return catalogErrors.NewDataCatalogErrorf(codes.NotFound, "entry not found") default: - return errors.NewDataCatalogErrorf(codes.Internal, unexpectedType, err) + return catalogErrors.NewDataCatalogErrorf(codes.Internal, unexpectedType, err) } } func (p *postgresErrorTransformer) ToDataCatalogError(err error) error { - cErr, ok := err.(ConnectError) + if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil { + err = unwrappedErr + } + + pqError, ok := err.(*pgconn.PgError) if !ok { + logger.InfofNoCtx("Unable to cast to pgconn.PgError. Error type: [%v]", + reflect.TypeOf(err)) return p.fromGormError(err) } - pqError := cErr.Unwrap().(*pgconn.PgError) + switch pqError.Code { case uniqueConstraintViolationCode: - return errors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) + return catalogErrors.NewDataCatalogErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) case undefinedTable: - return errors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) + return catalogErrors.NewDataCatalogErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) default: - return errors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Message)) + return catalogErrors.NewDataCatalogErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Code, pqError.Message)) } } diff --git a/pkg/repositories/gormimpl/artifact_test.go b/pkg/repositories/gormimpl/artifact_test.go index 49292fb8..1bff6f85 100644 --- a/pkg/repositories/gormimpl/artifact_test.go +++ b/pkg/repositories/gormimpl/artifact_test.go @@ -275,7 +275,7 @@ func TestCreateArtifactAlreadyExists(t *testing.T) { assert.Error(t, err) dcErr, ok := err.(apiErrors.DataCatalogError) assert.True(t, ok) - assert.Equal(t, dcErr.Code(), codes.AlreadyExists) + assert.Equal(t, dcErr.Code().String(), codes.AlreadyExists.String()) } func TestListArtifactsWithPartition(t *testing.T) { diff --git a/pkg/repositories/gormimpl/dataset_test.go b/pkg/repositories/gormimpl/dataset_test.go index 2f2ef8e9..a4904da1 100644 --- a/pkg/repositories/gormimpl/dataset_test.go +++ b/pkg/repositories/gormimpl/dataset_test.go @@ -246,7 +246,7 @@ func TestCreateDatasetAlreadyExists(t *testing.T) { assert.Error(t, err) dcErr, ok := err.(datacatalog_error.DataCatalogError) assert.True(t, ok) - assert.Equal(t, dcErr.Code(), codes.AlreadyExists) + assert.Equal(t, dcErr.Code().String(), codes.AlreadyExists.String()) } func TestListDatasets(t *testing.T) { diff --git a/pkg/repositories/gormimpl/tag_test.go b/pkg/repositories/gormimpl/tag_test.go index ee33417a..9dbb4270 100644 --- a/pkg/repositories/gormimpl/tag_test.go +++ b/pkg/repositories/gormimpl/tag_test.go @@ -157,5 +157,5 @@ func TestTagAlreadyExists(t *testing.T) { assert.Error(t, err) dcErr, ok := err.(datacatalog_error.DataCatalogError) assert.True(t, ok) - assert.Equal(t, dcErr.Code(), codes.AlreadyExists) + assert.Equal(t, dcErr.Code().String(), codes.AlreadyExists.String()) } From bb05d0fbc3cc9ac90571a5917778ac73597433e6 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Thu, 23 Sep 2021 06:59:46 -0500 Subject: [PATCH 46/58] added support for heartbeat_interval definition in reservation manager Signed-off-by: Daniel Rammer --- datacatalog_config.yaml | 2 +- pkg/manager/impl/reservation_manager.go | 26 ++++++++++++++-------- pkg/rpc/datacatalogservice/service.go | 2 +- pkg/runtime/configs/data_catalog_config.go | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/datacatalog_config.yaml b/datacatalog_config.yaml index 874d9de7..0f811395 100644 --- a/datacatalog_config.yaml +++ b/datacatalog_config.yaml @@ -12,7 +12,7 @@ datacatalog: metrics-scope: "datacatalog" profiler-port: 10254 heartbeat-grace-period-multiplier: 3 - heartbeat-interval-sec: 5 + max-reservation-heartbeat-sec: 10 storage: connection: access-key: minio diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 788a8e3b..fb6f37d4 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -33,7 +33,7 @@ type NowFunc func() time.Time type reservationManager struct { repo repositories.RepositoryInterface heartbeatGracePeriodMultiplier time.Duration - heartbeatInterval time.Duration + maxHeartbeatInterval time.Duration now NowFunc systemMetrics reservationMetrics } @@ -41,7 +41,7 @@ type reservationManager struct { func NewReservationManager( repo repositories.RepositoryInterface, heartbeatGracePeriodMultiplier time.Duration, - heartbeatInterval time.Duration, + maxHeartbeatInterval time.Duration, nowFunc NowFunc, // Easier to mock time.Time for testing reservationScope promutils.Scope, ) interfaces.ReservationManager { @@ -75,7 +75,7 @@ func NewReservationManager( return &reservationManager{ repo: repo, heartbeatGracePeriodMultiplier: heartbeatGracePeriodMultiplier, - heartbeatInterval: heartbeatInterval, + maxHeartbeatInterval: maxHeartbeatInterval, now: nowFunc, systemMetrics: systemMetrics, } @@ -83,7 +83,15 @@ func NewReservationManager( func (r *reservationManager) GetOrExtendReservation(ctx context.Context, request *datacatalog.GetOrExtendReservationRequest) (*datacatalog.GetOrExtendReservationResponse, error) { reservationID := request.ReservationId - reservation, err := r.tryAcquireReservation(ctx, reservationID, request.OwnerId) + + // Use minimum of maxHeartbeatInterval and requested heartbeat interval + heartbeatInterval := r.maxHeartbeatInterval + requestHeartbeatInterval := request.GetHeartbeatInterval() + if requestHeartbeatInterval != nil && requestHeartbeatInterval.AsDuration() < heartbeatInterval { + heartbeatInterval = requestHeartbeatInterval.AsDuration() + } + + reservation, err := r.tryAcquireReservation(ctx, reservationID, request.OwnerId, heartbeatInterval) if err != nil { r.systemMetrics.acquireReservationFailure.Inc(ctx) return nil, err @@ -100,7 +108,7 @@ func (r *reservationManager) GetOrExtendReservation(ctx context.Context, request // to do a GET here because we want to know who owns the reservation // and show it to users on the UI. However, the reservation is held by a single // task most of the times and there is no need to do a write. -func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationID *datacatalog.ReservationID, ownerID string) (datacatalog.Reservation, error) { +func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservationID *datacatalog.ReservationID, ownerID string, heartbeatInterval time.Duration) (datacatalog.Reservation, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.FromReservationID(reservationID) repoReservation, err := repo.Get(ctx, reservationKey) @@ -119,7 +127,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat newRepoReservation := models.Reservation{ ReservationKey: reservationKey, OwnerID: ownerID, - ExpiresAt: now.Add(r.heartbeatInterval * r.heartbeatGracePeriodMultiplier), + ExpiresAt: now.Add(heartbeatInterval * r.heartbeatGracePeriodMultiplier), } // Conditional upsert on reservation. Race conditions are handled @@ -132,7 +140,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat } else { logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationKey, repoReservation.OwnerID) - reservation, err := transformers.CreateReservation(&repoReservation, r.heartbeatInterval) + reservation, err := transformers.CreateReservation(&repoReservation, heartbeatInterval) if err != nil { return reservation, err } @@ -150,7 +158,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat return datacatalog.Reservation{}, err } - reservation, err := transformers.CreateReservation(&rsv1, r.heartbeatInterval) + reservation, err := transformers.CreateReservation(&rsv1, heartbeatInterval) if err != nil { return reservation, err } @@ -163,7 +171,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat } // Reservation has been acquired or extended without error - reservation, err := transformers.CreateReservation(&newRepoReservation, r.heartbeatInterval) + reservation, err := transformers.CreateReservation(&newRepoReservation, heartbeatInterval) if err != nil { return reservation, err } diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 58706d99..8af223e6 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -120,7 +120,7 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(repos, time.Duration(dataCatalogConfig.HeartbeatGracePeriodMultiplier), time.Second*time.Duration(dataCatalogConfig.HeartbeatIntervalSec), time.Now, + ReservationManager: impl.NewReservationManager(repos, time.Duration(dataCatalogConfig.HeartbeatGracePeriodMultiplier), time.Second*time.Duration(dataCatalogConfig.MaxReservationHeartbeatSec), time.Now, catalogScope.NewSubScope("reservation")), } } diff --git a/pkg/runtime/configs/data_catalog_config.go b/pkg/runtime/configs/data_catalog_config.go index a1c331b6..e216755f 100644 --- a/pkg/runtime/configs/data_catalog_config.go +++ b/pkg/runtime/configs/data_catalog_config.go @@ -8,5 +8,5 @@ type DataCatalogConfig struct { MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` HeartbeatGracePeriodMultiplier int `json:"heartbeat-grace-period-multiplier" pflag:",Number of heartbeats before a reservation expires without an extension."` - HeartbeatIntervalSec int `json:"heartbeat-interval-sec" pflag:",Recommended reservation extension heartbeat interval."` + MaxReservationHeartbeatSec int `json:"max-reservation-heartbeat-sec" pflag:",The maximum available reservation extension heartbeat interval."` } From 5b46ec6b96515e9f752348210d8df1c3b5edb503 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Thu, 23 Sep 2021 10:41:35 -0500 Subject: [PATCH 47/58] updated test and fixed lint errors Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 2 +- pkg/manager/impl/reservation_manager_test.go | 138 ++++++++++-------- pkg/repositories/gormimpl/reservation_test.go | 2 +- .../transformers/reservation_test.go | 24 +-- 4 files changed, 89 insertions(+), 77 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index fb6f37d4..ad3fb75e 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -75,7 +75,7 @@ func NewReservationManager( return &reservationManager{ repo: repo, heartbeatGracePeriodMultiplier: heartbeatGracePeriodMultiplier, - maxHeartbeatInterval: maxHeartbeatInterval, + maxHeartbeatInterval: maxHeartbeatInterval, now: nowFunc, systemMetrics: systemMetrics, } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index f72825e8..689e95d1 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -13,7 +13,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/mocks" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" - "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" @@ -35,51 +35,62 @@ var reservationID = datacatalog.ReservationID{ TagName: tagName, } var heartbeatInterval = time.Second * 5 +var heartbeatIntervalPb = ptypes.DurationProto(heartbeatInterval) +var maxHeartbeatInterval = time.Second * 10 +var maxHeartbeatIntervalPb = ptypes.DurationProto(maxHeartbeatInterval) var heartbeatGracePeriodMultiplier = time.Second * 3 var prevOwner = "prevOwner" var currentOwner = "currentOwner" -func TestGetOrReserveArtifact_ArtifactExists(t *testing.T) { - serializedMetadata, err := proto.Marshal(&datacatalog.Metadata{}) - assert.Nil(t, err) - expectedArtifact := models.Artifact{ - ArtifactKey: models.ArtifactKey{ - ArtifactID: "123", - }, - SerializedMetadata: serializedMetadata, - } - +func TestGetOrExtendReservation_CreateReservation(t *testing.T) { dcRepo := getDatacatalogRepo() - dcRepo.MockTagRepo.On("Get", + setUpTagRepoGetNotFound(&dcRepo) + + dcRepo.MockReservationRepo.On("Get", mock.MatchedBy(func(ctx context.Context) bool { return true }), - mock.MatchedBy(func(tagKey models.TagKey) bool { - return tagKey.DatasetProject == datasetID.Project && - tagKey.DatasetName == datasetID.Name && - tagKey.DatasetDomain == datasetID.Domain && - tagKey.DatasetVersion == datasetID.Version && - tagKey.TagName == tagName + mock.MatchedBy(func(key models.ReservationKey) bool { + return key.DatasetProject == datasetID.Project && + key.DatasetDomain == datasetID.Domain && + key.DatasetVersion == datasetID.Version && + key.DatasetName == datasetID.Name && + key.TagName == tagName + })).Return(models.Reservation{}, errors2.NewDataCatalogErrorf(codes.NotFound, "entry not found")) + + now := time.Now() + + dcRepo.MockReservationRepo.On("Create", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservation models.Reservation) bool { + return reservation.DatasetProject == datasetID.Project && + reservation.DatasetDomain == datasetID.Domain && + reservation.DatasetName == datasetID.Name && + reservation.DatasetVersion == datasetID.Version && + reservation.TagName == tagName && + reservation.OwnerID == currentOwner && + reservation.ExpiresAt == now.Add(heartbeatInterval*heartbeatGracePeriodMultiplier) }), - ).Return(models.Tag{ - Artifact: expectedArtifact, - }, nil) + mock.MatchedBy(func(now time.Time) bool { return true }), + ).Return(nil) - reservationManager := NewReservationManager(&dcRepo, heartbeatGracePeriodMultiplier, - heartbeatInterval, time.Now, mockScope.NewTestScope()) + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) - req := datacatalog.GetOrReserveArtifactRequest{ - ReservationId: &reservationID, - OwnerId: currentOwner, + req := datacatalog.GetOrExtendReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + HeartbeatInterval: heartbeatIntervalPb, } - resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + resp, err := reservationManager.GetOrExtendReservation(context.Background(), &req) + assert.Nil(t, err) - artifact := resp.GetArtifact() - assert.NotNil(t, artifact) - assert.Equal(t, expectedArtifact.ArtifactKey.ArtifactID, artifact.Id) + assert.Equal(t, currentOwner, resp.GetReservation().OwnerId) + assert.Equal(t, heartbeatIntervalPb, resp.GetReservation().HeartbeatInterval) } -func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { +func TestGetOrExtendReservation_MaxHeartbeatInterval(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) @@ -114,19 +125,20 @@ func TestGetOrReserveArtifact_CreateReservation(t *testing.T) { heartbeatGracePeriodMultiplier, heartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) - req := datacatalog.GetOrReserveArtifactRequest{ - ReservationId: &reservationID, - OwnerId: currentOwner, + req := datacatalog.GetOrExtendReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + HeartbeatInterval: maxHeartbeatIntervalPb, } - resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + resp, err := reservationManager.GetOrExtendReservation(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) - assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) + assert.Equal(t, currentOwner, resp.GetReservation().OwnerId) + assert.Equal(t, heartbeatIntervalPb, resp.GetReservation().HeartbeatInterval) } -func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { +func TestGetOrExtendReservation_ExtendReservation(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) @@ -151,22 +163,22 @@ func TestGetOrReserveArtifact_ExtendReservation(t *testing.T) { ).Return(nil) reservationManager := NewReservationManager(&dcRepo, - heartbeatGracePeriodMultiplier, heartbeatInterval, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) - req := datacatalog.GetOrReserveArtifactRequest{ - ReservationId: &reservationID, - OwnerId: prevOwner, + req := datacatalog.GetOrExtendReservationRequest{ + ReservationId: &reservationID, + OwnerId: prevOwner, + HeartbeatInterval: heartbeatIntervalPb, } - resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + resp, err := reservationManager.GetOrExtendReservation(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) - assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) + assert.Equal(t, prevOwner, resp.GetReservation().OwnerId) } -func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { +func TestGetOrExtendReservation_TakeOverReservation(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) @@ -191,22 +203,22 @@ func TestGetOrReserveArtifact_TakeOverReservation(t *testing.T) { ).Return(nil) reservationManager := NewReservationManager(&dcRepo, - heartbeatGracePeriodMultiplier, heartbeatInterval, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) - req := datacatalog.GetOrReserveArtifactRequest{ - ReservationId: &reservationID, - OwnerId: currentOwner, + req := datacatalog.GetOrExtendReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + HeartbeatInterval: heartbeatIntervalPb, } - resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + resp, err := reservationManager.GetOrExtendReservation(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, currentOwner, resp.GetReservationStatus().OwnerId) - assert.Equal(t, datacatalog.ReservationStatus_ACQUIRED, resp.GetReservationStatus().State) + assert.Equal(t, currentOwner, resp.GetReservation().OwnerId) } -func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { +func TestGetOrExtendReservation_ReservationExists(t *testing.T) { dcRepo := getDatacatalogRepo() setUpTagRepoGetNotFound(&dcRepo) @@ -217,19 +229,19 @@ func TestGetOrReserveArtifact_AlreadyInProgress(t *testing.T) { setUpReservationRepoGet(&dcRepo, prevExpiresAt) reservationManager := NewReservationManager(&dcRepo, - heartbeatGracePeriodMultiplier, heartbeatInterval, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) - req := datacatalog.GetOrReserveArtifactRequest{ - ReservationId: &reservationID, - OwnerId: currentOwner, + req := datacatalog.GetOrExtendReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + HeartbeatInterval: heartbeatIntervalPb, } - resp, err := reservationManager.GetOrReserveArtifact(context.Background(), &req) + resp, err := reservationManager.GetOrExtendReservation(context.Background(), &req) assert.Nil(t, err) - assert.Equal(t, prevOwner, resp.GetReservationStatus().OwnerId) - assert.Equal(t, datacatalog.ReservationStatus_ALREADY_IN_PROGRESS, resp.GetReservationStatus().State) + assert.Equal(t, prevOwner, resp.GetReservation().OwnerId) } func TestReleaseReservation(t *testing.T) { @@ -249,7 +261,7 @@ func TestReleaseReservation(t *testing.T) { ).Return(nil) reservationManager := NewReservationManager(&dcRepo, - heartbeatGracePeriodMultiplier, heartbeatInterval, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.ReleaseReservationRequest{ @@ -284,7 +296,7 @@ func TestReleaseReservation_Failure(t *testing.T) { ).Return(reservationErr) reservationManager := NewReservationManager(&dcRepo, - heartbeatGracePeriodMultiplier, heartbeatInterval, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, func() time.Time { return now }, mockScope.NewTestScope()) req := datacatalog.ReleaseReservationRequest{ diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index 584b6797..cd7befc0 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -135,7 +135,7 @@ func TestUpdateFailure(t *testing.T) { err := reservationRepo.Update(context.Background(), expectedReservation, time.Now()) assert.Error(t, err) - assert.Equal(t, "reservation already in progress", err.Error()) + assert.Equal(t, "entity already exists", err.Error()) } func getReservationRepo(t *testing.T) interfaces.ReservationRepo { diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go index 87ff6bc2..803b2460 100644 --- a/pkg/repositories/transformers/reservation_test.go +++ b/pkg/repositories/transformers/reservation_test.go @@ -28,10 +28,10 @@ func TestFromReservationID(t *testing.T) { assert.Equal(t, reservationKey.TagName, reservationID.TagName) } -func TestCreateReservationStatus(t *testing.T) { +func TestCreateReservation(t *testing.T) { now := time.Now() heartbeatInterval := time.Second * 5 - reservation := models.Reservation{ + modelReservation := models.Reservation{ ReservationKey: models.ReservationKey{ DatasetProject: "p", DatasetName: "n", @@ -43,19 +43,19 @@ func TestCreateReservationStatus(t *testing.T) { ExpiresAt: now, } - reservationStatus, err := CreateReservationStatus(&reservation, heartbeatInterval, datacatalog.ReservationStatus_ACQUIRED) + reservation, err := CreateReservation(&modelReservation, heartbeatInterval) assert.Equal(t, err, nil) - assert.Equal(t, reservationStatus.ExpiresAt.AsTime(), reservation.ExpiresAt.UTC()) - assert.Equal(t, reservationStatus.HeartbeatInterval.AsDuration(), heartbeatInterval) - assert.Equal(t, reservationStatus.OwnerId, reservation.OwnerID) + assert.Equal(t, reservation.ExpiresAt.AsTime(), modelReservation.ExpiresAt.UTC()) + assert.Equal(t, reservation.HeartbeatInterval.AsDuration(), heartbeatInterval) + assert.Equal(t, reservation.OwnerId, modelReservation.OwnerID) - reservationID := reservationStatus.ReservationId - assert.Equal(t, reservationID.TagName, reservation.TagName) + reservationID := reservation.ReservationId + assert.Equal(t, reservationID.TagName, modelReservation.TagName) datasetID := reservationID.DatasetId - assert.Equal(t, datasetID.Project, reservation.DatasetProject) - assert.Equal(t, datasetID.Name, reservation.DatasetName) - assert.Equal(t, datasetID.Domain, reservation.DatasetDomain) - assert.Equal(t, datasetID.Version, reservation.DatasetVersion) + assert.Equal(t, datasetID.Project, modelReservation.DatasetProject) + assert.Equal(t, datasetID.Name, modelReservation.DatasetName) + assert.Equal(t, datasetID.Domain, modelReservation.DatasetDomain) + assert.Equal(t, datasetID.Version, modelReservation.DatasetVersion) } From d20ffe4a7aeedada9630f4939de2e111d5062069 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 28 Sep 2021 10:17:25 -0500 Subject: [PATCH 48/58] removed unnecessary dependencies from go.mod Signed-off-by: Daniel Rammer --- go.mod | 4 ++-- pkg/repositories/transformers/reservation_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4279b1ba..2fa498d9 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,6 @@ require ( github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 - github.com/lib/pq v1.3.0 - github.com/magiconair/properties v1.8.4 github.com/jackc/pgconn v1.8.1 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 @@ -19,3 +17,5 @@ require ( gorm.io/driver/postgres v1.1.0 gorm.io/gorm v1.21.9 ) + +replace github.com/flyteorg/flyteidl => ../flyteidl // TODO - tmp diff --git a/pkg/repositories/transformers/reservation_test.go b/pkg/repositories/transformers/reservation_test.go index 803b2460..190c153b 100644 --- a/pkg/repositories/transformers/reservation_test.go +++ b/pkg/repositories/transformers/reservation_test.go @@ -6,7 +6,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" - "github.com/magiconair/properties/assert" + "github.com/stretchr/testify/assert" ) func TestFromReservationID(t *testing.T) { From 0da1bf2dc350935808428860f22da2263d613f33 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 3 Nov 2021 07:46:20 -0500 Subject: [PATCH 49/58] updated flyteidl version - change before merging Signed-off-by: Daniel Rammer --- go.mod | 2 +- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2fa498d9..4a0ee6c3 100644 --- a/go.mod +++ b/go.mod @@ -18,4 +18,4 @@ require ( gorm.io/gorm v1.21.9 ) -replace github.com/flyteorg/flyteidl => ../flyteidl // TODO - tmp +replace github.com/flyteorg/flyteidl => github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51 diff --git a/go.sum b/go.sum index 0a3331b0..b2779407 100644 --- a/go.sum +++ b/go.sum @@ -308,6 +308,9 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51 h1:e9zvtfNKr+K84a7du4wJgC+MXYcEsA13yYGMsEGsjQs= +github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -523,6 +526,7 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -989,6 +993,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -1026,6 +1031,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= From c058b553f3e45888d8c10010e84e6ec5b61e4da1 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 3 Nov 2021 17:25:02 -0500 Subject: [PATCH 50/58] adding reservation model to migration Signed-off-by: Daniel Rammer --- pkg/repositories/handle.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/repositories/handle.go b/pkg/repositories/handle.go index 14c0039b..40420f78 100644 --- a/pkg/repositories/handle.go +++ b/pkg/repositories/handle.go @@ -97,5 +97,9 @@ func (h *DBHandle) Migrate(ctx context.Context) error { return err } + if err := h.db.AutoMigrate(&models.Reservation{}); err != nil { + return err + } + return nil } From 2faa7157bf96132103cf516648bcbd5726d98776 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 15 Nov 2021 19:32:52 -0600 Subject: [PATCH 51/58] udpated dockerfile go template to reflect current master fixing rebase overwrites Signed-off-by: Daniel Rammer --- .../flyte/golang_dockerfile/Dockerfile.GoTemplate | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate index 2b5b775c..2574b5d5 100644 --- a/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate +++ b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate @@ -3,7 +3,7 @@ # # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst -FROM golang:1.13.3-alpine3.10 as builder +FROM golang:1.17.1-alpine3.14 as builder RUN apk add git openssh-client make curl # COPY only the go mod files for efficient caching @@ -20,11 +20,17 @@ COPY . /go/src/github.com/flyteorg/{{REPOSITORY}}/ # The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} RUN make linux_compile +# install grpc-health-probe +RUN curl --silent --fail --show-error --location --output /artifacts/grpc_health_probe "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.5/grpc_health_probe-linux-amd64" && \ + chmod +x /artifacts/grpc_health_probe && \ + echo '8699c46352d752d8f533cae72728b0e65663f399fc28fb9cd854b14ad5f85f44 /artifacts/grpc_health_probe' > .grpc_checksum && \ + sha256sum -c .grpc_checksum + # update the PATH to include the /artifacts directory ENV PATH="/artifacts:${PATH}" # This will eventually move to centurylink/ca-certs:latest for minimum possible image size -FROM alpine:3.10 +FROM alpine:3.14 COPY --from=builder /artifacts /bin RUN apk --update add ca-certificates From 0a956dcb573f8f71dc8e423c40452890e34d9c89 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 15 Nov 2021 19:33:41 -0600 Subject: [PATCH 52/58] and again .. with a space Signed-off-by: Daniel Rammer --- boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate index 2574b5d5..a51f8e1b 100644 --- a/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate +++ b/boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate @@ -20,7 +20,7 @@ COPY . /go/src/github.com/flyteorg/{{REPOSITORY}}/ # The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} RUN make linux_compile -# install grpc-health-probe +# install grpc-health-probe RUN curl --silent --fail --show-error --location --output /artifacts/grpc_health_probe "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.5/grpc_health_probe-linux-amd64" && \ chmod +x /artifacts/grpc_health_probe && \ echo '8699c46352d752d8f533cae72728b0e65663f399fc28fb9cd854b14ad5f85f44 /artifacts/grpc_health_probe' > .grpc_checksum && \ From c61fef93372aa4b83ddd9fab7a027df718ac9247 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 15 Nov 2021 19:45:41 -0600 Subject: [PATCH 53/58] add docs on exports Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index ad3fb75e..0074d911 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -38,6 +38,7 @@ type reservationManager struct { systemMetrics reservationMetrics } +// Creates a new reservation manager with the specified properties func NewReservationManager( repo repositories.RepositoryInterface, heartbeatGracePeriodMultiplier time.Duration, @@ -81,6 +82,8 @@ func NewReservationManager( } } +// Attempt to acquire a reservation for the specified artifact. If ther e is not active reservation, successfully +// acquire it. If you are the owner of the active reservation, extend it. If another owner, return the existing reservation. func (r *reservationManager) GetOrExtendReservation(ctx context.Context, request *datacatalog.GetOrExtendReservationRequest) (*datacatalog.GetOrExtendReservationResponse, error) { reservationID := request.ReservationId @@ -180,6 +183,7 @@ func (r *reservationManager) tryAcquireReservation(ctx context.Context, reservat return reservation, nil } +// Release an active reservation with the specified owner. If one does not exist, gracefully return. func (r *reservationManager) ReleaseReservation(ctx context.Context, request *datacatalog.ReleaseReservationRequest) (*datacatalog.ReleaseReservationResponse, error) { repo := r.repo.ReservationRepo() reservationKey := transformers.FromReservationID(request.ReservationId) From efab0f2004f744d2bfb5443f2068cba4b292ee1e Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 15 Nov 2021 19:53:30 -0600 Subject: [PATCH 54/58] changed configuration to use config.Duration from flytestdlib instead of defining on seconds Signed-off-by: Daniel Rammer --- datacatalog_config.yaml | 2 +- pkg/manager/impl/reservation_manager.go | 2 +- pkg/rpc/datacatalogservice/service.go | 2 +- pkg/runtime/configs/data_catalog_config.go | 14 +++++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/datacatalog_config.yaml b/datacatalog_config.yaml index 0f811395..6065f482 100644 --- a/datacatalog_config.yaml +++ b/datacatalog_config.yaml @@ -12,7 +12,7 @@ datacatalog: metrics-scope: "datacatalog" profiler-port: 10254 heartbeat-grace-period-multiplier: 3 - max-reservation-heartbeat-sec: 10 + max-reservation-heartbeat: 10s storage: connection: access-key: minio diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 0074d911..98908b8d 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -82,7 +82,7 @@ func NewReservationManager( } } -// Attempt to acquire a reservation for the specified artifact. If ther e is not active reservation, successfully +// Attempt to acquire a reservation for the specified artifact. If there is not active reservation, successfully // acquire it. If you are the owner of the active reservation, extend it. If another owner, return the existing reservation. func (r *reservationManager) GetOrExtendReservation(ctx context.Context, request *datacatalog.GetOrExtendReservationRequest) (*datacatalog.GetOrExtendReservationResponse, error) { reservationID := request.ReservationId diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 8af223e6..53371d98 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -120,7 +120,7 @@ func NewDataCatalogService() *DataCatalogService { DatasetManager: impl.NewDatasetManager(repos, dataStorageClient, catalogScope.NewSubScope("dataset")), ArtifactManager: impl.NewArtifactManager(repos, dataStorageClient, storagePrefix, catalogScope.NewSubScope("artifact")), TagManager: impl.NewTagManager(repos, dataStorageClient, catalogScope.NewSubScope("tag")), - ReservationManager: impl.NewReservationManager(repos, time.Duration(dataCatalogConfig.HeartbeatGracePeriodMultiplier), time.Second*time.Duration(dataCatalogConfig.MaxReservationHeartbeatSec), time.Now, + ReservationManager: impl.NewReservationManager(repos, time.Duration(dataCatalogConfig.HeartbeatGracePeriodMultiplier), dataCatalogConfig.MaxReservationHeartbeat.Duration, time.Now, catalogScope.NewSubScope("reservation")), } } diff --git a/pkg/runtime/configs/data_catalog_config.go b/pkg/runtime/configs/data_catalog_config.go index e216755f..ec353a2c 100644 --- a/pkg/runtime/configs/data_catalog_config.go +++ b/pkg/runtime/configs/data_catalog_config.go @@ -1,12 +1,16 @@ package configs +import ( + "github.com/flyteorg/flytestdlib/config" +) + //go:generate pflags DataCatalogConfig // This configuration is the base configuration to start admin type DataCatalogConfig struct { - StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` - MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` - ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` - HeartbeatGracePeriodMultiplier int `json:"heartbeat-grace-period-multiplier" pflag:",Number of heartbeats before a reservation expires without an extension."` - MaxReservationHeartbeatSec int `json:"max-reservation-heartbeat-sec" pflag:",The maximum available reservation extension heartbeat interval."` + StoragePrefix string `json:"storage-prefix" pflag:",StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly."` + MetricsScope string `json:"metrics-scope" pflag:",Scope that the metrics will record under."` + ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` + HeartbeatGracePeriodMultiplier int `json:"heartbeat-grace-period-multiplier" pflag:",Number of heartbeats before a reservation expires without an extension."` + MaxReservationHeartbeat config.Duration `json:"max-reservation-heartbeat" pflag:",The maximum available reservation extension heartbeat interval."` } From 2ee1f4156ee6fb7b0f01e679cb48e9a5a8589366 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 16 Nov 2021 07:31:09 -0600 Subject: [PATCH 55/58] added owner id to reservation gorm impl delete function Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 2 +- pkg/manager/impl/reservation_manager_test.go | 6 ++++++ pkg/repositories/gormimpl/reservation.go | 3 ++- pkg/repositories/gormimpl/reservation_test.go | 8 ++++---- pkg/repositories/interfaces/reservation_repo.go | 2 +- pkg/repositories/mocks/reservation.go | 14 +++++++------- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index 98908b8d..bae9b7cf 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -188,7 +188,7 @@ func (r *reservationManager) ReleaseReservation(ctx context.Context, request *da repo := r.repo.ReservationRepo() reservationKey := transformers.FromReservationID(request.ReservationId) - err := repo.Delete(ctx, reservationKey) + err := repo.Delete(ctx, reservationKey, request.OwnerId) if err != nil { logger.Errorf(ctx, "Failed to release reservation: %+v, err: %v", reservationKey, err) diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 689e95d1..693e3161 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -258,6 +258,9 @@ func TestReleaseReservation(t *testing.T) { reservationKey.DatasetVersion == datasetID.Version && reservationKey.TagName == tagName }), + mock.MatchedBy(func(ownerID string) bool { + return ownerID == currentOwner + }), ).Return(nil) reservationManager := NewReservationManager(&dcRepo, @@ -293,6 +296,9 @@ func TestReleaseReservation_Failure(t *testing.T) { reservationKey.DatasetVersion == datasetID.Version && reservationKey.TagName == tagName }), + mock.MatchedBy(func(ownerID string) bool { + return ownerID == currentOwner + }), ).Return(reservationErr) reservationManager := NewReservationManager(&dcRepo, diff --git a/pkg/repositories/gormimpl/reservation.go b/pkg/repositories/gormimpl/reservation.go index 23be4576..fffeba36 100644 --- a/pkg/repositories/gormimpl/reservation.go +++ b/pkg/repositories/gormimpl/reservation.go @@ -49,7 +49,7 @@ func (r *reservationRepo) Create(ctx context.Context, reservation models.Reserva return nil } -func (r *reservationRepo) Delete(ctx context.Context, reservationKey models.ReservationKey) error { +func (r *reservationRepo) Delete(ctx context.Context, reservationKey models.ReservationKey, ownerID string) error { timer := r.repoMetrics.DeleteDuration.Start(ctx) defer timer.Stop() @@ -57,6 +57,7 @@ func (r *reservationRepo) Delete(ctx context.Context, reservationKey models.Rese result := r.db.Where(&models.Reservation{ ReservationKey: reservationKey, + OwnerID: ownerID, }).Delete(&reservation) if result.Error != nil { return r.errorTransformer.ToDataCatalogError(result.Error) diff --git a/pkg/repositories/gormimpl/reservation_test.go b/pkg/repositories/gormimpl/reservation_test.go index cd7befc0..6368e90b 100644 --- a/pkg/repositories/gormimpl/reservation_test.go +++ b/pkg/repositories/gormimpl/reservation_test.go @@ -44,12 +44,12 @@ func TestDelete(t *testing.T) { expectedReservation := GetReservation() GlobalMock.NewMock().WithQuery( - `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5`, + `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5 AND "reservations"."owner_id" = $6`, ).WithRowsNum(1) reservationRepo := getReservationRepo(t) - err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey) + err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey, expectedReservation.OwnerID) assert.NoError(t, err) } @@ -59,12 +59,12 @@ func TestDeleteFailure(t *testing.T) { expectedReservation := GetReservation() GlobalMock.NewMock().WithQuery( - `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5`, + `DELETE FROM "reservations" WHERE "reservations"."dataset_project" = $1 AND "reservations"."dataset_name" = $2 AND "reservations"."dataset_domain" = $3 AND "reservations"."dataset_version" = $4 AND "reservations"."tag_name" = $5 AND "reservations"."owner_id" = $6`, ).WithRowsNum(0) reservationRepo := getReservationRepo(t) - err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey) + err := reservationRepo.Delete(context.Background(), expectedReservation.ReservationKey, expectedReservation.OwnerID) assert.Error(t, err) assert.Equal(t, "missing entity of type Reservation with identifier dataset_id: tag_name:\"testTag\" ", err.Error()) } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 0592f3af..950742b9 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -14,7 +14,7 @@ type ReservationRepo interface { Create(ctx context.Context, reservation models.Reservation, now time.Time) error // Delete a reservation if it exists - Delete(ctx context.Context, reservation models.ReservationKey) error + Delete(ctx context.Context, reservation models.ReservationKey, ownerID string) error // Get reservation Get(ctx context.Context, reservationKey models.ReservationKey) (models.Reservation, error) diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation.go index c8f0782c..4cabc8a2 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation.go @@ -57,8 +57,8 @@ func (_m ReservationRepo_Delete) Return(_a0 error) *ReservationRepo_Delete { return &ReservationRepo_Delete{Call: _m.Call.Return(_a0)} } -func (_m *ReservationRepo) OnDelete(ctx context.Context, reservation models.ReservationKey) *ReservationRepo_Delete { - c := _m.On("Delete", ctx, reservation) +func (_m *ReservationRepo) OnDelete(ctx context.Context, reservation models.ReservationKey, ownerID string) *ReservationRepo_Delete { + c := _m.On("Delete", ctx, reservation, ownerID) return &ReservationRepo_Delete{Call: c} } @@ -67,13 +67,13 @@ func (_m *ReservationRepo) OnDeleteMatch(matchers ...interface{}) *ReservationRe return &ReservationRepo_Delete{Call: c} } -// Delete provides a mock function with given fields: ctx, reservation -func (_m *ReservationRepo) Delete(ctx context.Context, reservation models.ReservationKey) error { - ret := _m.Called(ctx, reservation) +// Delete provides a mock function with given fields: ctx, reservation, ownerID +func (_m *ReservationRepo) Delete(ctx context.Context, reservation models.ReservationKey, ownerID string) error { + ret := _m.Called(ctx, reservation, ownerID) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey) error); ok { - r0 = rf(ctx, reservation) + if rf, ok := ret.Get(0).(func(context.Context, models.ReservationKey, string) error); ok { + r0 = rf(ctx, reservation, ownerID) } else { r0 = ret.Error(0) } From f52a8cf164601006f3d8133f936297f3f588d99c Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 16 Nov 2021 10:40:33 -0600 Subject: [PATCH 56/58] if reservation is missing on release reservation (meaning another entity acquired it) then gracefully fail Signed-off-by: Daniel Rammer --- pkg/manager/impl/reservation_manager.go | 13 ++++++- pkg/manager/impl/reservation_manager_test.go | 37 +++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pkg/manager/impl/reservation_manager.go b/pkg/manager/impl/reservation_manager.go index bae9b7cf..3adee114 100644 --- a/pkg/manager/impl/reservation_manager.go +++ b/pkg/manager/impl/reservation_manager.go @@ -26,6 +26,7 @@ type reservationMetrics struct { reservationAlreadyInProgress labeled.Counter acquireReservationFailure labeled.Counter releaseReservationFailure labeled.Counter + reservationDoesNotExist labeled.Counter } type NowFunc func() time.Time @@ -71,6 +72,11 @@ func NewReservationManager( "Number of times we failed to release a reservation", reservationScope, ), + reservationDoesNotExist: labeled.NewCounter( + "reservation_does_not_exist", + "Number of times we attempt to modify a reservation that does not exist", + reservationScope, + ), } return &reservationManager{ @@ -190,8 +196,13 @@ func (r *reservationManager) ReleaseReservation(ctx context.Context, request *da err := repo.Delete(ctx, reservationKey, request.OwnerId) if err != nil { - logger.Errorf(ctx, "Failed to release reservation: %+v, err: %v", reservationKey, err) + if errors.IsDoesNotExistError(err) { + logger.Warnf(ctx, "Reservation does not exist id: %+v, err %v", request.ReservationId, err) + r.systemMetrics.reservationDoesNotExist.Inc(ctx) + return &datacatalog.ReleaseReservationResponse{}, nil + } + logger.Errorf(ctx, "Failed to release reservation: %+v, err: %v", reservationKey, err) r.systemMetrics.releaseReservationFailure.Inc(ctx) return nil, err } diff --git a/pkg/manager/impl/reservation_manager_test.go b/pkg/manager/impl/reservation_manager_test.go index 693e3161..207ce813 100644 --- a/pkg/manager/impl/reservation_manager_test.go +++ b/pkg/manager/impl/reservation_manager_test.go @@ -2,6 +2,7 @@ package impl import ( "context" + "fmt" mockScope "github.com/flyteorg/flytestdlib/promutils" @@ -280,6 +281,40 @@ func TestReleaseReservation(t *testing.T) { func TestReleaseReservation_Failure(t *testing.T) { dcRepo := getDatacatalogRepo() + now := time.Now() + reservationErr := fmt.Errorf("unknown error") + + dcRepo.MockReservationRepo.On("Delete", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(reservationKey models.ReservationKey) bool { + return reservationKey.DatasetProject == datasetID.Project && + reservationKey.DatasetDomain == datasetID.Domain && + reservationKey.DatasetName == datasetID.Name && + reservationKey.DatasetVersion == datasetID.Version && + reservationKey.TagName == tagName + }), + mock.MatchedBy(func(ownerID string) bool { + return ownerID == currentOwner + }), + ).Return(reservationErr) + + reservationManager := NewReservationManager(&dcRepo, + heartbeatGracePeriodMultiplier, maxHeartbeatInterval, + func() time.Time { return now }, mockScope.NewTestScope()) + + req := datacatalog.ReleaseReservationRequest{ + ReservationId: &reservationID, + OwnerId: currentOwner, + } + + _, err := reservationManager.ReleaseReservation(context.Background(), &req) + + assert.Equal(t, reservationErr, err) +} + +func TestReleaseReservation_GracefulFailure(t *testing.T) { + dcRepo := getDatacatalogRepo() + now := time.Now() reservationErr := errors3.GetMissingEntityError("Reservation", &datacatalog.ReservationID{ @@ -312,7 +347,7 @@ func TestReleaseReservation_Failure(t *testing.T) { _, err := reservationManager.ReleaseReservation(context.Background(), &req) - assert.Equal(t, err, reservationErr) + assert.Nil(t, err) } func getDatacatalogRepo() mocks.DataCatalogRepo { From 394e92a2b6907d213cb03ef7d13050b6ae47e7b0 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 17 Nov 2021 19:04:29 -0600 Subject: [PATCH 57/58] updated flyteidl version Signed-off-by: Daniel Rammer --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c0912113..35ed7043 100644 --- a/go.mod +++ b/go.mod @@ -19,4 +19,4 @@ require ( gorm.io/gorm v1.21.9 ) -replace github.com/flyteorg/flyteidl => github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51 +replace github.com/flyteorg/flyteidl => github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab diff --git a/go.sum b/go.sum index b2779407..edc99700 100644 --- a/go.sum +++ b/go.sum @@ -311,6 +311,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51 h1:e9zvtfNKr+K84a7du4wJgC+MXYcEsA13yYGMsEGsjQs= github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= +github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab h1:1FC+p4ilWvLJZDm6DJeKWPzu+L+DRN0Y27osXgD5dzE= +github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= From df8dba4723473f38164cadd0b7985a7f8611b89e Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 24 Nov 2021 09:47:04 -0600 Subject: [PATCH 58/58] remove flyteidl replace in go.mod and updating to latest version Signed-off-by: Daniel Rammer --- go.mod | 4 +--- go.sum | 16 ++-------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 35ed7043..75b47c72 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/Selvatico/go-mocket v1.0.7 - github.com/flyteorg/flyteidl v0.18.38 + github.com/flyteorg/flyteidl v0.21.11 github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 @@ -18,5 +18,3 @@ require ( gorm.io/driver/postgres v1.1.0 gorm.io/gorm v1.21.9 ) - -replace github.com/flyteorg/flyteidl => github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab diff --git a/go.sum b/go.sum index edc99700..60e520a0 100644 --- a/go.sum +++ b/go.sum @@ -88,7 +88,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -135,7 +134,6 @@ github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEka github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -175,8 +173,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.18.38 h1:XgAw9d2Q/UjWQyXbnZz/j4N6OVGDxr7jceden6PdCgY= -github.com/flyteorg/flyteidl v0.18.38/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= +github.com/flyteorg/flyteidl v0.21.11 h1:oH9YPoR7scO9GFF/I8D0gCTOB+JP5HRK7b7cLUBRz90= +github.com/flyteorg/flyteidl v0.21.11/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flytestdlib v0.3.13 h1:5ioA/q3ixlyqkFh5kDaHgmPyTP/AHtqq1K/TIbVLUzM= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= @@ -307,12 +305,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xC github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51 h1:e9zvtfNKr+K84a7du4wJgC+MXYcEsA13yYGMsEGsjQs= -github.com/hamersaw/flyteidl v0.19.26-0.20211103115633-100abab11c51/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= -github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab h1:1FC+p4ilWvLJZDm6DJeKWPzu+L+DRN0Y27osXgD5dzE= -github.com/hamersaw/flyteidl v0.19.26-0.20211118002926-987e338423ab/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -539,7 +532,6 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -750,7 +742,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -979,7 +970,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1022,7 +1012,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1066,7 +1055,6 @@ gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=