diff --git a/cla-backend-go/cmd/migrate_ecla/main.go b/cla-backend-go/cmd/migrate_ecla/main.go new file mode 100644 index 000000000..de509c214 --- /dev/null +++ b/cla-backend-go/cmd/migrate_ecla/main.go @@ -0,0 +1,229 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + // "fmt" + "os" + "sync" + + // "sync" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/communitybridge/easycla/cla-backend-go/company" + + // "github.com/aws/aws-sdk-go/service/dynamodb/expression" + "github.com/communitybridge/easycla/cla-backend-go/events" + "github.com/communitybridge/easycla/cla-backend-go/gen/v1/models" + eventOps "github.com/communitybridge/easycla/cla-backend-go/gen/v1/restapi/operations/events" + log "github.com/communitybridge/easycla/cla-backend-go/logging" + "github.com/communitybridge/easycla/cla-backend-go/project/repository" + "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" + "github.com/communitybridge/easycla/cla-backend-go/repositories" + "github.com/communitybridge/easycla/cla-backend-go/signatures" + "github.com/communitybridge/easycla/cla-backend-go/users" + "github.com/sirupsen/logrus" +) + +var stage string +var eventsRepo events.Repository +var eventsService events.Service +var signatureRepo signatures.SignatureRepository +var v1ProjectRepo repository.ProjectRepository +var usersRepo users.UserRepository +var companyRepo company.IRepository +var ghRepo repositories.Repository +var projectClaGroup projects_cla_groups.Repository +var awsSession = session.Must(session.NewSession(&aws.Config{})) + +type combinedRepo struct { + users.UserRepository + company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("STAGE environment variable not set") + } + + log.Infof("STAGE: %s", stage) + companyRepo = company.NewRepository(awsSession, stage) + usersRepo = users.NewRepository(awsSession, stage) + ghRepo = *repositories.NewRepository(awsSession, stage) + projectClaGroup = projects_cla_groups.NewRepository(awsSession, stage) + v1ProjectRepo = repository.NewRepository(awsSession, stage, &ghRepo, nil, projectClaGroup) + eventsRepo = events.NewRepository(awsSession, stage) + eventsService = events.NewService(eventsRepo, combinedRepo{ + usersRepo, + companyRepo, + v1ProjectRepo, + projectClaGroup, + }) + signatureRepo = signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, &ghRepo, nil, nil, nil) + + log.Infof("initializing migrate_ecla") +} + +func main() { + f := logrus.Fields{ + "functionName": "main", + "stage": stage, + } + log.WithFields(f).Info("migrate_ecla started") + + ctx := context.Background() + // Fetch all the companies + companies, err := companyRepo.GetCompanies(ctx) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem fetching companies") + return + } + + invalidSignatures := make([]*signatures.ItemSignature, 0) + + log.WithFields(f).Infof("processing %d companies", len(companies.Companies)) + // Fetch the ecla records + + for _, company := range companies.Companies { + // log.WithFields(f).Infof("processing company: %s", company.CompanyName) + // Fetch the ecla records for the company + invalidCompanyEcla := make([]*signatures.ItemSignature, 0) + eclaRecords, err := signatureRepo.GetCompanyECLASignatures(ctx, company.CompanyID) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem fetching ecla records") + return + } + // log.WithFields(f).Infof("processing %d records for company: %s", len(eclaRecords), company.CompanyID) + + if len(eclaRecords) == 0 { + log.WithFields(f).Infof("no records for company: %s", company.CompanyID) + continue + } + + var wg sync.WaitGroup + var mu sync.Mutex + for _, eclaRecord := range eclaRecords { + wg.Add(1) + go func(eclaRecord *signatures.ItemSignature) { + defer wg.Done() + // log.WithFields(f).Infof("processing signature record: %s", eclaRecord.SignatureID) + if eclaRecord.DateCreated == "" { + log.WithFields(f).Warnf("invalid signature record: %s", eclaRecord.SignatureID) + mu.Lock() + invalidCompanyEcla = append(invalidCompanyEcla, eclaRecord) + mu.Unlock() + return + } + }(eclaRecord) + } + wg.Wait() + + invalidSignatures = append(invalidSignatures, invalidCompanyEcla...) + + log.WithFields(f).Debugf("Found %d invalid eclas for company: %s", len(invalidCompanyEcla), company.CompanyID) + } + + log.WithFields(f).Infof("Processesing %d invalid records ", len(invalidSignatures)) + // For each ecla record, check the events table for the corresponding event and update the signature table + update := 0 + + for _, eclaRecord := range invalidSignatures { + events, err := getEvents(ctx, *eclaRecord) + if err != nil { + log.WithFields(f).WithError(err).Warnf("problem fetching events for ecla record: %s", eclaRecord.SignatureID) + return + } + + if len(events) == 0 { + log.WithFields(f).Warnf("no events found for ecla record: %s and user_id: %s", eclaRecord.SignatureID, eclaRecord.SignatureReferenceID) + continue + } + + log.WithFields(f).Infof("found %d events for ecla record: %s", len(events), eclaRecord.SignatureID) + + latestEvent := getLatestEvent(events) + if latestEvent == nil { + log.WithFields(f).Warnf("no latest event found for ecla record: %s", eclaRecord.SignatureID) + continue + } + + log.Debugf("Found event:%s for user %s", latestEvent.EventID, eclaRecord.SignatureReferenceID) + + // Update the signature record + update++ + + update := map[string]interface{}{ + "date_created": latestEvent.EventTime, + "date_modified": latestEvent.EventTime, + } + + log.WithFields(f).Infof("updating signature record: %s with : %+v", eclaRecord.SignatureID, update) + + err = signatureRepo.UpdateSignature(ctx, eclaRecord.SignatureID, update) + if err != nil { + log.WithFields(f).WithError(err).Warnf("problem updating signature record: %s", eclaRecord.SignatureID) + return + } + + log.WithFields(f).Infof("updated signature record: %s", eclaRecord.SignatureID) + } + + log.WithFields(f).Infof("updated :%d records", update) +} + +func getEvents(ctx context.Context, ecla signatures.ItemSignature) ([]*models.Event, error) { + + f := logrus.Fields{ + "functionName": "getEvents", + "companyID": ecla.SignatureUserCompanyID, + "projectID": ecla.SignatureProjectID, + "userID": ecla.SignatureReferenceID, + } + + log.WithFields(f).Debug("searching events") + + eventType := "EmployeeSignatureCreated" + + claGroupModel, err := v1ProjectRepo.GetCLAGroupByID(ctx, ecla.SignatureProjectID, false) + if err != nil { + log.WithFields(f).Debugf("Unable to fetch cla group model") + return nil, err + } + + eventParams := eventOps.SearchEventsParams{ + UserID: &ecla.SignatureReferenceID, + ProjectSFID: &claGroupModel.ProjectExternalID, + EventType: &eventType, + CompanyID: &ecla.SignatureUserCompanyID, + } + + events, err := eventsService.SearchEvents(&eventParams) + if err != nil { + log.WithFields(f).WithError(err).Infof("Unable to get clagroup by id: %s", claGroupModel.ProjectID) + return nil, err + } + + return events.Events, nil + +} + +func getLatestEvent(events []*models.Event) *models.Event { + // Fetch the latest event from the list of events + var latestEvent *models.Event + for _, event := range events { + if latestEvent == nil { + latestEvent = event + continue + } + if event.EventTime > latestEvent.EventTime { + latestEvent = event + } + } + return latestEvent +} diff --git a/cla-backend-go/signatures/mocks/mock_repo.go b/cla-backend-go/signatures/mocks/mock_repo.go index 8ebe7653d..cffd9e02e 100644 --- a/cla-backend-go/signatures/mocks/mock_repo.go +++ b/cla-backend-go/signatures/mocks/mock_repo.go @@ -260,6 +260,21 @@ func (mr *MockSignatureRepositoryMockRecorder) GetClaGroupICLASignatures(ctx, cl return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClaGroupICLASignatures", reflect.TypeOf((*MockSignatureRepository)(nil).GetClaGroupICLASignatures), ctx, claGroupID, searchTerm, approved, signed, pageSize, nextKey, withExtraDetails) } +// GetCompanyECLASignatures mocks base method. +func (m *MockSignatureRepository) GetCompanyECLASignatures(ctx context.Context, companyID string) ([]*signatures0.ItemSignature, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCompanyECLASignatures", ctx, companyID) + ret0, _ := ret[0].([]*signatures0.ItemSignature) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCompanyECLASignatures indicates an expected call of GetCompanyECLASignatures. +func (mr *MockSignatureRepositoryMockRecorder) GetCompanyECLASignatures(ctx, companyID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompanyECLASignatures", reflect.TypeOf((*MockSignatureRepository)(nil).GetCompanyECLASignatures), ctx, companyID) +} + // GetCompanyIDsWithSignedCorporateSignatures mocks base method. func (m *MockSignatureRepository) GetCompanyIDsWithSignedCorporateSignatures(ctx context.Context, claGroupID string) ([]signatures0.SignatureCompanyID, error) { m.ctrl.T.Helper() diff --git a/cla-backend-go/signatures/repository.go b/cla-backend-go/signatures/repository.go index 48db47aa4..ed0755907 100644 --- a/cla-backend-go/signatures/repository.go +++ b/cla-backend-go/signatures/repository.go @@ -106,6 +106,7 @@ type SignatureRepository interface { EclaAutoCreate(ctx context.Context, signatureID string, autoCreateECLA bool) error ActivateSignature(ctx context.Context, signatureID string) error GetICLAByDate(ctx context.Context, startDate string) ([]ItemSignature, error) + GetCompanyECLASignatures(ctx context.Context, companyID string) ([]*ItemSignature, error) } type iclaSignatureWithDetails struct { @@ -173,6 +174,50 @@ func (repo repository) CreateSignature(ctx context.Context, signature *ItemSigna } +func (repo repository) GetCompanyECLASignatures(ctx context.Context, companyID string) ([]*ItemSignature, error) { + f := logrus.Fields{ + "functionName": "v1.signatures.repository.GetCompanyECLASignatures", + "companyID": companyID, + } + log.WithFields(f).Debugf("fetching company ecla records") + var itemSignatures []*ItemSignature + + condition := expression.Key("signature_user_ccla_company_id").Equal(expression.Value(companyID)) + filter := expression.Name("signature_signed").Equal(expression.Value(true)).And(expression.Name("signature_approved").Equal(expression.Value(true))) + + expr, err := expression.NewBuilder().WithKeyCondition(condition).WithFilter(filter).Build() + if err != nil { + return nil, err + } + + // Assemble the query input parameters + queryInput := &dynamodb.QueryInput{ + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + KeyConditionExpression: expr.KeyCondition(), + FilterExpression: expr.Filter(), + TableName: aws.String(repo.signatureTableName), + IndexName: aws.String("signature-user-ccla-company-index"), + } + + results, err := repo.dynamoDBClient.Query(queryInput) + if err != nil { + return nil, err + } + + // No match, didn't find it + if *results.Count == 0 { + return nil, nil + } + + err = dynamodbattribute.UnmarshalListOfMaps(results.Items, &itemSignatures) + if err != nil { + return nil, err + } + + return itemSignatures, nil +} + // GetItemSignature returns the signature for the specified signature id func (repo repository) GetItemSignature(ctx context.Context, signatureID string) (*ItemSignature, error) { f := logrus.Fields{