From 0c04e926842f1a9b06df579cbbc020595a1b1e2d Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 13:57:14 +0200 Subject: [PATCH 01/11] stable version --- cmd/import.go | 263 ++++++++++++++++++++- cmd/provider_cmd_aws.go | 10 +- main_test.go | 39 +++ terraformutils/providerwrapper/provider.go | 19 +- terraformutils/utils.go | 75 +++++- 5 files changed, 391 insertions(+), 15 deletions(-) create mode 100644 main_test.go diff --git a/cmd/import.go b/cmd/import.go index 8e8ed0034e..0a0581e719 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -17,9 +17,12 @@ import ( "fmt" "io/ioutil" "log" + "math/rand" "os" "sort" "strings" + "sync" + "time" "github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring" @@ -51,6 +54,8 @@ type ImportOptions struct { Filter []string Plan bool `json:"-"` Output string + RetryCount int + RetrySleepMs int } const DefaultPathPattern = "{output}/{provider}/{service}/" @@ -77,6 +82,256 @@ func newImportCmd() *cobra.Command { return cmd } +func initOptionsAndWrapper(provider terraformutils.ProviderGenerator, options ImportOptions, args []string) (*providerwrapper.ProviderWrapper, ImportOptions, error) { + err := provider.Init(args) + if err != nil { + return nil, options, err + } + + if terraformerstring.ContainsString(options.Resources, "*") { + log.Println("Attempting an import of ALL resources in " + provider.GetName()) + options.Resources = providerServices(provider) + } + + if options.Excludes != nil { + localSlice := []string{} + for _, r := range options.Resources { + remove := false + for _, e := range options.Excludes { + if r == e { + remove = true + log.Println("Excluding resource " + e) + } + } + if !remove { + localSlice = append(localSlice, r) + } + } + options.Resources = localSlice + } + + providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose, options.RetryCount, options.RetrySleepMs) + if err != nil { + return nil, options, err + } + + return providerWrapper, options, nil +} + +func initAllServiceResources(providers []terraformutils.ProviderGenerator, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper) (map[terraformutils.ProviderGenerator]string, error) { + var wg sync.WaitGroup + numOfResources := len(options.Resources) + wg.Add(numOfResources) + failedServicesChan := make(chan string, numOfResources) + serviceByProvider := map[terraformutils.ProviderGenerator]string{} + filteredServiceByProvider := map[terraformutils.ProviderGenerator]string{} + for i, service := range options.Resources { + serviceProvider := providers[i] + serviceByProvider[serviceProvider] = service + err := serviceProvider.Init(args) + if err != nil { + return nil, err + } + go initServiceResources(service, serviceProvider, options, providerWrapper, &wg, failedServicesChan) + } + wg.Wait() + close(failedServicesChan) + var failedServices []string + for failedService := range failedServicesChan { + failedServices = append(failedServices, failedService) + } + + for provider := range serviceByProvider { + service := serviceByProvider[provider] + isFailed := false + for _, failedService := range failedServices { + if failedService == service { + isFailed = true + break + } + } + if !isFailed { + filteredServiceByProvider[provider] = service + } + } + + return filteredServiceByProvider, nil +} + +func shuffleResources(providers []terraformutils.ProviderGenerator, serviceByProvider map[terraformutils.ProviderGenerator]string) []map[*terraformutils.Resource]terraformutils.ProviderGenerator { + var allResources []map[*terraformutils.Resource]terraformutils.ProviderGenerator + for i := range providers { + provider := providers[i] + log.Printf("num of resources for service %s: %d", serviceByProvider[provider], len(provider.GetService().GetResources())) + providerResources := provider.GetService().GetResources() + for i := range providerResources { + resource := providerResources[i] + allResources = append(allResources, map[*terraformutils.Resource]terraformutils.ProviderGenerator{&resource: provider}) + } + } + + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(allResources), func(i, j int) { allResources[i], allResources[j] = allResources[j], allResources[i] }) + + return allResources +} + +func ImportRoundRobin(providers []terraformutils.ProviderGenerator, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper) error { + defer providerWrapper.Kill() + + serviceByProvider, err := initAllServiceResources(providers, options, args, providerWrapper) + if err != nil { + return err + } + + shuffledResources := shuffleResources(providers, serviceByProvider) + + refreshedResources, err := terraformutils.RefreshResourcesByProvider(shuffledResources, providerWrapper) + providerToResources := make(map[terraformutils.ProviderGenerator][]terraformutils.Resource) + + for resource := range refreshedResources { + p := refreshedResources[resource] + if providerToResources[p] == nil { + providerToResources[p] = []terraformutils.Resource{} + } + providerToResources[p] = append(providerToResources[p], *resource) + } + + err = importFromPlan2(providerToResources, options, args, providerWrapper, serviceByProvider, providers[0]) + + return err +} + +func importFromPlan2(providerToResources map[terraformutils.ProviderGenerator][]terraformutils.Resource, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper, serviceByProvider map[terraformutils.ProviderGenerator]string, provider terraformutils.ProviderGenerator) error { + plan := &ImportPlan{ + Provider: provider.GetName(), + Options: options, + Args: args, + ImportedResource: map[string][]terraformutils.Resource{}, + } + for p := range providerToResources { + service := serviceByProvider[p] + plan.ImportedResource[service] = append(plan.ImportedResource[service], providerToResources[p]...) + } + + if options.Plan { + path := Path(options.PathPattern, provider.GetName(), "terraformer", options.PathOutput) + return ExportPlanFile(plan, path, "plan.json") + } + + return ImportFromPlan(provider, plan) +} + +func importFromPlan(providerToResources map[terraformutils.ProviderGenerator][]terraformutils.Resource, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper, serviceByProvider map[terraformutils.ProviderGenerator]string) error { + var wg sync.WaitGroup + numOfProviders := len(providerToResources) + wg.Add(numOfProviders) + errors := make(chan error, numOfProviders) + for p := range providerToResources { + resources, err := refreshServiceResources(p, providerWrapper, providerToResources[p]) + if err != nil { + return err + } + go importFromPlanWorker(p, options, args, resources, serviceByProvider[p], &wg, errors) + } + + wg.Wait() + close(errors) + err, done := <-errors + if done { + return err + } + + return nil +} + +func importFromPlanWorker(provider terraformutils.ProviderGenerator, options ImportOptions, args []string, resources []terraformutils.Resource, service string, wg *sync.WaitGroup, errors chan error) { + plan := &ImportPlan{ + Provider: provider.GetName(), + Options: options, + Args: args, + ImportedResource: map[string][]terraformutils.Resource{}, + } + + plan.ImportedResource[service] = append(plan.ImportedResource[service], resources...) + + if options.Plan { + path := Path(options.PathPattern, provider.GetName(), "terraformer", options.PathOutput) + err := ExportPlanFile(plan, path, "plan.json") + wg.Done() + if err != nil { + errors <- err + } + } + err := ImportFromPlan(provider, plan) + wg.Done() + if err != nil { + errors <- err + } +} + +func initServiceResources(service string, provider terraformutils.ProviderGenerator, + options ImportOptions, providerWrapper *providerwrapper.ProviderWrapper, wg *sync.WaitGroup, failedServices chan string) { + log.Println(provider.GetName() + " importing... " + service) + err := provider.InitService(service, options.Verbose) + if err != nil { + failedServices <- service + log.Printf("%s error importing %s, err: %s\n", provider.GetName(), service, err) + wg.Done() + return + } + provider.GetService().ParseFilters(options.Filter) + err = provider.GetService().InitResources() + if err != nil { + failedServices <- service + log.Printf("%s error initializing resources in service %s, err: %s\n", provider.GetName(), service, err) + wg.Done() + return + } + + provider.GetService().PopulateIgnoreKeys(providerWrapper) + provider.GetService().InitialCleanup() + log.Println(provider.GetName() + " done importing " + service) + wg.Done() +} + +func refreshServiceResources(provider terraformutils.ProviderGenerator, providerWrapper *providerwrapper.ProviderWrapper, refreshedResources []terraformutils.Resource) ([]terraformutils.Resource, error) { + provider.GetService().SetResources(refreshedResources) + + for i := range provider.GetService().GetResources() { + err := provider.GetService().GetResources()[i].ConvertTFstate(providerWrapper) + if err != nil { + return nil, err + } + } + provider.GetService().PostRefreshCleanup() + + // change structs with additional data for each resource + err := provider.GetService().PostConvertHook() + if err != nil { + return nil, err + } + return provider.GetService().GetResources(), nil +} + +func getResourcesAddresses(resources []terraformutils.Resource) []*terraformutils.Resource { + results := []*terraformutils.Resource{} + for i := range resources { + results = append(results, &resources[i]) + } + + return results +} + +func getResourcesValues(resources []*terraformutils.Resource) []terraformutils.Resource { + results := []terraformutils.Resource{} + for i := range resources { + results = append(results, *resources[i]) + } + + return results +} + func Import(provider terraformutils.ProviderGenerator, options ImportOptions, args []string) error { err := provider.Init(args) if err != nil { @@ -112,7 +367,7 @@ func Import(provider terraformutils.ProviderGenerator, options ImportOptions, ar options.Resources = localSlice } - providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose) + providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose, options.RetryCount, options.RetrySleepMs) if err != nil { return err } @@ -150,11 +405,11 @@ func buildServiceResources(service string, provider terraformutils.ProviderGener provider.GetService().PopulateIgnoreKeys(providerWrapper) provider.GetService().InitialCleanup() - refreshedResources, err := terraformutils.RefreshResources(provider.GetService().GetResources(), providerWrapper) + refreshedResources, err := terraformutils.RefreshResources(getResourcesAddresses(provider.GetService().GetResources()), providerWrapper, nil) if err != nil { return nil, err } - provider.GetService().SetResources(refreshedResources) + provider.GetService().SetResources(getResourcesValues(refreshedResources)) for i := range provider.GetService().GetResources() { err = provider.GetService().GetResources()[i].ConvertTFstate(providerWrapper) @@ -358,4 +613,6 @@ func baseProviderFlags(flag *pflag.FlagSet, options *ImportOptions, sampleRes, s flag.StringSliceVarP(&options.Filter, "filter", "f", []string{}, sampleFilters) flag.BoolVarP(&options.Verbose, "verbose", "v", false, "") flag.StringVarP(&options.Output, "output", "O", "hcl", "output format hcl or json") + flag.IntVarP(&options.RetryCount, "retry-number", "n", 5, "number of retries to perform when refresh fails") + flag.IntVarP(&options.RetrySleepMs, "retry-sleep-ms", "m", 300, "time in ms to sleep between retries") } diff --git a/cmd/provider_cmd_aws.go b/cmd/provider_cmd_aws.go index 7a00c74315..fa168cc61d 100644 --- a/cmd/provider_cmd_aws.go +++ b/cmd/provider_cmd_aws.go @@ -109,7 +109,15 @@ func importRegionResources(options ImportOptions, originalPathPattern string, re } else { log.Println(provider.GetName() + " importing default region") } - err := Import(provider, options, []string{region, options.Profile}) + args := []string{region, options.Profile} + + providerWrapper, options, err := initOptionsAndWrapper(provider, options, args) + providersPerResources := make([]terraformutils.ProviderGenerator, len(options.Resources)) + for i := range options.Resources { + providersPerResources[i] = newAWSProvider() + } + err = ImportRoundRobin(providersPerResources, options, args, providerWrapper) + //err := Import(provider, options, []string{region, options.Profile}) if err != nil { return err } diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000000..886a62e1b4 --- /dev/null +++ b/main_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "github.com/GoogleCloudPlatform/terraformer/cmd" + "log" + "os" + "testing" + "time" +) + +func TestTerraformerMain(t1 *testing.T) { + t1.Run("run main", func(t1 *testing.T) { + tCommand := cmd.NewCmdRoot() + tCommand.SetArgs([]string{ + "import", + "aws", + "--regions=sa-east-1", + //"--regions=us-west-1", + //"--regions=us-east-1", + //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", + "--resources=\"*\"", + //"--resources=sg", + //"--resources=cloudformation,sg,s3", + //"--resources=cloudformation,sg", + //"--resources=sg,vpc", + //"--resources=kms", + "--profile=nubank", + //"--retry-number=5", + //"--retry-sleep-ms=300", + }) + start := time.Now() + if err := tCommand.Execute(); err != nil { + log.Println(err) + os.Exit(1) + } + log.Printf("Importing took %s", time.Since(start)) + + }) +} diff --git a/terraformutils/providerwrapper/provider.go b/terraformutils/providerwrapper/provider.go index 2165e895fd..30fb5efb1f 100644 --- a/terraformutils/providerwrapper/provider.go +++ b/terraformutils/providerwrapper/provider.go @@ -56,12 +56,23 @@ type ProviderWrapper struct { providerName string config cty.Value schema *providers.GetSchemaResponse + retryCount int + retrySleepMs int } -func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool) (*ProviderWrapper, error) { +func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool, retryOptions ...int) (*ProviderWrapper, error) { p := &ProviderWrapper{} p.providerName = providerName p.config = providerConfig + + if len(retryOptions) == 2 { + p.retryCount = retryOptions[0] + p.retrySleepMs = retryOptions[1] + } else { + p.retryCount = 5 + p.retrySleepMs = 300 + } + err := p.initProvider(verbose) return p, err @@ -149,15 +160,15 @@ func (p *ProviderWrapper) Refresh(info *terraform.InstanceInfo, state *terraform } successReadResource := false resp := providers.ReadResourceResponse{} - for i := 0; i < 5; i++ { + for i := 0; i < p.retryCount; i++ { resp = p.Provider.ReadResource(providers.ReadResourceRequest{ TypeName: info.Type, PriorState: priorState, Private: []byte{}, }) if resp.Diagnostics.HasErrors() { - log.Println("WARN: Fail read resource from provider, wait 300ms before retry") - time.Sleep(300 * time.Millisecond) + log.Printf("WARN: Fail read resource from provider, wait %dms before retry\n", p.retrySleepMs) + time.Sleep(time.Duration(p.retrySleepMs) * time.Millisecond) continue } else { successReadResource = true diff --git a/terraformutils/utils.go b/terraformutils/utils.go index 0ac39c72fa..317a408f3c 100644 --- a/terraformutils/utils.go +++ b/terraformutils/utils.go @@ -65,21 +65,35 @@ func PrintTfState(resources []Resource) ([]byte, error) { return buf.Bytes(), err } -func RefreshResources(resources []Resource, provider *providerwrapper.ProviderWrapper) ([]Resource, error) { - refreshedResources := []Resource{} +func RefreshResources(resources []*Resource, provider *providerwrapper.ProviderWrapper, slowProcessingResources [][]*Resource) ([]*Resource, error) { + refreshedResources := []*Resource{} input := make(chan *Resource, 100) var wg sync.WaitGroup poolSize := 15 - if slowProcessingRequired(resources) { - poolSize = 1 - } + //if slowProcessingRequired(resources) { + // poolSize = 1 + //} for i := 0; i < poolSize; i++ { go RefreshResourceWorker(input, &wg, provider) } for i := range resources { wg.Add(1) - input <- &resources[i] + input <- resources[i] + } + + spInputs := []chan *Resource{} + for i, resourceGroup := range slowProcessingResources { + spInputs = append(spInputs, make(chan *Resource, 100)) + for j := range resourceGroup { + spInputs[i] <- resourceGroup[j] + } + } + + for i := 0; i < len(spInputs); i++ { + go RefreshResourceWorker(spInputs[i], &wg, provider) + wg.Add(len(slowProcessingResources[i])) } + wg.Wait() close(input) for _, r := range resources { @@ -89,10 +103,57 @@ func RefreshResources(resources []Resource, provider *providerwrapper.ProviderWr log.Printf("ERROR: Unable to refresh resource %s", r.ResourceName) } } + + for _, resourceGroup := range slowProcessingResources { + for i := range resourceGroup { + r := resourceGroup[i] + if r.InstanceState != nil && r.InstanceState.ID != "" { + refreshedResources = append(refreshedResources, r) + } else { + log.Printf("ERROR: Unable to refresh resource %s", r.ResourceName) + } + } + } return refreshedResources, nil } -func slowProcessingRequired(resources []Resource) bool { +func RefreshResourcesByProvider(resourcesByProvider []map[*Resource]ProviderGenerator, providerWrapper *providerwrapper.ProviderWrapper) (map[*Resource]ProviderGenerator, error) { + flattenedResourcesByProvider := make(map[*Resource]ProviderGenerator) + refreshedResourcesByProvider := make(map[*Resource]ProviderGenerator) + slowProcessingResources := make(map[ProviderGenerator][]*Resource) + var resources []*Resource + for _, resourceProviderPair := range resourcesByProvider { + for resourcePtr := range resourceProviderPair { + if resourcePtr.SlowQueryRequired { + provider := resourceProviderPair[resourcePtr] + if slowProcessingResources[provider] == nil { + slowProcessingResources[provider] = []*Resource{} + } + slowProcessingResources[provider] = append(slowProcessingResources[provider], resourcePtr) + flattenedResourcesByProvider[resourcePtr] = resourceProviderPair[resourcePtr] + } else { + flattenedResourcesByProvider[resourcePtr] = resourceProviderPair[resourcePtr] + resources = append(resources, resourcePtr) + } + } + } + + var spResourcesList [][]*Resource + for p := range slowProcessingResources { + spResourcesList = append(spResourcesList, slowProcessingResources[p]) + } + + refreshedResources, err := RefreshResources(resources, providerWrapper, spResourcesList) + if err != nil { + return nil, err + } + for _, r := range refreshedResources { + refreshedResourcesByProvider[r] = flattenedResourcesByProvider[r] + } + return refreshedResourcesByProvider, nil +} + +func slowProcessingRequired(resources []*Resource) bool { for _, r := range resources { if r.SlowQueryRequired { return true From d71987afe9a867c5a4330b1bbbb5729e678d42fd Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:03:11 +0200 Subject: [PATCH 02/11] code cleanup + use provider_mapping --- cmd/import.go | 301 +++++----------------------- cmd/provider_cmd_aws.go | 10 +- main_test.go | 30 +++ terraformutils/providers_mapping.go | 151 ++++++++++++++ terraformutils/utils.go | 53 ++--- 5 files changed, 251 insertions(+), 294 deletions(-) create mode 100644 terraformutils/providers_mapping.go diff --git a/cmd/import.go b/cmd/import.go index 0a0581e719..935d6afe2c 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -15,16 +15,13 @@ package cmd import ( "fmt" + "github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring" "io/ioutil" "log" - "math/rand" "os" "sort" "strings" "sync" - "time" - - "github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring" "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" @@ -82,6 +79,34 @@ func newImportCmd() *cobra.Command { return cmd } +func Import(provider terraformutils.ProviderGenerator, options ImportOptions, args []string) error { + + providerWrapper, options, err := initOptionsAndWrapper(provider, options, args) + if err != nil { + return err + } + defer providerWrapper.Kill() + providerMapping := terraformutils.NewProvidersMapping(provider) + + err = initAllServicesResources(providerMapping, options, args, providerWrapper) + if err != nil { + return err + } + + err = terraformutils.RefreshResourcesByProvider(providerMapping, providerWrapper) + if err != nil { + return err + } + + providerMapping.ConvertTFStates(providerWrapper) + // change structs with additional data for each resource + providerMapping.CleanupProviders() + + err = importFromPlan(providerMapping, options, args) + + return err +} + func initOptionsAndWrapper(provider terraformutils.ProviderGenerator, options ImportOptions, args []string) (*providerwrapper.ProviderWrapper, ImportOptions, error) { err := provider.Init(args) if err != nil { @@ -118,159 +143,58 @@ func initOptionsAndWrapper(provider terraformutils.ProviderGenerator, options Im return providerWrapper, options, nil } -func initAllServiceResources(providers []terraformutils.ProviderGenerator, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper) (map[terraformutils.ProviderGenerator]string, error) { - var wg sync.WaitGroup +func initAllServicesResources(providersMapping *terraformutils.ProvidersMapping, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper) error { numOfResources := len(options.Resources) + var wg sync.WaitGroup wg.Add(numOfResources) + failedServicesChan := make(chan string, numOfResources) - serviceByProvider := map[terraformutils.ProviderGenerator]string{} - filteredServiceByProvider := map[terraformutils.ProviderGenerator]string{} - for i, service := range options.Resources { - serviceProvider := providers[i] - serviceByProvider[serviceProvider] = service + + for _, service := range options.Resources { + serviceProvider := providersMapping.AddServiceToProvider(service) err := serviceProvider.Init(args) if err != nil { - return nil, err + return err } - go initServiceResources(service, serviceProvider, options, providerWrapper, &wg, failedServicesChan) + go initServiceResourcesWorker(service, serviceProvider, options, providerWrapper, &wg, failedServicesChan) } wg.Wait() close(failedServicesChan) + + // remove providers that failed to init their service var failedServices []string for failedService := range failedServicesChan { failedServices = append(failedServices, failedService) } - for provider := range serviceByProvider { - service := serviceByProvider[provider] - isFailed := false - for _, failedService := range failedServices { - if failedService == service { - isFailed = true - break - } - } - if !isFailed { - filteredServiceByProvider[provider] = service - } - } - - return filteredServiceByProvider, nil -} - -func shuffleResources(providers []terraformutils.ProviderGenerator, serviceByProvider map[terraformutils.ProviderGenerator]string) []map[*terraformutils.Resource]terraformutils.ProviderGenerator { - var allResources []map[*terraformutils.Resource]terraformutils.ProviderGenerator - for i := range providers { - provider := providers[i] - log.Printf("num of resources for service %s: %d", serviceByProvider[provider], len(provider.GetService().GetResources())) - providerResources := provider.GetService().GetResources() - for i := range providerResources { - resource := providerResources[i] - allResources = append(allResources, map[*terraformutils.Resource]terraformutils.ProviderGenerator{&resource: provider}) - } - } - - rand.Seed(time.Now().UnixNano()) - rand.Shuffle(len(allResources), func(i, j int) { allResources[i], allResources[j] = allResources[j], allResources[i] }) - - return allResources -} - -func ImportRoundRobin(providers []terraformutils.ProviderGenerator, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper) error { - defer providerWrapper.Kill() + providersMapping.RemoveServices(failedServices) + providersMapping.ProcessResources() - serviceByProvider, err := initAllServiceResources(providers, options, args, providerWrapper) - if err != nil { - return err - } - - shuffledResources := shuffleResources(providers, serviceByProvider) - - refreshedResources, err := terraformutils.RefreshResourcesByProvider(shuffledResources, providerWrapper) - providerToResources := make(map[terraformutils.ProviderGenerator][]terraformutils.Resource) - - for resource := range refreshedResources { - p := refreshedResources[resource] - if providerToResources[p] == nil { - providerToResources[p] = []terraformutils.Resource{} - } - providerToResources[p] = append(providerToResources[p], *resource) - } - - err = importFromPlan2(providerToResources, options, args, providerWrapper, serviceByProvider, providers[0]) - - return err + return nil } -func importFromPlan2(providerToResources map[terraformutils.ProviderGenerator][]terraformutils.Resource, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper, serviceByProvider map[terraformutils.ProviderGenerator]string, provider terraformutils.ProviderGenerator) error { +func importFromPlan(providerMapping *terraformutils.ProvidersMapping, options ImportOptions, args []string) error { plan := &ImportPlan{ - Provider: provider.GetName(), + Provider: providerMapping.GetBaseProvider().GetName(), Options: options, Args: args, ImportedResource: map[string][]terraformutils.Resource{}, } - for p := range providerToResources { - service := serviceByProvider[p] - plan.ImportedResource[service] = append(plan.ImportedResource[service], providerToResources[p]...) + + resourcesByService := providerMapping.GetResourcesByService() + for service := range resourcesByService { + plan.ImportedResource[service] = append(plan.ImportedResource[service], resourcesByService[service]...) } if options.Plan { - path := Path(options.PathPattern, provider.GetName(), "terraformer", options.PathOutput) + path := Path(options.PathPattern, providerMapping.GetBaseProvider().GetName(), "terraformer", options.PathOutput) return ExportPlanFile(plan, path, "plan.json") } - return ImportFromPlan(provider, plan) -} - -func importFromPlan(providerToResources map[terraformutils.ProviderGenerator][]terraformutils.Resource, options ImportOptions, args []string, providerWrapper *providerwrapper.ProviderWrapper, serviceByProvider map[terraformutils.ProviderGenerator]string) error { - var wg sync.WaitGroup - numOfProviders := len(providerToResources) - wg.Add(numOfProviders) - errors := make(chan error, numOfProviders) - for p := range providerToResources { - resources, err := refreshServiceResources(p, providerWrapper, providerToResources[p]) - if err != nil { - return err - } - go importFromPlanWorker(p, options, args, resources, serviceByProvider[p], &wg, errors) - } - - wg.Wait() - close(errors) - err, done := <-errors - if done { - return err - } - - return nil -} - -func importFromPlanWorker(provider terraformutils.ProviderGenerator, options ImportOptions, args []string, resources []terraformutils.Resource, service string, wg *sync.WaitGroup, errors chan error) { - plan := &ImportPlan{ - Provider: provider.GetName(), - Options: options, - Args: args, - ImportedResource: map[string][]terraformutils.Resource{}, - } - - plan.ImportedResource[service] = append(plan.ImportedResource[service], resources...) - - if options.Plan { - path := Path(options.PathPattern, provider.GetName(), "terraformer", options.PathOutput) - err := ExportPlanFile(plan, path, "plan.json") - wg.Done() - if err != nil { - errors <- err - } - } - err := ImportFromPlan(provider, plan) - wg.Done() - if err != nil { - errors <- err - } + return ImportFromPlan(providerMapping.GetBaseProvider(), plan) } -func initServiceResources(service string, provider terraformutils.ProviderGenerator, +func initServiceResourcesWorker(service string, provider terraformutils.ProviderGenerator, options ImportOptions, providerWrapper *providerwrapper.ProviderWrapper, wg *sync.WaitGroup, failedServices chan string) { log.Println(provider.GetName() + " importing... " + service) err := provider.InitService(service, options.Verbose) @@ -295,25 +219,6 @@ func initServiceResources(service string, provider terraformutils.ProviderGenera wg.Done() } -func refreshServiceResources(provider terraformutils.ProviderGenerator, providerWrapper *providerwrapper.ProviderWrapper, refreshedResources []terraformutils.Resource) ([]terraformutils.Resource, error) { - provider.GetService().SetResources(refreshedResources) - - for i := range provider.GetService().GetResources() { - err := provider.GetService().GetResources()[i].ConvertTFstate(providerWrapper) - if err != nil { - return nil, err - } - } - provider.GetService().PostRefreshCleanup() - - // change structs with additional data for each resource - err := provider.GetService().PostConvertHook() - if err != nil { - return nil, err - } - return provider.GetService().GetResources(), nil -} - func getResourcesAddresses(resources []terraformutils.Resource) []*terraformutils.Resource { results := []*terraformutils.Resource{} for i := range resources { @@ -323,110 +228,6 @@ func getResourcesAddresses(resources []terraformutils.Resource) []*terraformutil return results } -func getResourcesValues(resources []*terraformutils.Resource) []terraformutils.Resource { - results := []terraformutils.Resource{} - for i := range resources { - results = append(results, *resources[i]) - } - - return results -} - -func Import(provider terraformutils.ProviderGenerator, options ImportOptions, args []string) error { - err := provider.Init(args) - if err != nil { - return err - } - - plan := &ImportPlan{ - Provider: provider.GetName(), - Options: options, - Args: args, - ImportedResource: map[string][]terraformutils.Resource{}, - } - - if terraformerstring.ContainsString(options.Resources, "*") { - log.Println("Attempting an import of ALL resources in " + provider.GetName()) - options.Resources = providerServices(provider) - } - - if options.Excludes != nil { - localSlice := []string{} - for _, r := range options.Resources { - remove := false - for _, e := range options.Excludes { - if r == e { - remove = true - log.Println("Excluding resource " + e) - } - } - if !remove { - localSlice = append(localSlice, r) - } - } - options.Resources = localSlice - } - - providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose, options.RetryCount, options.RetrySleepMs) - if err != nil { - return err - } - - defer providerWrapper.Kill() - - for _, service := range options.Resources { - resources, err := buildServiceResources(service, provider, options, providerWrapper) - if err != nil { - log.Println(err) - continue - } - plan.ImportedResource[service] = append(plan.ImportedResource[service], resources...) - } - if options.Plan { - path := Path(options.PathPattern, provider.GetName(), "terraformer", options.PathOutput) - return ExportPlanFile(plan, path, "plan.json") - } - return ImportFromPlan(provider, plan) -} - -func buildServiceResources(service string, provider terraformutils.ProviderGenerator, - options ImportOptions, providerWrapper *providerwrapper.ProviderWrapper) ([]terraformutils.Resource, error) { - log.Println(provider.GetName() + " importing... " + service) - err := provider.InitService(service, options.Verbose) - if err != nil { - return nil, err - } - provider.GetService().ParseFilters(options.Filter) - err = provider.GetService().InitResources() - if err != nil { - return nil, err - } - - provider.GetService().PopulateIgnoreKeys(providerWrapper) - provider.GetService().InitialCleanup() - - refreshedResources, err := terraformutils.RefreshResources(getResourcesAddresses(provider.GetService().GetResources()), providerWrapper, nil) - if err != nil { - return nil, err - } - provider.GetService().SetResources(getResourcesValues(refreshedResources)) - - for i := range provider.GetService().GetResources() { - err = provider.GetService().GetResources()[i].ConvertTFstate(providerWrapper) - if err != nil { - return nil, err - } - } - provider.GetService().PostRefreshCleanup() - - // change structs with additional data for each resource - err = provider.GetService().PostConvertHook() - if err != nil { - return nil, err - } - return provider.GetService().GetResources(), nil -} - func ImportFromPlan(provider terraformutils.ProviderGenerator, plan *ImportPlan) error { options := plan.Options importedResource := plan.ImportedResource diff --git a/cmd/provider_cmd_aws.go b/cmd/provider_cmd_aws.go index fa168cc61d..7a00c74315 100644 --- a/cmd/provider_cmd_aws.go +++ b/cmd/provider_cmd_aws.go @@ -109,15 +109,7 @@ func importRegionResources(options ImportOptions, originalPathPattern string, re } else { log.Println(provider.GetName() + " importing default region") } - args := []string{region, options.Profile} - - providerWrapper, options, err := initOptionsAndWrapper(provider, options, args) - providersPerResources := make([]terraformutils.ProviderGenerator, len(options.Resources)) - for i := range options.Resources { - providersPerResources[i] = newAWSProvider() - } - err = ImportRoundRobin(providersPerResources, options, args, providerWrapper) - //err := Import(provider, options, []string{region, options.Profile}) + err := Import(provider, options, []string{region, options.Profile}) if err != nil { return err } diff --git a/main_test.go b/main_test.go index 886a62e1b4..b50998bced 100644 --- a/main_test.go +++ b/main_test.go @@ -37,3 +37,33 @@ func TestTerraformerMain(t1 *testing.T) { }) } + +func TestTerraformerMain2(t1 *testing.T) { + t1.Run("run main", func(t1 *testing.T) { + tCommand := cmd.NewCmdRoot() + tCommand.SetArgs([]string{ + "import", + "aws", + //"--regions=sa-east-1", + "--regions=us-west-1", + //"--regions=us-east-1", + //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", + "--resources=\"*\"", + //"--resources=sg", + //"--resources=cloudformation,sg,s3", + //"--resources=cloudformation,sg", + //"--resources=sg,vpc", + //"--resources=kms", + "--profile=nubank", + //"--retry-number=5", + //"--retry-sleep-ms=300", + }) + start := time.Now() + if err := tCommand.Execute(); err != nil { + log.Println(err) + os.Exit(1) + } + log.Printf("Importing took %s", time.Since(start)) + + }) +} diff --git a/terraformutils/providers_mapping.go b/terraformutils/providers_mapping.go new file mode 100644 index 0000000000..ffe17ec7de --- /dev/null +++ b/terraformutils/providers_mapping.go @@ -0,0 +1,151 @@ +package terraformutils + +import ( + "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" + "log" + "math/rand" + "reflect" + "time" +) + +type ProvidersMapping struct { + baseProvider ProviderGenerator + Resources map[*Resource]bool + Services map[string]bool + Providers map[ProviderGenerator]bool + providerToService map[ProviderGenerator]string + serviceToProvider map[string]ProviderGenerator + resourceToProvider map[*Resource]ProviderGenerator +} + +func NewProvidersMapping(baseProvider ProviderGenerator) *ProvidersMapping { + providersMapping := &ProvidersMapping{ + baseProvider: baseProvider, + Resources: map[*Resource]bool{}, + Services: map[string]bool{}, + Providers: map[ProviderGenerator]bool{}, + providerToService: map[ProviderGenerator]string{}, + serviceToProvider: map[string]ProviderGenerator{}, + resourceToProvider: map[*Resource]ProviderGenerator{}, + } + + return providersMapping +} + +func deepCopyProvider(provider ProviderGenerator) ProviderGenerator { + var copy ProviderGenerator + copy = reflect.New(reflect.ValueOf(provider).Elem().Type()).Interface().(ProviderGenerator) + + return copy +} + +func (p *ProvidersMapping) GetBaseProvider() ProviderGenerator { + return p.baseProvider +} + +func (p *ProvidersMapping) AddServiceToProvider(service string) ProviderGenerator { + newProvider := deepCopyProvider(p.baseProvider) + p.Providers[newProvider] = true + p.Services[service] = true + p.providerToService[newProvider] = service + + return newProvider +} + +func (p *ProvidersMapping) GetServices() []string { + services := make([]string, len(p.Services)) + for service := range p.Services { + services = append(services, service) + } + + return services +} + +func (p *ProvidersMapping) RemoveServices(services []string) { + for _, service := range services { + delete(p.Services, service) + + matchingProvider := p.serviceToProvider[service] + delete(p.Providers, matchingProvider) + delete(p.providerToService, matchingProvider) + delete(p.serviceToProvider, service) + } +} + +func (p *ProvidersMapping) ShuffleResources() []*Resource { + resources := []*Resource{} + for resource := range p.Resources { + resources = append(resources, resource) + } + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(resources), func(i, j int) { resources[i], resources[j] = resources[j], resources[i] }) + + return resources +} + +func (p *ProvidersMapping) ProcessResources() { + for provider := range p.Providers { + resources := provider.GetService().GetResources() + for i := range resources { + resource := resources[i] + p.Resources[&resource] = true + p.resourceToProvider[&resource] = provider + } + } +} + +func (p *ProvidersMapping) MatchProvider(resource *Resource) ProviderGenerator { + return p.resourceToProvider[resource] +} + +func (p *ProvidersMapping) SetResources(resourceToKeep []*Resource) { + p.Resources = map[*Resource]bool{} + resourcesGroupsByProviders := map[ProviderGenerator][]Resource{} + for i := range resourceToKeep { + resource := resourceToKeep[i] + provider := p.resourceToProvider[resource] + if resourcesGroupsByProviders[provider] == nil { + resourcesGroupsByProviders[provider] = []Resource{} + } + resourcesGroupsByProviders[provider] = append(resourcesGroupsByProviders[provider], *resource) + p.Resources[resource] = true + } + + for provider := range p.Providers { + provider.GetService().SetResources(resourcesGroupsByProviders[provider]) + } +} + +func (p *ProvidersMapping) GetResourcesByService() map[string][]Resource { + mapping := map[string][]Resource{} + for service := range p.Services { + mapping[service] = []Resource{} + } + + for resource := range p.Resources { + provider := p.resourceToProvider[resource] + service := p.providerToService[provider] + mapping[service] = append(mapping[service], *resource) + } + + return mapping +} + +func (p *ProvidersMapping) ConvertTFStates(providerWrapper *providerwrapper.ProviderWrapper) { + for resource := range p.Resources { + err := resource.ConvertTFstate(providerWrapper) + if err != nil { + log.Printf("failed to convert resources %s because of error %s", resource.InstanceInfo.Id, err) + } + } +} + +func (p *ProvidersMapping) CleanupProviders() { + for provider := range p.Providers { + provider.GetService().PostRefreshCleanup() + err := provider.GetService().PostConvertHook() + if err != nil { + log.Printf("failed run PostConvertHook because of error %s", err) + } + } +} diff --git a/terraformutils/utils.go b/terraformutils/utils.go index 317a408f3c..d5d5e05305 100644 --- a/terraformutils/utils.go +++ b/terraformutils/utils.go @@ -16,11 +16,10 @@ package terraformutils import ( "bytes" + "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" "log" "sync" - "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" - "github.com/hashicorp/terraform/terraform" ) @@ -70,9 +69,7 @@ func RefreshResources(resources []*Resource, provider *providerwrapper.ProviderW input := make(chan *Resource, 100) var wg sync.WaitGroup poolSize := 15 - //if slowProcessingRequired(resources) { - // poolSize = 1 - //} + for i := 0; i < poolSize; i++ { go RefreshResourceWorker(input, &wg, provider) } @@ -117,24 +114,20 @@ func RefreshResources(resources []*Resource, provider *providerwrapper.ProviderW return refreshedResources, nil } -func RefreshResourcesByProvider(resourcesByProvider []map[*Resource]ProviderGenerator, providerWrapper *providerwrapper.ProviderWrapper) (map[*Resource]ProviderGenerator, error) { - flattenedResourcesByProvider := make(map[*Resource]ProviderGenerator) - refreshedResourcesByProvider := make(map[*Resource]ProviderGenerator) +func RefreshResourcesByProvider(providersMapping *ProvidersMapping, providerWrapper *providerwrapper.ProviderWrapper) error { + allResources := providersMapping.ShuffleResources() slowProcessingResources := make(map[ProviderGenerator][]*Resource) - var resources []*Resource - for _, resourceProviderPair := range resourcesByProvider { - for resourcePtr := range resourceProviderPair { - if resourcePtr.SlowQueryRequired { - provider := resourceProviderPair[resourcePtr] - if slowProcessingResources[provider] == nil { - slowProcessingResources[provider] = []*Resource{} - } - slowProcessingResources[provider] = append(slowProcessingResources[provider], resourcePtr) - flattenedResourcesByProvider[resourcePtr] = resourceProviderPair[resourcePtr] - } else { - flattenedResourcesByProvider[resourcePtr] = resourceProviderPair[resourcePtr] - resources = append(resources, resourcePtr) + regularResources := []*Resource{} + for i := range allResources { + resource := allResources[i] + if resource.SlowQueryRequired { + provider := providersMapping.MatchProvider(resource) + if slowProcessingResources[provider] == nil { + slowProcessingResources[provider] = []*Resource{} } + slowProcessingResources[provider] = append(slowProcessingResources[provider], resource) + } else { + regularResources = append(regularResources, resource) } } @@ -143,23 +136,13 @@ func RefreshResourcesByProvider(resourcesByProvider []map[*Resource]ProviderGene spResourcesList = append(spResourcesList, slowProcessingResources[p]) } - refreshedResources, err := RefreshResources(resources, providerWrapper, spResourcesList) + refreshedResources, err := RefreshResources(regularResources, providerWrapper, spResourcesList) if err != nil { - return nil, err - } - for _, r := range refreshedResources { - refreshedResourcesByProvider[r] = flattenedResourcesByProvider[r] + return err } - return refreshedResourcesByProvider, nil -} -func slowProcessingRequired(resources []*Resource) bool { - for _, r := range resources { - if r.SlowQueryRequired { - return true - } - } - return false + providersMapping.SetResources(refreshedResources) + return nil } func RefreshResourceWorker(input chan *Resource, wg *sync.WaitGroup, provider *providerwrapper.ProviderWrapper) { From 4f2825dea4ba9937c3dc2389e9b0155c0df72563 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:18:11 +0200 Subject: [PATCH 03/11] fixed problem with provider_mapping --- main_test.go | 4 ++-- terraformutils/providers_mapping.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/main_test.go b/main_test.go index b50998bced..87a7a5cb7b 100644 --- a/main_test.go +++ b/main_test.go @@ -48,11 +48,11 @@ func TestTerraformerMain2(t1 *testing.T) { "--regions=us-west-1", //"--regions=us-east-1", //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", - "--resources=\"*\"", + //"--resources=\"*\"", //"--resources=sg", //"--resources=cloudformation,sg,s3", //"--resources=cloudformation,sg", - //"--resources=sg,vpc", + "--resources=sg,vpc,codepipeline,efs,ebs", //"--resources=kms", "--profile=nubank", //"--retry-number=5", diff --git a/terraformutils/providers_mapping.go b/terraformutils/providers_mapping.go index ffe17ec7de..3936dbdb2f 100644 --- a/terraformutils/providers_mapping.go +++ b/terraformutils/providers_mapping.go @@ -48,6 +48,7 @@ func (p *ProvidersMapping) AddServiceToProvider(service string) ProviderGenerato p.Providers[newProvider] = true p.Services[service] = true p.providerToService[newProvider] = service + p.serviceToProvider[service] = newProvider return newProvider } @@ -86,6 +87,7 @@ func (p *ProvidersMapping) ShuffleResources() []*Resource { func (p *ProvidersMapping) ProcessResources() { for provider := range p.Providers { resources := provider.GetService().GetResources() + log.Printf("num of resources for service %s: %d", p.providerToService[provider], len(provider.GetService().GetResources())) for i := range resources { resource := resources[i] p.Resources[&resource] = true From c47bdf9e1e6f7d1d1b5fc4b436920931e8496412 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:18:24 +0200 Subject: [PATCH 04/11] deleted main_test.go --- main_test.go | 69 ---------------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 main_test.go diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 87a7a5cb7b..0000000000 --- a/main_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "github.com/GoogleCloudPlatform/terraformer/cmd" - "log" - "os" - "testing" - "time" -) - -func TestTerraformerMain(t1 *testing.T) { - t1.Run("run main", func(t1 *testing.T) { - tCommand := cmd.NewCmdRoot() - tCommand.SetArgs([]string{ - "import", - "aws", - "--regions=sa-east-1", - //"--regions=us-west-1", - //"--regions=us-east-1", - //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", - "--resources=\"*\"", - //"--resources=sg", - //"--resources=cloudformation,sg,s3", - //"--resources=cloudformation,sg", - //"--resources=sg,vpc", - //"--resources=kms", - "--profile=nubank", - //"--retry-number=5", - //"--retry-sleep-ms=300", - }) - start := time.Now() - if err := tCommand.Execute(); err != nil { - log.Println(err) - os.Exit(1) - } - log.Printf("Importing took %s", time.Since(start)) - - }) -} - -func TestTerraformerMain2(t1 *testing.T) { - t1.Run("run main", func(t1 *testing.T) { - tCommand := cmd.NewCmdRoot() - tCommand.SetArgs([]string{ - "import", - "aws", - //"--regions=sa-east-1", - "--regions=us-west-1", - //"--regions=us-east-1", - //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", - //"--resources=\"*\"", - //"--resources=sg", - //"--resources=cloudformation,sg,s3", - //"--resources=cloudformation,sg", - "--resources=sg,vpc,codepipeline,efs,ebs", - //"--resources=kms", - "--profile=nubank", - //"--retry-number=5", - //"--retry-sleep-ms=300", - }) - start := time.Now() - if err := tCommand.Execute(); err != nil { - log.Println(err) - os.Exit(1) - } - log.Printf("Importing took %s", time.Since(start)) - - }) -} From 74272869ef22e0537b128dd23eb5487ab5090236 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:20:42 +0200 Subject: [PATCH 05/11] Revert "deleted main_test.go" This reverts commit c47bdf9e --- main_test.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 main_test.go diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000000..87a7a5cb7b --- /dev/null +++ b/main_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "github.com/GoogleCloudPlatform/terraformer/cmd" + "log" + "os" + "testing" + "time" +) + +func TestTerraformerMain(t1 *testing.T) { + t1.Run("run main", func(t1 *testing.T) { + tCommand := cmd.NewCmdRoot() + tCommand.SetArgs([]string{ + "import", + "aws", + "--regions=sa-east-1", + //"--regions=us-west-1", + //"--regions=us-east-1", + //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", + "--resources=\"*\"", + //"--resources=sg", + //"--resources=cloudformation,sg,s3", + //"--resources=cloudformation,sg", + //"--resources=sg,vpc", + //"--resources=kms", + "--profile=nubank", + //"--retry-number=5", + //"--retry-sleep-ms=300", + }) + start := time.Now() + if err := tCommand.Execute(); err != nil { + log.Println(err) + os.Exit(1) + } + log.Printf("Importing took %s", time.Since(start)) + + }) +} + +func TestTerraformerMain2(t1 *testing.T) { + t1.Run("run main", func(t1 *testing.T) { + tCommand := cmd.NewCmdRoot() + tCommand.SetArgs([]string{ + "import", + "aws", + //"--regions=sa-east-1", + "--regions=us-west-1", + //"--regions=us-east-1", + //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", + //"--resources=\"*\"", + //"--resources=sg", + //"--resources=cloudformation,sg,s3", + //"--resources=cloudformation,sg", + "--resources=sg,vpc,codepipeline,efs,ebs", + //"--resources=kms", + "--profile=nubank", + //"--retry-number=5", + //"--retry-sleep-ms=300", + }) + start := time.Now() + if err := tCommand.Execute(); err != nil { + log.Println(err) + os.Exit(1) + } + log.Printf("Importing took %s", time.Since(start)) + + }) +} From dc94acd78e70c724746109226be1d1ea026f2d2c Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:28:04 +0200 Subject: [PATCH 06/11] Revert "Revert "deleted main_test.go"" This reverts commit 74272869 --- main_test.go | 69 ---------------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 main_test.go diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 87a7a5cb7b..0000000000 --- a/main_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "github.com/GoogleCloudPlatform/terraformer/cmd" - "log" - "os" - "testing" - "time" -) - -func TestTerraformerMain(t1 *testing.T) { - t1.Run("run main", func(t1 *testing.T) { - tCommand := cmd.NewCmdRoot() - tCommand.SetArgs([]string{ - "import", - "aws", - "--regions=sa-east-1", - //"--regions=us-west-1", - //"--regions=us-east-1", - //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", - "--resources=\"*\"", - //"--resources=sg", - //"--resources=cloudformation,sg,s3", - //"--resources=cloudformation,sg", - //"--resources=sg,vpc", - //"--resources=kms", - "--profile=nubank", - //"--retry-number=5", - //"--retry-sleep-ms=300", - }) - start := time.Now() - if err := tCommand.Execute(); err != nil { - log.Println(err) - os.Exit(1) - } - log.Printf("Importing took %s", time.Since(start)) - - }) -} - -func TestTerraformerMain2(t1 *testing.T) { - t1.Run("run main", func(t1 *testing.T) { - tCommand := cmd.NewCmdRoot() - tCommand.SetArgs([]string{ - "import", - "aws", - //"--regions=sa-east-1", - "--regions=us-west-1", - //"--regions=us-east-1", - //"--regions=ap-northeast-1,eu-central-1,eu-west-1,us-east-1,af-south-1,ap-northeast-2,eu-west-2,us-west-1,us-west-2,ap-east-1,ap-south-1,ap-southeast-2,ca-central-1,sa-east-1,us-east-2,ap-southeast-1,aws-global,eu-north-1,eu-south-1,eu-west-3,me-south-1", - //"--resources=\"*\"", - //"--resources=sg", - //"--resources=cloudformation,sg,s3", - //"--resources=cloudformation,sg", - "--resources=sg,vpc,codepipeline,efs,ebs", - //"--resources=kms", - "--profile=nubank", - //"--retry-number=5", - //"--retry-sleep-ms=300", - }) - start := time.Now() - if err := tCommand.Execute(); err != nil { - log.Println(err) - os.Exit(1) - } - log.Printf("Importing took %s", time.Since(start)) - - }) -} From 9c32a1f787223d83683c35bc1c39250bc6fc31d5 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Tue, 19 Jan 2021 18:39:20 +0200 Subject: [PATCH 07/11] refactored initAllServicesResources, added flags to README.md --- README.md | 2 ++ cmd/import.go | 20 +++----------------- terraformutils/providers_mapping.go | 4 ++-- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8acd5fe220..698bf964ba 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ Flags: -r, --resources strings firewall,networks or * for all services -s, --state string local or bucket (default "local") -v, --verbose verbose mode + -n, --retry-number number of retries to perform if refresh fails + -m, --retry-sleep-ms time in ms to sleep between retries Use " import [provider] [command] --help" for more information about a command. ``` diff --git a/cmd/import.go b/cmd/import.go index 935d6afe2c..ea75df00bb 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -148,7 +148,7 @@ func initAllServicesResources(providersMapping *terraformutils.ProvidersMapping, var wg sync.WaitGroup wg.Add(numOfResources) - failedServicesChan := make(chan string, numOfResources) + failedServices := make(chan string, numOfResources) for _, service := range options.Resources { serviceProvider := providersMapping.AddServiceToProvider(service) @@ -156,17 +156,12 @@ func initAllServicesResources(providersMapping *terraformutils.ProvidersMapping, if err != nil { return err } - go initServiceResourcesWorker(service, serviceProvider, options, providerWrapper, &wg, failedServicesChan) + go initServiceResourcesWorker(service, serviceProvider, options, providerWrapper, &wg, failedServices) } wg.Wait() - close(failedServicesChan) + close(failedServices) // remove providers that failed to init their service - var failedServices []string - for failedService := range failedServicesChan { - failedServices = append(failedServices, failedService) - } - providersMapping.RemoveServices(failedServices) providersMapping.ProcessResources() @@ -219,15 +214,6 @@ func initServiceResourcesWorker(service string, provider terraformutils.Provider wg.Done() } -func getResourcesAddresses(resources []terraformutils.Resource) []*terraformutils.Resource { - results := []*terraformutils.Resource{} - for i := range resources { - results = append(results, &resources[i]) - } - - return results -} - func ImportFromPlan(provider terraformutils.ProviderGenerator, plan *ImportPlan) error { options := plan.Options importedResource := plan.ImportedResource diff --git a/terraformutils/providers_mapping.go b/terraformutils/providers_mapping.go index 3936dbdb2f..26e143c48b 100644 --- a/terraformutils/providers_mapping.go +++ b/terraformutils/providers_mapping.go @@ -62,8 +62,8 @@ func (p *ProvidersMapping) GetServices() []string { return services } -func (p *ProvidersMapping) RemoveServices(services []string) { - for _, service := range services { +func (p *ProvidersMapping) RemoveServices(services chan string) { + for service := range services { delete(p.Services, service) matchingProvider := p.serviceToProvider[service] From 2667d2c7be5d1ca8d2ccbff7488cad00c6cbb864 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Wed, 20 Jan 2021 09:28:52 +0200 Subject: [PATCH 08/11] fixed linters and problem with PostConvertHook --- cmd/import.go | 3 ++- terraformutils/providers_mapping.go | 22 +++++++++++++++++----- terraformutils/utils.go | 3 ++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index ea75df00bb..ad16e66f48 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -15,7 +15,6 @@ package cmd import ( "fmt" - "github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring" "io/ioutil" "log" "os" @@ -23,6 +22,8 @@ import ( "strings" "sync" + "github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring" + "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" "github.com/spf13/pflag" diff --git a/terraformutils/providers_mapping.go b/terraformutils/providers_mapping.go index 26e143c48b..13d8f686fb 100644 --- a/terraformutils/providers_mapping.go +++ b/terraformutils/providers_mapping.go @@ -1,11 +1,12 @@ package terraformutils import ( - "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" "log" "math/rand" "reflect" "time" + + "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" ) type ProvidersMapping struct { @@ -33,10 +34,7 @@ func NewProvidersMapping(baseProvider ProviderGenerator) *ProvidersMapping { } func deepCopyProvider(provider ProviderGenerator) ProviderGenerator { - var copy ProviderGenerator - copy = reflect.New(reflect.ValueOf(provider).Elem().Type()).Interface().(ProviderGenerator) - - return copy + return reflect.New(reflect.ValueOf(provider).Elem().Type()).Interface().(ProviderGenerator) } func (p *ProvidersMapping) GetBaseProvider() ProviderGenerator { @@ -140,6 +138,20 @@ func (p *ProvidersMapping) ConvertTFStates(providerWrapper *providerwrapper.Prov log.Printf("failed to convert resources %s because of error %s", resource.InstanceInfo.Id, err) } } + + resourcesGroupsByProviders := map[ProviderGenerator][]Resource{} + for resource := range p.Resources { + provider := p.resourceToProvider[resource] + if resourcesGroupsByProviders[provider] == nil { + resourcesGroupsByProviders[provider] = []Resource{} + } + resourcesGroupsByProviders[provider] = append(resourcesGroupsByProviders[provider], *resource) + } + + for provider := range p.Providers { + provider.GetService().SetResources(resourcesGroupsByProviders[provider]) + } + } func (p *ProvidersMapping) CleanupProviders() { diff --git a/terraformutils/utils.go b/terraformutils/utils.go index d5d5e05305..4da9420b64 100644 --- a/terraformutils/utils.go +++ b/terraformutils/utils.go @@ -16,10 +16,11 @@ package terraformutils import ( "bytes" - "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" "log" "sync" + "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" + "github.com/hashicorp/terraform/terraform" ) From c59dc4663ba9cc114dfb6477e7d6a517f4077a02 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Wed, 3 Feb 2021 11:12:11 +0200 Subject: [PATCH 09/11] Fixed `RefreshResources` function - increase channel size to avoid getting stuck --- terraformutils/utils.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/terraformutils/utils.go b/terraformutils/utils.go index 4da9420b64..684a9f7e09 100644 --- a/terraformutils/utils.go +++ b/terraformutils/utils.go @@ -67,33 +67,34 @@ func PrintTfState(resources []Resource) ([]byte, error) { func RefreshResources(resources []*Resource, provider *providerwrapper.ProviderWrapper, slowProcessingResources [][]*Resource) ([]*Resource, error) { refreshedResources := []*Resource{} - input := make(chan *Resource, 100) + input := make(chan *Resource, len(resources)) var wg sync.WaitGroup poolSize := 15 - - for i := 0; i < poolSize; i++ { - go RefreshResourceWorker(input, &wg, provider) - } for i := range resources { wg.Add(1) input <- resources[i] } + close(input) + + for i := 0; i < poolSize; i++ { + go RefreshResourceWorker(input, &wg, provider) + } spInputs := []chan *Resource{} for i, resourceGroup := range slowProcessingResources { - spInputs = append(spInputs, make(chan *Resource, 100)) + spInputs = append(spInputs, make(chan *Resource, len(resourceGroup))) for j := range resourceGroup { spInputs[i] <- resourceGroup[j] } + close(spInputs[i]) } for i := 0; i < len(spInputs); i++ { - go RefreshResourceWorker(spInputs[i], &wg, provider) wg.Add(len(slowProcessingResources[i])) + go RefreshResourceWorker(spInputs[i], &wg, provider) } wg.Wait() - close(input) for _, r := range resources { if r.InstanceState != nil && r.InstanceState.ID != "" { refreshedResources = append(refreshedResources, r) From 9bd9081a21bca14c052c144d77eddaeba9be7e2b Mon Sep 17 00:00:00 2001 From: rotemavni Date: Wed, 3 Feb 2021 11:58:33 +0200 Subject: [PATCH 10/11] allow to pass multiple options to NewProviderWrapper --- cmd/import.go | 2 +- terraformutils/providerwrapper/provider.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index ad16e66f48..6fdc3e6189 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -136,7 +136,7 @@ func initOptionsAndWrapper(provider terraformutils.ProviderGenerator, options Im options.Resources = localSlice } - providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose, options.RetryCount, options.RetrySleepMs) + providerWrapper, err := providerwrapper.NewProviderWrapper(provider.GetName(), provider.GetConfig(), options.Verbose, map[string]int{"retryCount": options.RetryCount, "retrySleepMs": options.RetrySleepMs}) if err != nil { return nil, options, err } diff --git a/terraformutils/providerwrapper/provider.go b/terraformutils/providerwrapper/provider.go index 30fb5efb1f..e6cf3f5ee5 100644 --- a/terraformutils/providerwrapper/provider.go +++ b/terraformutils/providerwrapper/provider.go @@ -60,17 +60,20 @@ type ProviderWrapper struct { retrySleepMs int } -func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool, retryOptions ...int) (*ProviderWrapper, error) { - p := &ProviderWrapper{} +func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool, options ...map[string]int) (*ProviderWrapper, error) { + p := &ProviderWrapper{retryCount: 5, retrySleepMs: 300} p.providerName = providerName p.config = providerConfig - if len(retryOptions) == 2 { - p.retryCount = retryOptions[0] - p.retrySleepMs = retryOptions[1] - } else { - p.retryCount = 5 - p.retrySleepMs = 300 + if len(options) > 0 { + retryCount, hasOption := options[0]["retryCount"] + if hasOption { + p.retryCount = retryCount + } + retrySleepMs, hasOption := options[0]["retrySleepMs"] + if hasOption { + p.retrySleepMs = retrySleepMs + } } err := p.initProvider(verbose) From 08e3753f9f2b85e0f4f8e7765c3eb983d6bdf0e7 Mon Sep 17 00:00:00 2001 From: rotemavni Date: Wed, 3 Feb 2021 13:27:28 +0200 Subject: [PATCH 11/11] Run resource initialization sequentially --- cmd/import.go | 24 +++++++++++------------- terraformutils/providers_mapping.go | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 6fdc3e6189..f3331a14e5 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -149,7 +149,7 @@ func initAllServicesResources(providersMapping *terraformutils.ProvidersMapping, var wg sync.WaitGroup wg.Add(numOfResources) - failedServices := make(chan string, numOfResources) + failedServices := []string{} for _, service := range options.Resources { serviceProvider := providersMapping.AddServiceToProvider(service) @@ -157,10 +157,11 @@ func initAllServicesResources(providersMapping *terraformutils.ProvidersMapping, if err != nil { return err } - go initServiceResourcesWorker(service, serviceProvider, options, providerWrapper, &wg, failedServices) + err = initServiceResources(service, serviceProvider, options, providerWrapper) + if err != nil { + failedServices = append(failedServices, service) + } } - wg.Wait() - close(failedServices) // remove providers that failed to init their service providersMapping.RemoveServices(failedServices) @@ -190,29 +191,26 @@ func importFromPlan(providerMapping *terraformutils.ProvidersMapping, options Im return ImportFromPlan(providerMapping.GetBaseProvider(), plan) } -func initServiceResourcesWorker(service string, provider terraformutils.ProviderGenerator, - options ImportOptions, providerWrapper *providerwrapper.ProviderWrapper, wg *sync.WaitGroup, failedServices chan string) { +func initServiceResources(service string, provider terraformutils.ProviderGenerator, + options ImportOptions, providerWrapper *providerwrapper.ProviderWrapper) error { log.Println(provider.GetName() + " importing... " + service) err := provider.InitService(service, options.Verbose) if err != nil { - failedServices <- service log.Printf("%s error importing %s, err: %s\n", provider.GetName(), service, err) - wg.Done() - return + return err } provider.GetService().ParseFilters(options.Filter) err = provider.GetService().InitResources() if err != nil { - failedServices <- service log.Printf("%s error initializing resources in service %s, err: %s\n", provider.GetName(), service, err) - wg.Done() - return + return err } provider.GetService().PopulateIgnoreKeys(providerWrapper) provider.GetService().InitialCleanup() log.Println(provider.GetName() + " done importing " + service) - wg.Done() + + return nil } func ImportFromPlan(provider terraformutils.ProviderGenerator, plan *ImportPlan) error { diff --git a/terraformutils/providers_mapping.go b/terraformutils/providers_mapping.go index 13d8f686fb..ce9fd3dd3d 100644 --- a/terraformutils/providers_mapping.go +++ b/terraformutils/providers_mapping.go @@ -60,8 +60,8 @@ func (p *ProvidersMapping) GetServices() []string { return services } -func (p *ProvidersMapping) RemoveServices(services chan string) { - for service := range services { +func (p *ProvidersMapping) RemoveServices(services []string) { + for _, service := range services { delete(p.Services, service) matchingProvider := p.serviceToProvider[service]