From a08e356092b6cce7eeb371212eeeb745ea0c0317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krisztia=CC=81n=20Go=CC=88drei?= Date: Wed, 26 Jun 2024 16:10:04 +0200 Subject: [PATCH 1/4] stepman setup and update step lib improvements and benchmark --- _tests/integration/update_test.go | 142 ++++++++++++++++ stepman/library.go | 266 +++++++++++++++++++++++++++++- 2 files changed, 407 insertions(+), 1 deletion(-) diff --git a/_tests/integration/update_test.go b/_tests/integration/update_test.go index 5dd3d187..a9ed96de 100644 --- a/_tests/integration/update_test.go +++ b/_tests/integration/update_test.go @@ -1,12 +1,15 @@ package integration import ( + "errors" + "fmt" "os" "testing" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/command/git" "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/stepman/stepman" "github.com/stretchr/testify/require" ) @@ -50,3 +53,142 @@ func TestUpdate(t *testing.T) { require.NoError(t, err, out) } } + +func Test_StepLibSetup(t *testing.T) { + steplib := "https://github.com/bitrise-io/bitrise-steplib.git" + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + err := setupStepLib(steplib, oldCommit) + if err != nil { + t.Fatal(err) + } +} + +func Test_StepLibUpdate(t *testing.T) { + steplib := "https://github.com/bitrise-io/bitrise-steplib.git" + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + err := setupStepLib(steplib, oldCommit) + if err != nil { + t.Fatal(err) + } + + if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { + t.Fatal(err) + } +} + +func Test_StepLibUpdate2(t *testing.T) { + steplib := "https://github.com/bitrise-io/bitrise-steplib.git" + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + err := setupStepLib(steplib, oldCommit) + if err != nil { + t.Fatal(err) + } + + if _, err := stepman.UpdateLibrary2(steplib, NopeLogger{}); err != nil { + t.Fatal(err) + } +} + +func BenchmarkStepLibUpdate(b *testing.B) { + steplib := "https://github.com/bitrise-io/bitrise-steplib.git" + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + + for _, mode := range []string{"old", "new", "file"} { + b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + err := setupStepLib(steplib, oldCommit) + if err != nil { + b.Fatal(err) + } + b.StartTimer() + + if mode == "old" { + // 3251444262 ns/op + if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } else if mode == "new" { + // 3052922411 ns/op + if _, err := stepman.UpdateLibrary2(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } else { + if err := stepman.SetupLibrary3(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + } + }) + } +} + +func BenchmarkStepLibSetup(b *testing.B) { + steplib := "https://github.com/bitrise-io/bitrise-steplib.git" + + for _, mode := range []string{"old", "new", "file"} { + b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + route, found := stepman.ReadRoute(steplib) + if found { + if err := stepman.CleanupRoute(route); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + + if mode == "old" { + // 4959929200 ns/op + if err := stepman.SetupLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } else if mode == "new" { + // 4120265216 ns/op + if err := stepman.SetupLibrary2(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } else { + // 2762603411 ns/op + if err := stepman.SetupLibrary3(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + } + }) + } +} + +func setupStepLib(uri, commit string) error { + route, found := stepman.ReadRoute(uri) + if found { + if err := stepman.CleanupRoute(route); err != nil { + return err + } + } + + if err := stepman.SetupLibrary(uri, NopeLogger{}); err != nil { + return err + } + + route, found = stepman.ReadRoute(uri) + if !found { + return errors.New("no rout found") + } + + pth := stepman.GetLibraryBaseDirPath(route) + cmd := command.New("git", "reset", "--hard", commit) + cmd.SetDir(pth) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +type NopeLogger struct { +} + +func (l NopeLogger) Warnf(format string, v ...interface{}) { + return +} diff --git a/stepman/library.go b/stepman/library.go index dfe01c48..d632625c 100644 --- a/stepman/library.go +++ b/stepman/library.go @@ -1,8 +1,13 @@ package stepman import ( + "errors" "fmt" + "io" + "net/http" "os" + "os/exec" + "path/filepath" "strings" "time" @@ -90,6 +95,196 @@ func SetupLibrary(libraryURI string, log Logger) error { return nil } +// SetupLibrary2 ... +func SetupLibrary2(libraryURI string, log Logger) error { + if exist, err := RootExistForLibrary(libraryURI); err != nil { + return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) + } else if exist { + return nil + } + + alias := GenerateFolderAlias() + route := SteplibRoute{ + SteplibURI: libraryURI, + FolderAlias: alias, + } + + // Cleanup + isSuccess := false + defer func() { + if !isSuccess { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) + } + } + }() + + // Setup + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + + pth := GetLibraryBaseDirPath(route) + if !isLocalLibrary { + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + if err := os.MkdirAll(pth, 0755); err != nil { + return err + } + + cmd := command.New("git", "clone", "--no-tags", "--single-branch", "--depth=1", libraryURI, ".") + cmd.SetDir(pth) + log.Warnf("$ %s", cmd.PrintableCommandArgs()) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + + return nil + + //repo, err := git.New(pth) + //if err != nil { + // return err + //} + //return repo.Clone(libraryURI).Run() + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) + } + } else { + // Local spec path + if err := os.MkdirAll(pth, 0777); err != nil { + return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) + } + + libraryFilePath := libraryURI + if strings.HasPrefix(libraryURI, filePathPrefix) { + libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) + } + + if err := command.CopyDir(libraryFilePath, pth, true); err != nil { + return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) + } + } + + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) + } + + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } + + isSuccess = true + + return nil +} + +// SetupLibrary3 ... +func SetupLibrary3(libraryURI string, log Logger) error { + if exist, err := RootExistForLibrary(libraryURI); err != nil { + return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) + } else if exist { + return nil + } + + alias := GenerateFolderAlias() + route := SteplibRoute{ + SteplibURI: libraryURI, + FolderAlias: alias, + } + + // Cleanup + isSuccess := false + defer func() { + if !isSuccess { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) + } + } + }() + + // Setup + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + + pth := GetLibraryBaseDirPath(route) + if !isLocalLibrary { + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + specPath := GetStepSpecPath(route) + + if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { + return err + } + + specURL := "https://github.com/godrei/ios-EnvVar/releases/download/1.0.0/spec.json" + if err := downloadSpec(specPath, specURL); err != nil { + return err + } + + return nil + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) + } + } else { + // Local spec path + if err := os.MkdirAll(pth, 0777); err != nil { + return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) + } + + libraryFilePath := libraryURI + if strings.HasPrefix(libraryURI, filePathPrefix) { + libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) + } + + if err := command.CopyDir(libraryFilePath, pth, true); err != nil { + return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) + } + } + + if isLocalLibrary { + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) + } + } + + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } + + isSuccess = true + + return nil +} + +func downloadSpec(filepath string, url string) error { + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Get the data + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check server response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + // Writer the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} + // UpdateLibrary ... func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, error) { route, found := ReadRoute(libraryURI) @@ -123,7 +318,76 @@ func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, e if err != nil { return err } - return repo.Pull().Run() + cmd := repo.Pull() + //cmd.SetStdout(os.Stdout) + //cmd.SetStderr(os.Stderr) + return cmd.Run() + }); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) + } + + if err := ReGenerateLibrarySpec(route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) + } + } + + return ReadStepSpec(libraryURI) +} + +// UpdateLibrary2 ... +func UpdateLibrary2(libraryURI string, log Logger) (models.StepCollectionModel, error) { + route, found := ReadRoute(libraryURI) + if !found { + if err := CleanupDanglingLibrary(libraryURI); err != nil { + log.Warnf("Failed to cleaning up library (%s), error: %s", libraryURI, err) + } + return models.StepCollectionModel{}, fmt.Errorf("no route found for library: %s", libraryURI) + } + + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + + if isLocalLibrary { + if err := CleanupRoute(route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err) + } + + if err := SetupLibrary(libraryURI, log); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err) + } + } else { + pth := GetLibraryBaseDirPath(route) + if exists, err := pathutil.IsPathExists(pth); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to check if library (%s) directory (%s) exist, error: %s", libraryURI, pth, err) + } else if !exists { + return models.StepCollectionModel{}, fmt.Errorf("library (%s) not initialized", libraryURI) + } + + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + cmd := command.New("git", "fetch", "--jobs=10", "--depth=1", "--no-tags", "origin", "refs/heads/master") + cmd.SetDir(pth) + log.Warnf("$ %s", cmd.PrintableCommandArgs()) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + + cmd = command.New("git", "checkout", "FETCH_HEAD") + cmd.SetDir(pth) + log.Warnf("$ %s", cmd.PrintableCommandArgs()) + out, err = cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + + return nil }); err != nil { return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) } From 8f9f52939e04f6c381997295a02326c1e0bc42a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krisztia=CC=81n=20Go=CC=88drei?= Date: Fri, 28 Jun 2024 16:04:05 +0200 Subject: [PATCH 2/4] Setup StepLib with remote spec.json --- _tests/integration/update_test.go | 74 +++--- stepman/library.go | 390 ++++++++++-------------------- 2 files changed, 158 insertions(+), 306 deletions(-) diff --git a/_tests/integration/update_test.go b/_tests/integration/update_test.go index a9ed96de..0d2a8423 100644 --- a/_tests/integration/update_test.go +++ b/_tests/integration/update_test.go @@ -84,39 +84,33 @@ func Test_StepLibUpdate2(t *testing.T) { t.Fatal(err) } - if _, err := stepman.UpdateLibrary2(steplib, NopeLogger{}); err != nil { + if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { t.Fatal(err) } } func BenchmarkStepLibUpdate(b *testing.B) { - steplib := "https://github.com/bitrise-io/bitrise-steplib.git" oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 - for _, mode := range []string{"old", "new", "file"} { + for _, mode := range []string{"old", "new"} { b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { for i := 0; i < b.N; i++ { - b.StopTimer() - err := setupStepLib(steplib, oldCommit) - if err != nil { - b.Fatal(err) - } - b.StartTimer() - + var steplib string if mode == "old" { - // 3251444262 ns/op - if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } - } else if mode == "new" { - // 3052922411 ns/op - if _, err := stepman.UpdateLibrary2(steplib, NopeLogger{}); err != nil { + steplib = "https://github.com/bitrise-io/bitrise-steplib" + + b.StopTimer() + err := setupStepLib(steplib, oldCommit) + if err != nil { b.Fatal(err) } + b.StartTimer() } else { - if err := stepman.SetupLibrary3(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + + if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) } } }) @@ -124,11 +118,16 @@ func BenchmarkStepLibUpdate(b *testing.B) { } func BenchmarkStepLibSetup(b *testing.B) { - steplib := "https://github.com/bitrise-io/bitrise-steplib.git" - - for _, mode := range []string{"old", "new", "file"} { + for _, mode := range []string{"old", "new"} { b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { for i := 0; i < b.N; i++ { + var steplib string + if mode == "old" { + steplib = "https://github.com/bitrise-io/bitrise-steplib" + } else { + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + b.StopTimer() route, found := stepman.ReadRoute(steplib) if found { @@ -138,21 +137,8 @@ func BenchmarkStepLibSetup(b *testing.B) { } b.StartTimer() - if mode == "old" { - // 4959929200 ns/op - if err := stepman.SetupLibrary(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } - } else if mode == "new" { - // 4120265216 ns/op - if err := stepman.SetupLibrary2(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } - } else { - // 2762603411 ns/op - if err := stepman.SetupLibrary3(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } + if err := stepman.SetupLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) } } }) @@ -192,3 +178,15 @@ type NopeLogger struct { func (l NopeLogger) Warnf(format string, v ...interface{}) { return } + +func (l NopeLogger) Debugf(format string, v ...interface{}) { + return +} + +func (l NopeLogger) Errorf(format string, v ...interface{}) { + return +} + +func (l NopeLogger) Infof(format string, v ...interface{}) { + return +} diff --git a/stepman/library.go b/stepman/library.go index d632625c..6ac85e31 100644 --- a/stepman/library.go +++ b/stepman/library.go @@ -1,25 +1,26 @@ package stepman import ( - "errors" "fmt" "io" "net/http" "os" - "os/exec" "path/filepath" "strings" "time" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/command/git" - "github.com/bitrise-io/go-utils/pathutil" "github.com/bitrise-io/go-utils/retry" + "github.com/bitrise-io/stepman/models" ) const filePathPrefix = "file://" +const defaultStepLib = "https://github.com/bitrise-io/bitrise-steplib.git" +const DefaultStepLibSpecJSONURL = "http://bitrise-steplib-collection.s3.amazonaws.com/spec.json" + // Logger ... type Logger interface { Debugf(format string, v ...interface{}) @@ -28,7 +29,6 @@ type Logger interface { Infof(format string, v ...interface{}) } -// SetupLibrary ... func SetupLibrary(libraryURI string, log Logger) error { if exist, err := RootExistForLibrary(libraryURI); err != nil { return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) @@ -42,129 +42,102 @@ func SetupLibrary(libraryURI string, log Logger) error { FolderAlias: alias, } - // Cleanup - isSuccess := false - defer func() { - if !isSuccess { + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + if isLocalLibrary { + if err := setupWithLocalStepLib(libraryURI, route); err != nil { + // TODO: is cleanup needed? if err := CleanupRoute(route); err != nil { log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) } + + return err } - }() - // Setup - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + return nil + } - pth := GetLibraryBaseDirPath(route) - if !isLocalLibrary { - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - repo, err := git.New(pth) - if err != nil { - return err + isCustomStepLibrary := libraryURI != defaultStepLib + if isCustomStepLibrary { + if err := setupWithStepLibRepo(libraryURI, route); err != nil { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) } - return repo.Clone(libraryURI).Run() - }); err != nil { - return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) - } - } else { - // Local spec path - if err := os.MkdirAll(pth, 0777); err != nil { - return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) - } - - libraryFilePath := libraryURI - if strings.HasPrefix(libraryURI, filePathPrefix) { - libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) - } - if err := command.CopyDir(libraryFilePath, pth, true); err != nil { - return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) + return err } + return nil } - if err := ReGenerateLibrarySpec(route); err != nil { - return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) - } + if err := setupWithStepLibSpecURL(libraryURI, DefaultStepLibSpecJSONURL, route); err != nil { + if err := setupWithStepLibRepo(libraryURI, route); err != nil { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) + } - if err := AddRoute(route); err != nil { - return fmt.Errorf("failed to add routing, error: %s", err) + return err + } + return nil } - isSuccess = true - return nil } -// SetupLibrary2 ... -func SetupLibrary2(libraryURI string, log Logger) error { - if exist, err := RootExistForLibrary(libraryURI); err != nil { - return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) - } else if exist { - return nil +func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, error) { + route, found := ReadRoute(libraryURI) + if !found { + if err := CleanupDanglingLibrary(libraryURI); err != nil { + log.Warnf("Failed to cleaning up library (%s), error: %s", libraryURI, err) + } + return models.StepCollectionModel{}, fmt.Errorf("no route found for library: %s", libraryURI) } - alias := GenerateFolderAlias() - route := SteplibRoute{ - SteplibURI: libraryURI, - FolderAlias: alias, + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + if isLocalLibrary { + if err := CleanupRoute(route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err) + } + + if err := setupWithLocalStepLib(libraryURI, route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err) + } + + return ReadStepSpec(libraryURI) } - // Cleanup - isSuccess := false - defer func() { - if !isSuccess { - if err := CleanupRoute(route); err != nil { - log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) - } + isCustomStepLibrary := libraryURI != defaultStepLib + if isCustomStepLibrary { + if err := updateWithStepLibRepo(libraryURI, route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err) } - }() - // Setup - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + return ReadStepSpec(libraryURI) + } - pth := GetLibraryBaseDirPath(route) - if !isLocalLibrary { - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - if err := os.MkdirAll(pth, 0755); err != nil { - return err - } + if err := setupWithStepLibSpecURL(libraryURI, DefaultStepLibSpecJSONURL, route); err != nil { + if err := updateWithStepLibRepo(libraryURI, route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err) + } - cmd := command.New("git", "clone", "--no-tags", "--single-branch", "--depth=1", libraryURI, ".") - cmd.SetDir(pth) - log.Warnf("$ %s", cmd.PrintableCommandArgs()) - out, err := cmd.RunAndReturnTrimmedCombinedOutput() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) - } - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) - } + return ReadStepSpec(libraryURI) + } - return nil + return ReadStepSpec(libraryURI) +} - //repo, err := git.New(pth) - //if err != nil { - // return err - //} - //return repo.Clone(libraryURI).Run() - }); err != nil { - return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) - } - } else { - // Local spec path - if err := os.MkdirAll(pth, 0777); err != nil { - return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) - } +func setupWithLocalStepLib(libraryURI string, route SteplibRoute) error { + libraryBaseDir := GetLibraryBaseDirPath(route) - libraryFilePath := libraryURI - if strings.HasPrefix(libraryURI, filePathPrefix) { - libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) - } + if err := os.MkdirAll(libraryBaseDir, 0777); err != nil { + return fmt.Errorf("failed to create library dir (%s), error: %s", libraryBaseDir, err) + } - if err := command.CopyDir(libraryFilePath, pth, true); err != nil { - return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) - } + libraryFilePath := libraryURI + if strings.HasPrefix(libraryFilePath, filePathPrefix) { + libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) + } + + if err := command.CopyDir(libraryFilePath, libraryBaseDir, true); err != nil { + return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, libraryBaseDir, err) } if err := ReGenerateLibrarySpec(route); err != nil { @@ -174,85 +147,81 @@ func SetupLibrary2(libraryURI string, log Logger) error { if err := AddRoute(route); err != nil { return fmt.Errorf("failed to add routing, error: %s", err) } - - isSuccess = true - return nil } -// SetupLibrary3 ... -func SetupLibrary3(libraryURI string, log Logger) error { - if exist, err := RootExistForLibrary(libraryURI); err != nil { - return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) - } else if exist { - return nil +func setupWithStepLibRepo(libraryURI string, route SteplibRoute) error { + libraryBaseDir := GetLibraryBaseDirPath(route) + + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + repo, err := git.New(libraryBaseDir) + if err != nil { + return err + } + return repo.Clone(libraryURI).Run() + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) } - alias := GenerateFolderAlias() - route := SteplibRoute{ - SteplibURI: libraryURI, - FolderAlias: alias, + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) } - // Cleanup - isSuccess := false - defer func() { - if !isSuccess { - if err := CleanupRoute(route); err != nil { - log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) - } - } - }() + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } - // Setup - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + return nil +} +func updateWithStepLibRepo(libraryURI string, route SteplibRoute) error { pth := GetLibraryBaseDirPath(route) - if !isLocalLibrary { - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - specPath := GetStepSpecPath(route) - if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { - return err - } + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + repo, err := git.New(pth) + if err != nil { + return err + } + cmd := repo.Pull() + return cmd.Run() + }); err != nil { + return fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) + } - specURL := "https://github.com/godrei/ios-EnvVar/releases/download/1.0.0/spec.json" - if err := downloadSpec(specPath, specURL); err != nil { - return err - } + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) + } - return nil - }); err != nil { - return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) - } - } else { - // Local spec path - if err := os.MkdirAll(pth, 0777); err != nil { - return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) - } + return nil +} - libraryFilePath := libraryURI - if strings.HasPrefix(libraryURI, filePathPrefix) { - libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) - } +func setupWithStepLibSpecURL(libraryURI string, specURL string, route SteplibRoute) error { + specPath := GetStepSpecPath(route) - if err := command.CopyDir(libraryFilePath, pth, true); err != nil { - return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) - } + if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { + return err } - if isLocalLibrary { - if err := ReGenerateLibrarySpec(route); err != nil { - return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + specPath := GetStepSpecPath(route) + + if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { + return err } + + if err := downloadSpec(specPath, specURL); err != nil { + return err + } + + return nil + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) } if err := AddRoute(route); err != nil { return fmt.Errorf("failed to add routing, error: %s", err) } - isSuccess = true - return nil } @@ -284,118 +253,3 @@ func downloadSpec(filepath string, url string) error { return nil } - -// UpdateLibrary ... -func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, error) { - route, found := ReadRoute(libraryURI) - if !found { - if err := CleanupDanglingLibrary(libraryURI); err != nil { - log.Warnf("Failed to cleaning up library (%s), error: %s", libraryURI, err) - } - return models.StepCollectionModel{}, fmt.Errorf("no route found for library: %s", libraryURI) - } - - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) - - if isLocalLibrary { - if err := CleanupRoute(route); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err) - } - - if err := SetupLibrary(libraryURI, log); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err) - } - } else { - pth := GetLibraryBaseDirPath(route) - if exists, err := pathutil.IsPathExists(pth); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to check if library (%s) directory (%s) exist, error: %s", libraryURI, pth, err) - } else if !exists { - return models.StepCollectionModel{}, fmt.Errorf("library (%s) not initialized", libraryURI) - } - - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - repo, err := git.New(pth) - if err != nil { - return err - } - cmd := repo.Pull() - //cmd.SetStdout(os.Stdout) - //cmd.SetStderr(os.Stderr) - return cmd.Run() - }); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) - } - - if err := ReGenerateLibrarySpec(route); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) - } - } - - return ReadStepSpec(libraryURI) -} - -// UpdateLibrary2 ... -func UpdateLibrary2(libraryURI string, log Logger) (models.StepCollectionModel, error) { - route, found := ReadRoute(libraryURI) - if !found { - if err := CleanupDanglingLibrary(libraryURI); err != nil { - log.Warnf("Failed to cleaning up library (%s), error: %s", libraryURI, err) - } - return models.StepCollectionModel{}, fmt.Errorf("no route found for library: %s", libraryURI) - } - - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) - - if isLocalLibrary { - if err := CleanupRoute(route); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err) - } - - if err := SetupLibrary(libraryURI, log); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err) - } - } else { - pth := GetLibraryBaseDirPath(route) - if exists, err := pathutil.IsPathExists(pth); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to check if library (%s) directory (%s) exist, error: %s", libraryURI, pth, err) - } else if !exists { - return models.StepCollectionModel{}, fmt.Errorf("library (%s) not initialized", libraryURI) - } - - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - cmd := command.New("git", "fetch", "--jobs=10", "--depth=1", "--no-tags", "origin", "refs/heads/master") - cmd.SetDir(pth) - log.Warnf("$ %s", cmd.PrintableCommandArgs()) - out, err := cmd.RunAndReturnTrimmedCombinedOutput() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) - } - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) - } - - cmd = command.New("git", "checkout", "FETCH_HEAD") - cmd.SetDir(pth) - log.Warnf("$ %s", cmd.PrintableCommandArgs()) - out, err = cmd.RunAndReturnTrimmedCombinedOutput() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) - } - return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) - } - - return nil - }); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) - } - - if err := ReGenerateLibrarySpec(route); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) - } - } - - return ReadStepSpec(libraryURI) -} From bf5008805264d60827e58dd0e1714c2b8f3d2013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krisztia=CC=81n=20Go=CC=88drei?= Date: Fri, 28 Jun 2024 16:14:58 +0200 Subject: [PATCH 3/4] Move update benchamrks to the library_test.go file --- _tests/integration/update_test.go | 140 ------------------------------ stepman/library.go | 14 +-- stepman/library_test.go | 113 ++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 147 deletions(-) create mode 100644 stepman/library_test.go diff --git a/_tests/integration/update_test.go b/_tests/integration/update_test.go index 0d2a8423..5dd3d187 100644 --- a/_tests/integration/update_test.go +++ b/_tests/integration/update_test.go @@ -1,15 +1,12 @@ package integration import ( - "errors" - "fmt" "os" "testing" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/command/git" "github.com/bitrise-io/go-utils/pathutil" - "github.com/bitrise-io/stepman/stepman" "github.com/stretchr/testify/require" ) @@ -53,140 +50,3 @@ func TestUpdate(t *testing.T) { require.NoError(t, err, out) } } - -func Test_StepLibSetup(t *testing.T) { - steplib := "https://github.com/bitrise-io/bitrise-steplib.git" - oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 - err := setupStepLib(steplib, oldCommit) - if err != nil { - t.Fatal(err) - } -} - -func Test_StepLibUpdate(t *testing.T) { - steplib := "https://github.com/bitrise-io/bitrise-steplib.git" - oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 - err := setupStepLib(steplib, oldCommit) - if err != nil { - t.Fatal(err) - } - - if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { - t.Fatal(err) - } -} - -func Test_StepLibUpdate2(t *testing.T) { - steplib := "https://github.com/bitrise-io/bitrise-steplib.git" - oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 - err := setupStepLib(steplib, oldCommit) - if err != nil { - t.Fatal(err) - } - - if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { - t.Fatal(err) - } -} - -func BenchmarkStepLibUpdate(b *testing.B) { - oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 - - for _, mode := range []string{"old", "new"} { - b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { - for i := 0; i < b.N; i++ { - var steplib string - if mode == "old" { - steplib = "https://github.com/bitrise-io/bitrise-steplib" - - b.StopTimer() - err := setupStepLib(steplib, oldCommit) - if err != nil { - b.Fatal(err) - } - b.StartTimer() - } else { - steplib = "https://github.com/bitrise-io/bitrise-steplib.git" - } - - if _, err := stepman.UpdateLibrary(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } - } - }) - } -} - -func BenchmarkStepLibSetup(b *testing.B) { - for _, mode := range []string{"old", "new"} { - b.Run(fmt.Sprintf("Benchmarking stepman.UpdateLibrary=%s", mode), func(b *testing.B) { - for i := 0; i < b.N; i++ { - var steplib string - if mode == "old" { - steplib = "https://github.com/bitrise-io/bitrise-steplib" - } else { - steplib = "https://github.com/bitrise-io/bitrise-steplib.git" - } - - b.StopTimer() - route, found := stepman.ReadRoute(steplib) - if found { - if err := stepman.CleanupRoute(route); err != nil { - b.Fatal(err) - } - } - b.StartTimer() - - if err := stepman.SetupLibrary(steplib, NopeLogger{}); err != nil { - b.Fatal(err) - } - } - }) - } -} - -func setupStepLib(uri, commit string) error { - route, found := stepman.ReadRoute(uri) - if found { - if err := stepman.CleanupRoute(route); err != nil { - return err - } - } - - if err := stepman.SetupLibrary(uri, NopeLogger{}); err != nil { - return err - } - - route, found = stepman.ReadRoute(uri) - if !found { - return errors.New("no rout found") - } - - pth := stepman.GetLibraryBaseDirPath(route) - cmd := command.New("git", "reset", "--hard", commit) - cmd.SetDir(pth) - if err := cmd.Run(); err != nil { - return err - } - - return nil -} - -type NopeLogger struct { -} - -func (l NopeLogger) Warnf(format string, v ...interface{}) { - return -} - -func (l NopeLogger) Debugf(format string, v ...interface{}) { - return -} - -func (l NopeLogger) Errorf(format string, v ...interface{}) { - return -} - -func (l NopeLogger) Infof(format string, v ...interface{}) { - return -} diff --git a/stepman/library.go b/stepman/library.go index 6ac85e31..ca439034 100644 --- a/stepman/library.go +++ b/stepman/library.go @@ -12,14 +12,14 @@ import ( "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/command/git" "github.com/bitrise-io/go-utils/retry" - "github.com/bitrise-io/stepman/models" ) -const filePathPrefix = "file://" - -const defaultStepLib = "https://github.com/bitrise-io/bitrise-steplib.git" -const DefaultStepLibSpecJSONURL = "http://bitrise-steplib-collection.s3.amazonaws.com/spec.json" +const ( + filePathPrefix = "file://" + defaultStepLib = "https://github.com/bitrise-io/bitrise-steplib.git" + defaultStepLibSpecJSONURL = "http://bitrise-steplib-collection.s3.amazonaws.com/spec.json" +) // Logger ... type Logger interface { @@ -68,7 +68,7 @@ func SetupLibrary(libraryURI string, log Logger) error { return nil } - if err := setupWithStepLibSpecURL(libraryURI, DefaultStepLibSpecJSONURL, route); err != nil { + if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil { if err := setupWithStepLibRepo(libraryURI, route); err != nil { if err := CleanupRoute(route); err != nil { log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) @@ -113,7 +113,7 @@ func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, e return ReadStepSpec(libraryURI) } - if err := setupWithStepLibSpecURL(libraryURI, DefaultStepLibSpecJSONURL, route); err != nil { + if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil { if err := updateWithStepLibRepo(libraryURI, route); err != nil { return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err) } diff --git a/stepman/library_test.go b/stepman/library_test.go new file mode 100644 index 00000000..a04c9068 --- /dev/null +++ b/stepman/library_test.go @@ -0,0 +1,113 @@ +package stepman + +import ( + "errors" + "fmt" + "testing" + + "github.com/bitrise-io/go-utils/command" +) + +func BenchmarkStepLibSetup(b *testing.B) { + for _, mode := range []string{"repo-based", "spec-json-based"} { + b.Run(fmt.Sprintf("Benchmarking UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + var steplib string + if mode == "repo-based" { + // spec.json based update is only supported for the default StepLib URI ( "https://github.com/bitrise-io/bitrise-steplib.git") + steplib = "https://github.com/bitrise-io/bitrise-steplib" + } else { + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + + b.StopTimer() + route, found := ReadRoute(steplib) + if found { + if err := CleanupRoute(route); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + + if err := SetupLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkStepLibUpdate(b *testing.B) { + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + + for _, mode := range []string{"repo-based", "spec-json-based"} { + b.Run(fmt.Sprintf("Benchmarking UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + var steplib string + if mode == "repo-based" { + // spec.json based update is only supported for the default StepLib URI ( "https://github.com/bitrise-io/bitrise-steplib.git") + steplib = "https://github.com/bitrise-io/bitrise-steplib" + + b.StopTimer() + err := setupStepLib(steplib, oldCommit) + if err != nil { + b.Fatal(err) + } + b.StartTimer() + } else { + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + + if _, err := UpdateLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func setupStepLib(uri, commit string) error { + route, found := ReadRoute(uri) + if found { + if err := CleanupRoute(route); err != nil { + return err + } + } + + if err := SetupLibrary(uri, NopeLogger{}); err != nil { + return err + } + + route, found = ReadRoute(uri) + if !found { + return errors.New("no rout found") + } + + pth := GetLibraryBaseDirPath(route) + cmd := command.New("git", "reset", "--hard", commit) + cmd.SetDir(pth) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +type NopeLogger struct { +} + +func (l NopeLogger) Warnf(format string, v ...interface{}) { + return +} + +func (l NopeLogger) Debugf(format string, v ...interface{}) { + return +} + +func (l NopeLogger) Errorf(format string, v ...interface{}) { + return +} + +func (l NopeLogger) Infof(format string, v ...interface{}) { + return +} From 150fa9dda9a36cce2e0e6a37c2f5831a13e5b6a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krisztia=CC=81n=20Go=CC=88drei?= Date: Fri, 28 Jun 2024 16:18:40 +0200 Subject: [PATCH 4/4] Fix lint issue --- stepman/library_test.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/stepman/library_test.go b/stepman/library_test.go index a04c9068..81eb3cf2 100644 --- a/stepman/library_test.go +++ b/stepman/library_test.go @@ -96,18 +96,10 @@ func setupStepLib(uri, commit string) error { type NopeLogger struct { } -func (l NopeLogger) Warnf(format string, v ...interface{}) { - return -} +func (l NopeLogger) Warnf(format string, v ...interface{}) {} -func (l NopeLogger) Debugf(format string, v ...interface{}) { - return -} +func (l NopeLogger) Debugf(format string, v ...interface{}) {} -func (l NopeLogger) Errorf(format string, v ...interface{}) { - return -} +func (l NopeLogger) Errorf(format string, v ...interface{}) {} -func (l NopeLogger) Infof(format string, v ...interface{}) { - return -} +func (l NopeLogger) Infof(format string, v ...interface{}) {}