Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stepman setup improvements #331

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Stepman setup improvements #331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a08e356
stepman setup and update step lib improvements and benchmark
godrei Jun 26, 2024
8f9f529
Setup StepLib with remote spec.json
godrei Jun 28, 2024
a1287ae
Merge branch 'master' into stepman-setup-improvements
godrei Jun 28, 2024
bf50088
Move update benchamrks to the library_test.go file
godrei Jun 28, 2024
5f61f29
Merge branch 'stepman-setup-improvements' of github.com:bitrise-io/st…
godrei Jun 28, 2024
150fa9d
Fix lint issue
godrei Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 177 additions & 59 deletions stepman/library.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,24 @@ package stepman

import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"

"github.com/bitrise-io/go-utils/command"
"github.com/bitrise-io/go-utils/command/git"
"github.com/bitrise-io/go-utils/pathutil"
"github.com/bitrise-io/go-utils/retry"
"github.com/bitrise-io/stepman/models"
)

const filePathPrefix = "file://"
const (
filePathPrefix = "file://"
defaultStepLib = "https://github.com/bitrise-io/bitrise-steplib.git"
defaultStepLibSpecJSONURL = "http://bitrise-steplib-collection.s3.amazonaws.com/spec.json"
)

// Logger ...
type Logger interface {
Expand All @@ -23,7 +29,6 @@ type Logger interface {
Infof(format string, v ...interface{})
}

// SetupLibrary ...
func SetupLibrary(libraryURI string, log Logger) error {
if exist, err := RootExistForLibrary(libraryURI); err != nil {
return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err)
Expand All @@ -37,60 +42,46 @@ func SetupLibrary(libraryURI string, log Logger) error {
FolderAlias: alias,
}

// Cleanup
isSuccess := false
defer func() {
if !isSuccess {
isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix)
if isLocalLibrary {
if err := setupWithLocalStepLib(libraryURI, route); err != nil {
// TODO: is cleanup needed?
if err := CleanupRoute(route); err != nil {
log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err)
}

return err
}
}()

// Setup
isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix)
return nil
}

pth := GetLibraryBaseDirPath(route)
if !isLocalLibrary {
if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
repo, err := git.New(pth)
if err != nil {
return err
isCustomStepLibrary := libraryURI != defaultStepLib
if isCustomStepLibrary {
if err := setupWithStepLibRepo(libraryURI, route); err != nil {
if err := CleanupRoute(route); err != nil {
log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err)
}
return repo.Clone(libraryURI).Run()
}); err != nil {
return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err)
}
} else {
// Local spec path
if err := os.MkdirAll(pth, 0777); err != nil {
return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err)
}

libraryFilePath := libraryURI
if strings.HasPrefix(libraryURI, filePathPrefix) {
libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix)
}

if err := command.CopyDir(libraryFilePath, pth, true); err != nil {
return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err)
return err
}
return nil
}

if err := ReGenerateLibrarySpec(route); err != nil {
return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err)
}
if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil {
if err := setupWithStepLibRepo(libraryURI, route); err != nil {
if err := CleanupRoute(route); err != nil {
log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err)
}

if err := AddRoute(route); err != nil {
return fmt.Errorf("failed to add routing, error: %s", err)
return err
}
return nil
}

isSuccess = true

return nil
}

// UpdateLibrary ...
func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, error) {
route, found := ReadRoute(libraryURI)
if !found {
Expand All @@ -101,37 +92,164 @@ func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, e
}

isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix)

if isLocalLibrary {
if err := CleanupRoute(route); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err)
}

if err := SetupLibrary(libraryURI, log); err != nil {
if err := setupWithLocalStepLib(libraryURI, route); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err)
}
} else {
pth := GetLibraryBaseDirPath(route)
if exists, err := pathutil.IsPathExists(pth); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to check if library (%s) directory (%s) exist, error: %s", libraryURI, pth, err)
} else if !exists {
return models.StepCollectionModel{}, fmt.Errorf("library (%s) not initialized", libraryURI)
}

if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
repo, err := git.New(pth)
if err != nil {
return err
}
return repo.Pull().Run()
}); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err)
return ReadStepSpec(libraryURI)
}

isCustomStepLibrary := libraryURI != defaultStepLib
if isCustomStepLibrary {
if err := updateWithStepLibRepo(libraryURI, route); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err)
}

if err := ReGenerateLibrarySpec(route); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err)
return ReadStepSpec(libraryURI)
}

if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil {
if err := updateWithStepLibRepo(libraryURI, route); err != nil {
return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err)
}

return ReadStepSpec(libraryURI)
}

return ReadStepSpec(libraryURI)
}

func setupWithLocalStepLib(libraryURI string, route SteplibRoute) error {
libraryBaseDir := GetLibraryBaseDirPath(route)

if err := os.MkdirAll(libraryBaseDir, 0777); err != nil {
return fmt.Errorf("failed to create library dir (%s), error: %s", libraryBaseDir, err)
}

libraryFilePath := libraryURI
if strings.HasPrefix(libraryFilePath, filePathPrefix) {
libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix)
}

if err := command.CopyDir(libraryFilePath, libraryBaseDir, true); err != nil {
return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, libraryBaseDir, err)
}

if err := ReGenerateLibrarySpec(route); err != nil {
return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err)
}

if err := AddRoute(route); err != nil {
return fmt.Errorf("failed to add routing, error: %s", err)
}
return nil
}

func setupWithStepLibRepo(libraryURI string, route SteplibRoute) error {
libraryBaseDir := GetLibraryBaseDirPath(route)

if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
repo, err := git.New(libraryBaseDir)
if err != nil {
return err
}
return repo.Clone(libraryURI).Run()
}); err != nil {
return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err)
}

if err := ReGenerateLibrarySpec(route); err != nil {
return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err)
}

if err := AddRoute(route); err != nil {
return fmt.Errorf("failed to add routing, error: %s", err)
}

return nil
}

func updateWithStepLibRepo(libraryURI string, route SteplibRoute) error {
pth := GetLibraryBaseDirPath(route)

if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
repo, err := git.New(pth)
if err != nil {
return err
}
cmd := repo.Pull()
return cmd.Run()
}); err != nil {
return fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err)
}

if err := ReGenerateLibrarySpec(route); err != nil {
return fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err)
}

return nil
}

func setupWithStepLibSpecURL(libraryURI string, specURL string, route SteplibRoute) error {
specPath := GetStepSpecPath(route)

if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil {
return err
}

if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error {
specPath := GetStepSpecPath(route)

if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil {
return err
}

if err := downloadSpec(specPath, specURL); err != nil {
return err
}

return nil
}); err != nil {
return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err)
}

if err := AddRoute(route); err != nil {
return fmt.Errorf("failed to add routing, error: %s", err)
}

return nil
}

func downloadSpec(filepath string, url string) error {
// Create the file
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()

// Get the data
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()

// Check server response
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad status: %s", resp.Status)
}

// Writer the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}

return nil
}
Loading