From a2cab92d9ac4d154241d773e36f2760c6fae134d Mon Sep 17 00:00:00 2001 From: lwsanty Date: Tue, 30 Jul 2019 18:19:12 +0300 Subject: [PATCH] add Jenkinsfile to trigger auto-update all drivers repositories The logic is triggered on curl and runs updater utility Signed-off-by: lwsanty --- Jenkinsfile | 62 ++++++++++++++++++ cmd/bblfsh-drivers-updater/update.go | 28 +++++---- cmd/bblfsh-drivers-updater/utils/utils.go | 77 ++++++++++++----------- 3 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..5358cf02 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,62 @@ +pipeline { + agent { + kubernetes { + label 'sdk-drivers-updater' + defaultContainer 'sdk-drivers-updater' + yaml """ +spec: + nodeSelector: + srcd.host/type: jenkins-worker + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: jenkins + operator: In + values: + - slave + topologyKey: kubernetes.io/hostname + containers: + - name: sdk-drivers-updater + image: bblfsh/performance:latest + imagePullPolicy: Always + securityContext: + privileged: true + command: + - dockerd + tty: true +""" + } + } + triggers { + GenericTrigger( + genericVariables: [ + [key: 'target', value: '$.target'], + [key: 'sdk_version', value: '$.sdk_version'], + [key: 'branch', value: '$.branch'], + [key: 'commit_msg', value: '$.commit_msg'], + [key: 'script', value: '$.script'] + ], + token: 'update', + causeString: 'Triggered on $target', + + printContributedVariables: true, + printPostContent: true, + + regexpFilterText: '$target', + regexpFilterExpression: 'master' + ) + } + stages { + stage('Run updater') { + when { branch 'master' } + steps { + withCredentials([usernamePassword(credentialsId: '87b3cad8-8b12-4e91-8f47-33f3d7d45620', passwordVariable: 'token', usernameVariable: 'user')]) { + sh 'echo ${script} > /etc/script.sh ; chmod +x /etc/script.sh' + sh 'GITHUB_TOKEN=${token} go run cmd/bblfsh-drivers-updater/update.go --script="/etc/script.sh" --sdk-version="${sdk_version}" --branch="${branch}" --commit-msg="${commit_msg}" --dockerfile=true' + } + } + } + } +} diff --git a/cmd/bblfsh-drivers-updater/update.go b/cmd/bblfsh-drivers-updater/update.go index fcb032fa..6efcd688 100644 --- a/cmd/bblfsh-drivers-updater/update.go +++ b/cmd/bblfsh-drivers-updater/update.go @@ -5,6 +5,7 @@ import ( "flag" "io/ioutil" "os" + "strings" "github.com/bblfsh/sdk/v3/cmd/bblfsh-drivers-updater/utils" "github.com/bblfsh/sdk/v3/driver/manifest/discovery" @@ -22,6 +23,7 @@ func main() { scriptPathPtr := flag.String("script", "", "path to the script that will be executed") commitMsgPtr := flag.String("commit-msg", commitMsg, "commit message of the update") dockerfilePtr := flag.Bool("dockerfile", false, "use dockerfile to create a branch") + explicitCredsPtr := flag.Bool("explicit-creds", false, "use explicit credentials inside dockerfile") dryRunPtr := flag.Bool("dry-run", false, "dry run") flag.Parse() @@ -34,18 +36,19 @@ func main() { var scriptText string scriptData, err := ioutil.ReadFile(*scriptPathPtr) + switch { case err != nil && !os.IsNotExist(err): log.Errorf(err, "error") os.Exit(1) case os.IsNotExist(err): log.Infof("script %v does not exist", *scriptPathPtr) - if *SDKVersionPtr == "" { + if strings.TrimSpace(*SDKVersionPtr) == "" { log.Infof("both script and SDK version not found, exiting") os.Exit(0) } fallthrough - case string(scriptData) == "" && *SDKVersionPtr == "": + case strings.TrimSpace(string(scriptData)) == "" && strings.TrimSpace(*SDKVersionPtr) == "": log.Infof("script and SDK version are empty, nothing to do here") os.Exit(0) default: @@ -71,26 +74,29 @@ func main() { case tmpSDKVersion == d.SDKVersion: log.Infof("driver %v: sdk %v is already installed", d.Language, tmpSDKVersion) tmpSDKVersion = "" - if scriptText == "" { + if strings.TrimSpace(scriptText) == "" { log.Infof("skipping driver %v: script is empty and version update is not required", d.Language) continue } fallthrough default: - err := utils.PrepareBranch(d, &utils.UpdateOptions{ - Branch: *branchPtr, - SDKVersion: tmpSDKVersion, - Script: scriptText, - CommitMsg: *commitMsgPtr, - Dockerfile: *dockerfilePtr, - DryRun: *dryRunPtr, + githubToken := os.Getenv("GITHUB_TOKEN") + err := utils.PrepareBranch(d, githubToken, &utils.UpdateOptions{ + Branch: *branchPtr, + SDKVersion: tmpSDKVersion, + Script: scriptText, + CommitMsg: *commitMsgPtr, + Dockerfile: *dockerfilePtr, + ExplicitCredentials: *explicitCredsPtr, + DryRun: *dryRunPtr, }) if utils.ErrNothingToCommit.Is(err) { log.Warningf("skipping driver %s: nothing to change", d.Language) continue } + handleErr(err) - handleErr(utils.PreparePR(d, *branchPtr, *commitMsgPtr, *dryRunPtr)) + handleErr(utils.PreparePR(d, githubToken, *branchPtr, *commitMsgPtr, *dryRunPtr)) } } } diff --git a/cmd/bblfsh-drivers-updater/utils/utils.go b/cmd/bblfsh-drivers-updater/utils/utils.go index 0846cb07..9eb5bfa5 100644 --- a/cmd/bblfsh-drivers-updater/utils/utils.go +++ b/cmd/bblfsh-drivers-updater/utils/utils.go @@ -21,8 +21,8 @@ const ( org = "bblfsh" tmpFolder = "/var/lib/tmp" - gitUser = "grzegorz-brzęczyszczykiewicz" - gitMail = "<>" + gitUser = "bblfsh-release-bot" + gitMail = "" errSpecialText = "nothing to commit" ) @@ -51,17 +51,19 @@ type pipeLineNode struct { // UpdateOptions represents git metadata for changes and ways of execution of update script type UpdateOptions struct { - Branch string - SDKVersion string - Script string - CommitMsg string - Dockerfile bool - DryRun bool + Branch string + SDKVersion string + Script string + CommitMsg string + Dockerfile bool + ExplicitCredentials bool + DryRun bool } -func newPipeLine(d discovery.Driver, o *UpdateOptions) *pipeLine { +func newPipeLine(d discovery.Driver, githubToken string, o *UpdateOptions) *pipeLine { url := d.RepositoryURL() - origin := getOrigin(url, o.Dockerfile) + processOptions(o) + origin := getOrigin(url, githubToken, o) tmpDir := filepath.Join(tmpFolder, d.Language) var nodes []pipeLineNode @@ -102,7 +104,7 @@ func newPipeLine(d discovery.Driver, o *UpdateOptions) *pipeLine { nodes = append(nodes, pipeLineNode{ logFormat: "set git user info", logArgs: []interface{}{}, - command: fmt.Sprintf("cd %s ; git config --global user.name %v ; git config --global user.email %v", shell.Quote(tmpDir), getEnv("GITHUB_NAME", gitUser), shell.Quote(getEnv("GITHUB_EMAIL", gitMail))), + command: fmt.Sprintf("cd %s ; git config --global user.name %v ; git config --global user.email %v", shell.Quote(tmpDir), gitUser, shell.Quote(gitMail)), }, pipeLineNode{ logFormat: "committing the changes", logArgs: []interface{}{}, @@ -151,7 +153,7 @@ ARG GITHUB_TOKEN return path, nil } -func (p *pipeLine) exec() error { +func (p *pipeLine) exec(githubToken string) error { if p.dockerfile { dockerPath, err := p.createDockerfile() if err != nil { @@ -161,7 +163,7 @@ func (p *pipeLine) exec() error { return nil } command := fmt.Sprintf("docker build --build-arg GITHUB_TOKEN=%v -t %v-driver-update %v", - os.Getenv("GITHUB_TOKEN"), p.driver.Language, filepath.Dir(dockerPath)) + githubToken, p.driver.Language, filepath.Dir(dockerPath)) if err := ExecCmd(command); err != nil { err = errFailedToPrepareBranch.New(p.driver.Language, err) if strings.Contains(err.Error(), errSpecialText) { @@ -186,7 +188,9 @@ func (p *pipeLine) exec() error { } func (p *pipeLine) close() { - os.RemoveAll(p.tmpDir) + if err := os.RemoveAll(p.tmpDir); err != nil { + log.Warningf("could not remove directory %v: %v", p.tmpDir, err) + } } // PrepareBranch does the next steps: @@ -195,10 +199,10 @@ func (p *pipeLine) close() { // 3) executes custom script if it's not empty // 4) updates SDK version if it's not empty // 5) commits and pushes changes to the previously created branch -func PrepareBranch(d discovery.Driver, o *UpdateOptions) error { - p := newPipeLine(d, o) +func PrepareBranch(d discovery.Driver, githubToken string, o *UpdateOptions) error { + p := newPipeLine(d, githubToken, o) defer p.close() - if err := p.exec(); err != nil { + if err := p.exec(githubToken); err != nil { return err } @@ -207,23 +211,27 @@ func PrepareBranch(d discovery.Driver, o *UpdateOptions) error { } // PreparePR creates pull request for a given driver's branch -func PreparePR(d discovery.Driver, branch, commitMsg string, dryRun bool) error { +func PreparePR(d discovery.Driver, githubToken, branch, commitMsg string, dryRun bool) error { ctx := context.Background() client := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + &oauth2.Token{AccessToken: githubToken}, ))) log.Infof("Preparing pr %v -> master", branch) - if dryRun { - return nil - } - pr, _, err := client.PullRequests.Create(ctx, org, d.Language+"-driver", &github.NewPullRequest{ + newPR := &github.NewPullRequest{ Title: &branch, Head: &branch, Base: strPtr("master"), Body: strPtr(commitMsg), MaintainerCanModify: newTrue(), - }) + } + if dryRun { + log.Infof("pr to be created:\ntitle: %v\nhead: %v\nbase: %v\nbody: %v\nmaintainers can modify: %v", + newPR.GetTitle(), newPR.GetHead(), newPR.GetBase(), newPR.GetBody(), newPR.GetMaintainerCanModify()) + return nil + } + + pr, _, err := client.PullRequests.Create(ctx, org, d.Language+"-driver", newPR) if err != nil { return errFailedToPreparePR.New(d.Language, branch, err) } @@ -247,20 +255,19 @@ func ExecCmd(command string) error { return nil } -func getOrigin(url string, isDockerfile bool) string { - token := "${GITHUB_TOKEN}" - if !isDockerfile { - token = os.Getenv("GITHUB_TOKEN") +func getOrigin(url string, githubToken string, o *UpdateOptions) string { + token := githubToken + if o.Dockerfile && !o.ExplicitCredentials { + token = "${GITHUB_TOKEN}" } - return strings.Replace(url, "github.com", getEnv("GITHUB_NAME", gitUser)+":"+token+"@github.com", -1) + ".git" + return strings.Replace(url, "github.com", gitUser+":"+token+"@github.com", -1) } -func getEnv(key, fallback string) string { - value := os.Getenv(key) - if len(value) == 0 { - return fallback - } - return value +func processOptions(o *UpdateOptions) { + o.Branch = strings.TrimSpace(o.Branch) + o.CommitMsg = strings.TrimSpace(o.CommitMsg) + o.SDKVersion = strings.TrimSpace(o.SDKVersion) + o.Script = strings.TrimSpace(o.Script) } func strPtr(s string) *string {