Merge pull request #199 from go-atomci/feat-#198-checkout-split

Feat #198 checkout split

constant/const.go

@@ -72,5 +72,6 @@ var ScmIntegratetypes = []string{SCMGitlab, SCMGithub, SCMGitea, SCMGitee, SCMGo
 
 const (
 	DefaultContainerName    = "jnlp"
+	CheckoutContainerName   = "checkout"
 	BuildImageContainerName = "kaniko"
 )

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/drone/go-scm v1.20.0
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-atomci/workflow v1.0.2
+	github.com/go-atomci/workflow v1.0.3-0.20230630103220-4f8f444edbaa
 	github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gorilla/websocket v1.4.2

go.sum

@@ -189,6 +189,14 @@ github.com/go-asn1-ber/asn1-ber v1.5.0 h1:/S4hO/AO6tLMlPX0oftGSOcdGJJN/MuYzfgWRM
 github.com/go-asn1-ber/asn1-ber v1.5.0/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-atomci/workflow v1.0.2 h1:IGcSE5F36hWDP8Xx1zEKcwMGu7fKwWKy9wJBCIfQ4y8=
 github.com/go-atomci/workflow v1.0.2/go.mod h1:U396Nlydy0L3cVrlkdqp2DLn8e4DtwN+wJ+nmWIaS0s=
+github.com/go-atomci/workflow v1.0.3-0.20230630031545-beb50c2eaffa h1:jjzwKJwAfuHDgy0komZs8kYaJ5KLS0tzVw6mxu9XVuY=
+github.com/go-atomci/workflow v1.0.3-0.20230630031545-beb50c2eaffa/go.mod h1:U396Nlydy0L3cVrlkdqp2DLn8e4DtwN+wJ+nmWIaS0s=
+github.com/go-atomci/workflow v1.0.3-0.20230630033439-2e6002705112 h1:aC6igjAo5yAoMXS4aNi2zRAX2BMf37njPd+8RQN7et8=
+github.com/go-atomci/workflow v1.0.3-0.20230630033439-2e6002705112/go.mod h1:U396Nlydy0L3cVrlkdqp2DLn8e4DtwN+wJ+nmWIaS0s=
+github.com/go-atomci/workflow v1.0.3-0.20230630072558-4261cca14b22 h1:SQwN89DSZT9u/t2nkA9LAAWbZwd2ULSMnApm4rj9poQ=
+github.com/go-atomci/workflow v1.0.3-0.20230630072558-4261cca14b22/go.mod h1:U396Nlydy0L3cVrlkdqp2DLn8e4DtwN+wJ+nmWIaS0s=
+github.com/go-atomci/workflow v1.0.3-0.20230630103220-4f8f444edbaa h1:k0aY1K1c62QtHB+AKnHc9WvPJBBJRHWxQFQ8wvtLNiE=
+github.com/go-atomci/workflow v1.0.3-0.20230630103220-4f8f444edbaa/go.mod h1:U396Nlydy0L3cVrlkdqp2DLn8e4DtwN+wJ+nmWIaS0s=
 github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=

internal/api/integrate.go

@@ -203,7 +203,7 @@ func (p *IntegrateController) DeleteIntegrateSetting() {
 // GetCompileEnvs ..
 func (p *IntegrateController) GetCompileEnvs() {
 	pm := settings.NewSettingManager()
-	rsp, err := pm.GetCompileEnvs("")
+	rsp, err := pm.GetCompileEnvs()
 	if err != nil {
 		p.HandleInternalServerError(err.Error())
 		log.Log.Error("Get compile envs occur error: %s", err.Error())

internal/core/pipelinemgr/uitls.go

@@ -273,6 +273,12 @@ func (pm *PipelineManager) CreateBuildJob(creator string, projectID, publishID i
 		log.Log.Error("when create build job, get sys default %v compile env error: %s", constant.DefaultContainerName, err.Error())
 		return 0, "", err
 	}
+
+	jenkinsCheckoutTemplate, err := pm.getSysDefaultCompileEnv(constant.CheckoutContainerName)
+	if err != nil {
+		log.Log.Error("when create build job, get sys default checkout compile env  error: %s", err.Error())
+		return 0, "", err
+	}
 	jenkinsKanikoTemplate, err := pm.getSysDefaultCompileEnv(constant.BuildImageContainerName)
 	if err != nil {
 		log.Log.Error("when create build job, get sys default kaniko compile env  error: %s", err.Error())
@@ -282,6 +288,7 @@ func (pm *PipelineManager) CreateBuildJob(creator string, projectID, publishID i
 	// default container template
 	containerTemplates := []jenkins.ContainerEnv{
 		jenkinsJNLPTemplate,
+		jenkinsCheckoutTemplate,
 		jenkinsKanikoTemplate,
 	}
 	// TaskTmplItem.SubTask
@@ -410,8 +417,9 @@ func (pm *PipelineManager) CreateBuildJob(creator string, projectID, publishID i
 		ContainerTemplates: containerTemplates,
 		Stages:             pipelineStagesStr,
 		CommonContext: jenkins.CommonContext{
-			Namespace: CIInfo[4],
-			PodName:   strings.ReplaceAll(jobName, "_", "-"),
+			Namespace:        CIInfo[4],
+			PodName:          strings.ReplaceAll(jobName, "_", "-"),
+			DefaultContainer: constant.DefaultContainerName,
 		},
 		CallBack: jenkins.CallbackRequest{
 			Token: adminToken,
@@ -542,9 +550,16 @@ func (pm *PipelineManager) CreateDeployJob(creator string, projectID, publishID
 		return 0, "", err
 	}
 
+	jenkinsCheckoutTemplate, err := pm.getSysDefaultCompileEnv(constant.CheckoutContainerName)
+	if err != nil {
+		log.Log.Error("when create deploy job, get sys default checkout compile env  error: %s", err.Error())
+		return 0, "", err
+	}
+
 	// default container template
 	containerTemplates := []jenkins.ContainerEnv{
 		jenkinsJNLPTemplate,
+		jenkinsCheckoutTemplate,
 	}
 
 	flowProcessor := &jenkins.DeployContext{
@@ -557,8 +572,9 @@ func (pm *PipelineManager) CreateDeployJob(creator string, projectID, publishID
 			Body:  callBackRequestBody,
 		},
 		CommonContext: jenkins.CommonContext{
-			Namespace: CIInfo[4],
-			PodName:   strings.ReplaceAll(jobName, "_", "-"),
+			Namespace:        CIInfo[4],
+			PodName:          strings.ReplaceAll(jobName, "_", "-"),
+			DefaultContainer: constant.DefaultContainerName,
 		},
 	}
 
@@ -1020,10 +1036,9 @@ func (pm *PipelineManager) generateAutoDeployStep(publishID int64) (*DeployStepR
 
 /*  auto Trigger part end */
 
-func (pm *PipelineManager) generateBaseInfo(projectID, stageID, publishJobID int64) (string, string) {
-	scriptsDir := "/home/admin/scripts_dev"
+func (pm *PipelineManager) generateBaseInfo(projectID, stageID, publishJobID int64) string {
 	scriptBaseInfo := fmt.Sprintf(" --project-id %d --stage-id %d --publish-job-id %d ", projectID, stageID, publishJobID)
-	return scriptsDir, scriptBaseInfo
+	return scriptBaseInfo
 }
 func (pm *PipelineManager) generateAppPth(stageID, projectID int64, workSpace string, appArgs *RunBuildAllParms) string {
 	appPath := strings.Join([]string{workSpace, strconv.Itoa(int(projectID)), strconv.Itoa(int(stageID)), appArgs.Name, appArgs.Branch, appArgs.BuildPath}, "/")
@@ -1034,7 +1049,7 @@ func (pm *PipelineManager) generateAppPth(stageID, projectID int64, workSpace st
 func (pm *PipelineManager) renderAppCheckoutItemsForBuild(projectID, stageID, publishJobID int64, allParms []*RunBuildAllParms) ([]jenkins.StepItem, error) {
 	appCheckoutItems := []jenkins.StepItem{}
 
-	scriptsDir, buildBaseInfo := pm.generateBaseInfo(projectID, stageID, publishJobID)
+	buildBaseInfo := pm.generateBaseInfo(projectID, stageID, publishJobID)
 	for _, app := range allParms {
 		// TODO: if GitAPP type is not app, how to deal with this, skip ??
 		item := jenkins.StepItem{}
@@ -1043,8 +1058,9 @@ func (pm *PipelineManager) renderAppCheckoutItemsForBuild(projectID, stageID, pu
 		// TODO: app build vcsType use git
 		appInfoStr := fmt.Sprintf(" --scm-app-id %d --app-name %s --app-language %s --branch-url %s --vcs-type %s --build-path %s ", app.ProjectAppID, app.Name, app.Language, app.Path, "git", app.BuildPath)
 		appParms := fmt.Sprintf(" --branch-name %s ", app.Branch)
-		Command := fmt.Sprintf("sh 'python3 %s/app_checkout.py %s %s %s'", scriptsDir, buildBaseInfo, appInfoStr, appParms)
+		Command := fmt.Sprintf("sh 'checkout.py %s %s %s'", buildBaseInfo, appInfoStr, appParms)
 		item.Command = Command
+		item.ContainerName = constant.CheckoutContainerName
 		appCheckoutItems = append(appCheckoutItems, item)
 	}
 
@@ -1116,6 +1132,7 @@ func (pm *PipelineManager) renderAppImageitemsForBuild(projectID, publishID, sta
 		}
 		Command := fmt.Sprintf("sh \"cd %v; export DOCKER_CONFIG=$DOCKER_CONFIG; /kaniko/executor -f %v -c ./  -d %v %s \"", appPath, dockerfile, imageURL, insecure)
 		item.Command = Command
+		item.ContainerName = constant.BuildImageContainerName
 		appImageItems = append(appImageItems, item)
 	}
 
@@ -1125,7 +1142,7 @@ func (pm *PipelineManager) renderAppImageitemsForBuild(projectID, publishID, sta
 // Rendering parameters for healthcheck command
 func (pm *PipelineManager) renderHealthCheckCommand(projectID, stageID, publishJobID int64, allParms []*AppParamsForHealthCheck, stageJSON *PipelineStageStruct) ([]*jenkins.StepItem, error) {
 	healthCheckItems := []*jenkins.StepItem{}
-	scriptsDir, buildBaseInfo := pm.generateBaseInfo(projectID, stageID, publishJobID)
+	buildBaseInfo := pm.generateBaseInfo(projectID, stageID, publishJobID)
 
 	envStage, err := pm.modelProject.GetProjectEnvByID(stageJSON.StageID)
 	if err != nil {
@@ -1140,6 +1157,8 @@ func (pm *PipelineManager) renderHealthCheckCommand(projectID, stageID, publishJ
 		}
 		item := &jenkins.StepItem{}
 		item.Name = app.Name
+		// TODO: refactor(WIP) healthcheck stage, use checkout container name tmp.
+		item.ContainerName = constant.CheckoutContainerName
 
 		appArrange, err := pm.appHandler.GetRealArrange(app.ID, stageID)
 		if err != nil {
@@ -1165,7 +1184,7 @@ func (pm *PipelineManager) renderHealthCheckCommand(projectID, stageID, publishJ
 		for _, appRes := range appResItems {
 			svcName := appRes.Name
 			svcInfo := fmt.Sprintf(" --cluster %s --namespace %s --app-name %s --service-name %s", settingKubernetesItem.Name, envStage.Namespace, app.Name, svcName)
-			item.Command = fmt.Sprintf("sh 'python3 %s/healthcheck.py %s %s'", scriptsDir, buildBaseInfo, svcInfo)
+			item.Command = fmt.Sprintf("sh 'healthcheck.py %s %s'", buildBaseInfo, svcInfo)
 			healthCheckItems = append(healthCheckItems, item)
 		}
 	}

internal/core/settings/compile_env.go

@@ -34,8 +34,8 @@ type CompileEnvReq struct {
 }
 
 // GetCompileEnvs ..
-func (pm *SettingManager) GetCompileEnvs(integrateType string) ([]*models.CompileEnv, error) {
-	items, err := pm.model.GetCompileEnvs(integrateType)
+func (pm *SettingManager) GetCompileEnvs() ([]*models.CompileEnv, error) {
+	items, err := pm.model.GetCompileEnvs()
 	if err != nil {
 		log.Log.Error("get interate settings error: %s", err.Error())
 		return nil, err

internal/dao/integrate_settings.go

@@ -146,12 +146,10 @@ func (model *SysSettingModel) GetCompileEnvByName(compileEnvItem string) (*model
 }
 
 // GetCompileEnvs ...
-func (model *SysSettingModel) GetCompileEnvs(integrateType string) ([]*models.CompileEnv, error) {
+func (model *SysSettingModel) GetCompileEnvs() ([]*models.CompileEnv, error) {
 	integrateSettings := []*models.CompileEnv{}
 	qs := model.ormer.QueryTable(model.CompileEnvTableName).Filter("deleted", false)
-	if integrateType != "" {
-		qs = qs.Filter("type", integrateType)
-	}
+
 	_, err := qs.All(&integrateSettings)
 	if err != nil {
 		return nil, err

internal/migrations/migration20220415.go

@@ -39,8 +39,33 @@ func (m Migration20220415) Upgrade(ormer orm.Ormer) error {
 	// init component
 	_ = initComponent()
 
+	compileEnvs := []settings.CompileEnvReq{
+		{
+			Name:        "jnlp",
+			Image:       "colynn/jenkins-jnlp-agent:latest",
+			Description: "",
+		},
+		{
+			Name:    "kaniko",
+			Image:   "colynn/kaniko-executor:debug",
+			Command: "/bin/sh -c",
+			Args:    "cat",
+		},
+		{
+			Name:        "node",
+			Image:       "node:12.12-alpine",
+			Description: "nodejs编译环境",
+		},
+		{
+			Name:    "maven",
+			Image:   "maven:3.8.2-openjdk-8",
+			Command: "/bin/sh -c",
+			Args:    "cat",
+		},
+	}
+
 	// init compile envs
-	_ = initCompileEnvs()
+	_ = initCompileEnvs(compileEnvs)
 
 	// init task tmpls
 	_ = initTaskTemplates()
@@ -93,32 +118,7 @@ func initComponent() error {
 	return nil
 }
 
-var compileEnvs = []settings.CompileEnvReq{
-	{
-		Name:        "jnlp",
-		Image:       "colynn/jenkins-jnlp-agent:latest",
-		Description: "",
-	},
-	{
-		Name:    "kaniko",
-		Image:   "colynn/kaniko-executor:debug",
-		Command: "/bin/sh -c",
-		Args:    "cat",
-	},
-	{
-		Name:        "node",
-		Image:       "node:12.12-alpine",
-		Description: "nodejs编译环境",
-	},
-	{
-		Name:    "maven",
-		Image:   "maven:3.8.2-openjdk-8",
-		Command: "/bin/sh -c",
-		Args:    "cat",
-	},
-}
-
-func initCompileEnvs() error {
+func initCompileEnvs(compileEnvs []settings.CompileEnvReq) error {
 	settingModel := dao.NewSysSettingModel()
 	for _, item := range compileEnvs {
 		_, err := settingModel.GetCompileEnvByName(item.Name)

internal/migrations/migration20230703.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2021 The AtomCI Group Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrations
+
+import (
+	"time"
+
+	"github.com/astaxie/beego/orm"
+	"github.com/go-atomci/atomci/internal/core/settings"
+)
+
+type Migration20230703 struct {
+}
+
+func (m Migration20230703) GetCreateAt() time.Time {
+	return time.Date(2023, 7, 03, 0, 0, 0, 0, time.Local)
+}
+
+func (m Migration20230703) Upgrade(ormer orm.Ormer) error {
+
+	addedCompileEnvs := []settings.CompileEnvReq{
+		{
+			Name:        "checkout",
+			Image:       "colynn/checkout:latest",
+			Description: "代码检出",
+		},
+	}
+
+	// init compile envs
+	_ = initCompileEnvs(addedCompileEnvs)
+
+	// update origin jnlp image
+	return updateCompileEnvs(ormer)
+}
+
+func updateCompileEnvs(ormer orm.Ormer) error {
+	_, err := ormer.Raw("UPDATE `sys_compile_env` SET `image`='jenkins/inbound-agent:latest',`description`='默认jenkins jnlp agent' WHERE `name`='jnlp';").Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/migrations/migrations.go

@@ -56,6 +56,7 @@ func initMigration() {
 		new(Migration20220324),
 		new(Migration20220414),
 		new(Migration20220415),
+		new(Migration20230703),
 	}
 
 	migrateInTx(migrationTypes)

web/src/views/setting/components/CompileEnvCreate.vue

@@ -102,14 +102,15 @@ export default {
           args: item.args || '',
           description: item.description || '',
         };
-        if (this.form.name == 'jnlp' || this.form.name == 'kaniko') {
+        if (this.form.name == 'jnlp' || this.form.name == 'checkout' || this.form.name == 'kaniko') {
           this.systemReserved = true
         } else {
           this.systemReserved = false
         }
         this.rowId = item.id;
       } else {
         this.title = '新增';
+        this.systemReserved = false;
         this.form = {
           name: '',
           image: '',