Skip to content

Commit

Permalink
Merge pull request #100 from walter-cd/refactor-engine
Browse files Browse the repository at this point in the history 
Refactoring pipeline and engine
  • Loading branch information
@ainoya
ainoya committed Mar 30, 2015
2 parents e681756 + 9a4ee53 commit 2e53e91
Show file tree
Hide file tree
Showing 7 changed files with 89 additions and 77 deletions.
14 changes: 7 additions & 7 deletions config/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ func getStageTypeModuleName(stageType string) string {
return strings.ToLower(stageType)
}

func Parse(configData *map[interface{}]interface{}) (*pipelines.Pipeline, error) {
func Parse(configData *map[interface{}]interface{}) (*pipelines.Resources, error) {
envs := NewEnvVariables()
return ParseWithSpecifiedEnvs(configData, envs)
}

// TODO: need refactoring
// TODO: make parser process a struct (for simplifying redundant functions and reducing the number of function parameters)
func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
envs *EnvVariables) (*pipelines.Pipeline, error) {
envs *EnvVariables) (*pipelines.Resources, error) {
// parse service block
serviceOps, ok := (*configData)["service"].(map[interface{}]interface{})
var repoService services.Service
Expand Down Expand Up @@ -93,9 +93,7 @@ func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
}

// parse pipeline block
var pipeline *pipelines.Pipeline = &pipelines.Pipeline{
Reporter: messenger, RepoService: repoService, Cleanup: cleanup,
}
var pipeline *pipelines.Pipeline = &pipelines.Pipeline{}

pipelineData, ok := (*configData)["pipeline"].([]interface{})
if ok == false {
Expand All @@ -108,7 +106,9 @@ func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
pipeline.AddStage(stageItem.Value.(stages.Stage))
}
return pipeline, nil
var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

return resources, nil
}

func mapMessenger(messengerMap map[interface{}]interface{}, envs *EnvVariables) (messengers.Messenger, error) {
Expand Down
22 changes: 11 additions & 11 deletions config/parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (

func TestParseFromFile(t *testing.T) {
configData := ReadConfig("../tests/fixtures/pipeline.yml")
pipeline, err := Parse(configData)
actual := pipeline.Stages.Front().Value.(*stages.CommandStage).Command
resources, err := Parse(configData)
actual := resources.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
assert.Nil(t, err)
}
Expand Down Expand Up @@ -60,10 +60,10 @@ func TestParseConfWithChildren(t *testing.T) {
type: command
command: echo "hello, world, command_stage_3_group_1"`))
result, err := Parse(configData)
assert.Equal(t, 1, result.Size())
assert.Equal(t, 1, result.Pipeline.Size())
assert.Nil(t, err)

childStages := result.Stages.Front().Value.(stages.Stage).GetChildStages()
childStages := result.Pipeline.Stages.Front().Value.(stages.Stage).GetChildStages()
assert.Equal(t, 2, childStages.Len())
}

Expand All @@ -73,7 +73,7 @@ func TestParseConfDefaultStageTypeIsCommand(t *testing.T) {
command: echo "hello, world"
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
assert.Nil(t, err)
}
Expand All @@ -86,7 +86,7 @@ func TestParseConfWithDirectory(t *testing.T) {
directory: /usr/local
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.CommandStage).Directory
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Directory
assert.Nil(t, err)
assert.Equal(t, "/usr/local", actual)
}
Expand All @@ -98,7 +98,7 @@ func TestParseConfWithShellScriptStage(t *testing.T) {
file: ../stages/test_sample.sh
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.ShellScriptStage).File
actual := result.Pipeline.Stages.Front().Value.(*stages.ShellScriptStage).File
assert.Equal(t, "../stages/test_sample.sh", actual)
assert.Nil(t, err)
}
Expand Down Expand Up @@ -179,7 +179,7 @@ func TestParseConfWithEnvVariable(t *testing.T) {
envs := NewEnvVariables()
envs.Add("USER_NAME", "takahi-i")
result, err := ParseWithSpecifiedEnvs(configData, envs)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello $USER_NAME\"", actual)
assert.Nil(t, err)
}
Expand All @@ -205,7 +205,7 @@ func TestParseConfWithNoExistEnvVariable(t *testing.T) {
envs := NewEnvVariables()
envs.Add("USER_NAME", "takahi-i")
result, err := ParseWithSpecifiedEnvs(configData, envs)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello $NO_SUCH_A_ENV_VARIABLE\"", actual) // NOTE: No env variable name is shown when there is no env variable
assert.Nil(t, err)
}
Expand Down Expand Up @@ -240,8 +240,8 @@ func TestParseConfigWithDeprecatedProperties(t *testing.T) {
command: echo "hello, world"
`))
result, err := Parse(configData)
assert.Equal(t, 1, result.Size())
assert.Equal(t, 1, result.Pipeline.Size())
assert.Nil(t, err)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
}
10 changes: 5 additions & 5 deletions engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
)

type Engine struct {
Pipeline *pipelines.Pipeline
Opts *config.Opts
Resources *pipelines.Resources
MonitorCh *chan stages.Mediator
Opts *config.Opts
}

type Result struct {
Expand All @@ -45,8 +45,8 @@ func (r *Result) IsSucceeded() bool {
}

func (e *Engine) RunOnce() *Result {
pipe_result := e.executePipeline(e.Pipeline, "pipeline")
cleanup_result := e.executePipeline(e.Pipeline.Cleanup, "cleanup")
pipe_result := e.executePipeline(e.Resources.Pipeline, "pipeline")
cleanup_result := e.executePipeline(e.Resources.Cleanup, "cleanup")
return &Result{Pipeline: &pipe_result, Cleanup: &cleanup_result}
}

Expand Down Expand Up @@ -92,7 +92,7 @@ func (e *Engine) ExecuteStage(stage stages.Stage) {
result = false
}
log.Debugf("Stage execution results: %+v, %+v", stage.GetStageName(), result)
e.Pipeline.ReportStageResult(stage, result)
e.Resources.ReportStageResult(stage, result)

mediator := stages.Mediator{States: make(map[string]string)}
mediator.States[stage.GetStageName()] = fmt.Sprintf("%v", result)
Expand Down
80 changes: 44 additions & 36 deletions engine/engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func execute(stage stages.Stage) stages.Mediator {
mon := make(chan stages.Mediator)
e := &Engine{
MonitorCh: &mon,
Pipeline: &pipelines.Pipeline{
Resources: &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
},
}
Expand Down Expand Up @@ -102,15 +102,16 @@ func execute(stage stages.Stage) stages.Mediator {
}

func TestRunOnce(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -122,14 +123,15 @@ func TestRunOnce(t *testing.T) {
}

func TestRunOnceWithShellScriptStage(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createShellScriptStage("foobar-shell", "../stages/test_sample.sh"))
resources.Pipeline.AddStage(createShellScriptStage("foobar-shell", "../stages/test_sample.sh"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -141,17 +143,18 @@ func TestRunOnceWithShellScriptStage(t *testing.T) {
}

func TestRunOnceWithOptsOffStopOnAnyFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("thisiserrorcommand"))
pipeline.AddStage(createCommandStage("echo foobar2"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("thisiserrorcommand"))
resources.Pipeline.AddStage(createCommandStage("echo foobar2"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{StopOnAnyFailure: false}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -162,17 +165,18 @@ func TestRunOnceWithOptsOffStopOnAnyFailure(t *testing.T) {
}

func TestRunOnceWithOptsOnStopOnAnyFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("thisiserrorcommand"))
pipeline.AddStage(createCommandStage("echo foobar2"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("thisiserrorcommand"))
resources.Pipeline.AddStage(createCommandStage("echo foobar2"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{StopOnAnyFailure: true}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -186,17 +190,18 @@ func TestRunOnceWithOptsOnStopOnAnyFailure(t *testing.T) {
}

func TestRunOnceWithOnlyIfFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -lt 1"))
pipeline.AddStage(createCommandStageWithName("second", "echo second"))
pipeline.AddStage(createCommandStageWithName("third", "echo third"))
resources.Pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -lt 1"))
resources.Pipeline.AddStage(createCommandStageWithName("second", "echo second"))
resources.Pipeline.AddStage(createCommandStageWithName("third", "echo third"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -212,17 +217,18 @@ func TestRunOnceWithOnlyIfFailure(t *testing.T) {
}

func TestRunOnceWithOnlyIfSuccess(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -eq 1"))
pipeline.AddStage(createCommandStageWithName("second", "echo second"))
pipeline.AddStage(createCommandStageWithName("third", "echo third"))
resources.Pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -eq 1"))
resources.Pipeline.AddStage(createCommandStageWithName("second", "echo second"))
resources.Pipeline.AddStage(createCommandStageWithName("third", "echo third"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -241,16 +247,17 @@ func TestRunOnceWithCleanup(t *testing.T) {
cleanup := &pipelines.Pipeline{}
cleanup.AddStage(createCommandStage("echo cleanup"))
cleanup.AddStage(createCommandStage("echo baz"))
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: cleanup,
}

pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -264,16 +271,17 @@ func TestRunOnceWithCleanup(t *testing.T) {
func TestRunOnceWithFailedCleanup(t *testing.T) {
cleanup := &pipelines.Pipeline{}
cleanup.AddStage(createCommandStage("nosuchacommand"))
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: cleanup,
}

pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand Down
10 changes: 7 additions & 3 deletions pipelines/pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,17 @@ import (
)

type Pipeline struct {
Stages list.List
Stages list.List
}

type Resources struct {
Pipeline *Pipeline
Cleanup *Pipeline
Reporter messengers.Messenger
RepoService services.Service
Cleanup *Pipeline
}

func (self *Pipeline) ReportStageResult(stage stages.Stage, result bool) {
func (self *Resources) ReportStageResult(stage stages.Stage, result bool) {
name := stage.GetStageName()
self.Reporter.Post(
fmt.Sprintf("Stage execution results: %+v, %+v", name, result))
Expand Down
4 changes: 2 additions & 2 deletions pipelines/pipeline_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (mock *MockMessenger) Post(msg string) bool {

func TestReportStageResult(t *testing.T) {
mock := &MockMessenger{}
p := Pipeline{
p := Resources{
Reporter: mock,
}

Expand All @@ -76,7 +76,7 @@ func TestReportStageResult(t *testing.T) {

func TestReportStageResultWithFullOutput(t *testing.T) {
mock := &MockMessenger{}
p := Pipeline{
p := Resources{
Reporter: mock,
}

Expand Down
Loading

0 comments on commit 2e53e91

Please sign in to comment.