add get tag to files section: get describe GET request for re…

…mote storage

.github/workflows/go.yml → .github/workflows/test.yml

@@ -1,9 +1,9 @@
-name: Go package
+name: Test
 
 on: [push]
 
 jobs:
-  build:
+  test:
 
     runs-on: ubuntu-latest
     steps:

Readme.md

@@ -1,4 +1,6 @@
 # ProGen
+![test](https://github.com/kozmod/progen/actions/workflows/test.yml/badge.svg)
+![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/kozmod/progen)
 
 Simple project's generator.
 
@@ -22,18 +24,24 @@ go install github.com/kozmod/progen@latest
 
 #### Allowed config file's keys
 
-| Key   |                              Type                              |          Description           |
-|:------|:--------------------------------------------------------------:|:------------------------------:|
-| dirs  |                          string slice                          | list of directories to create  |
-| files | Path string \`yaml:"path"\`  <br/> Data string \`yaml:"data"\` | list file's `path` and `data`  |
-| cmd   |                          string slice                          | list of directories to execute |
+| Key               |       Type        |                      Description                      |
+|:------------------|:-----------------:|:-----------------------------------------------------:|
+| dirs              |   string slice    |             list of directories to create             |
+| files             |      struct       |             list file's `path` and `data`             |
+| files.path        |      string       |                       list file                       |
+| files.data        |      string       |                       file data                       |
+| files.get         |      struct       | struct describe `GET` request for getting file's data |
+| files.get.url     |      string       |                      request URL                      |
+| files.get.headers | map[string]string |                    request headers                    |
+| cmd               |   string slice    |            list of directories to execute             |
 
 <b>Note</b>: preprocessing of "raw" config use [text/template](https://pkg.go.dev/text/template) package
-that allow to add custom `yaml` keys tree to avoid duplication
+that allow to add custom `yaml` keys tree to avoid duplication (all tags could be used as template's value)
 
 #### Example
+
 ```yaml
-# custom variables to avoid duplication
+# custom variables to avoid duplication ( for example "{{.vars.GOPROXY}}")
 vars:
   GOPROXY: https://127.0.0.1:8081
   TOKEN: PRIVATE-TOKEN:token
@@ -47,6 +55,12 @@ dirs:
 
 # list files to create
 files:
+  - path: .gitlab-ci.yml
+    # GET file from remote storage
+    get:
+      url: "https://some_file_server.com/files/.gitlab-ci.yml"
+      headers:
+        some_header: header
   - path: .gitignore
     data: |
       .DS_Store
@@ -82,10 +96,13 @@ files:
 cmd:
   - curl -H {{.vars.TOKEN}} {{.vars.REPO_1}}/.gitignore/raw?ref=master -o .gitignore
 ```
+
 ```
 progen -v -f conf.yml
 ```
+
 generated project structure
+
 ```
 .
 ├── api
@@ -94,7 +111,8 @@ generated project structure
 │   └── Dockerfile
 ├── internal
 │   └── client
-└── pkg
+├── pkg
+└── .gitlab-ci.yml
 
 ```
 

go.mod

@@ -3,6 +3,7 @@ module github.com/kozmod/progen
 go 1.19
 
 require (
+	github.com/go-resty/resty/v2 v2.7.0
 	github.com/stretchr/testify v1.8.1
 	go.uber.org/zap v1.24.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -13,4 +14,5 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/net v0.0.0-20211029224645-99673261e6eb // indirect
 )

go.sum

@@ -2,6 +2,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -20,6 +22,13 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/config/config.go

@@ -2,6 +2,10 @@ package config
 
 import (
 	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/kozmod/progen/internal/entity"
 
 	"gopkg.in/yaml.v3"
 )
@@ -19,16 +23,58 @@ type Config struct {
 }
 
 type File struct {
-	Path string `yaml:"path"`
-	Data string `yaml:"data"`
+	Path string  `yaml:"path"`
+	Data *string `yaml:"data"`
+	Get  *Get    `yaml:"get"`
+}
+
+type Get struct {
+	Headers map[string]string `yaml:"headers"`
+	URL     AddrURL           `yaml:"url"`
+}
+
+type AddrURL struct {
+	*url.URL
+}
+
+func (addr *AddrURL) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var raw string
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+	u, err := url.Parse(raw)
+	if err != nil {
+		return fmt.Errorf("unmarshal url: %w", err)
+	}
+	addr.URL = u
+	return nil
 }
 
 func UnmarshalYamlConfig(in []byte) (Config, error) {
 	var conf Config
 	err := yaml.Unmarshal(in, &conf)
 	if err != nil {
-		return conf, fmt.Errorf("config.unmarshalConfig: %w", err)
+		return conf, fmt.Errorf("unmarshal config: %w", err)
+	}
+
+	for i, file := range conf.Files {
+		err = ValidateFile(file)
+		if err != nil {
+			return conf, fmt.Errorf("unmarshal config: validate files: %d [%s]: %w", i, file.Path, err)
+		}
 	}
 
 	return conf, nil
 }
+
+func ValidateFile(file File) error {
+	switch {
+	case file.Get != nil && file.Data != nil:
+		return fmt.Errorf("files: `get` and `data` tags can not be present both")
+	case file.Get == nil && file.Data == nil:
+		return fmt.Errorf("files: `get` and `data` are empty both")
+	case strings.TrimSpace(file.Path) == entity.Empty:
+		return fmt.Errorf("files: `path` is empty")
+	}
+	return nil
+}

internal/config/config_test.go

@@ -106,3 +106,57 @@ steps:
 		assert.Equal(t, expected, string(res))
 	})
 }
+
+func Test_ValidateFile(t *testing.T) {
+	const (
+		path = "some_path"
+	)
+	t.Run("not_error_when_get_is_not_nil", func(t *testing.T) {
+		in := File{
+			Path: path,
+			Data: nil,
+			Get:  &Get{},
+		}
+		err := ValidateFile(in)
+		assert.NoError(t, err)
+	})
+	t.Run("not_error_when_data_is_not_nil", func(t *testing.T) {
+		in := File{
+			Path: path,
+			Data: func(s string) *string { return &s }("some data"),
+			Get:  nil,
+		}
+		err := ValidateFile(in)
+		assert.NoError(t, err)
+	})
+	t.Run("error_when_data_and_get_are_nil", func(t *testing.T) {
+		in := File{
+			Path: path,
+			Data: nil,
+			Get:  nil,
+		}
+		err := ValidateFile(in)
+		assert.Error(t, err)
+	})
+	t.Run("error_when_data_and_get_are_not_nil_both", func(t *testing.T) {
+		in := File{
+			Path: path,
+			Data: func(s string) *string { return &s }("some data"),
+			Get:  &Get{},
+		}
+		err := ValidateFile(in)
+		assert.Error(t, err)
+	})
+	t.Run("error_when_path_is_empty", func(t *testing.T) {
+		in := File{
+			Path: "",
+			Get:  &Get{},
+		}
+		err := ValidateFile(in)
+		assert.Error(t, err)
+
+		in.Path = "     "
+		err = ValidateFile(in)
+		assert.Error(t, err)
+	})
+}

internal/entity/entity.go

@@ -5,6 +5,10 @@ const (
 	Empty = ""
 )
 
+type FileProducer interface {
+	Get() (*File, error)
+}
+
 //goland:noinspection SpellCheckingInspection
 type Logger interface {
 	Infof(template string, args ...any)
@@ -16,6 +20,13 @@ type File struct {
 	Data []byte
 }
 
+type RemoteFile struct {
+	Path    string
+	Name    string
+	URL     string
+	Headers map[string]string
+}
+
 type Command struct {
 	Cmd  string
 	Args []string

internal/factory/file_proc.go

@@ -1,6 +1,7 @@
 package factory
 
 import (
+	"fmt"
 	"path/filepath"
 
 	"github.com/kozmod/progen/internal/config"
@@ -13,15 +14,32 @@ func NewFileProc(conf config.Config, logger entity.Logger) (proc.Proc, error) {
 		return nil, nil
 	}
 
-	files := make([]entity.File, 0, len(conf.Files))
+	producers := make([]entity.FileProducer, 0, len(conf.Files))
 	for _, f := range conf.Files {
-		tmpl := entity.File{
-			Name: filepath.Base(f.Path),
-			Path: filepath.Dir(f.Path),
-			Data: []byte(f.Data),
+
+		var producer entity.FileProducer
+		switch {
+		case f.Data != nil:
+			file := entity.File{
+				Name: filepath.Base(f.Path),
+				Path: filepath.Dir(f.Path),
+				Data: []byte(*f.Data),
+			}
+			producer = proc.NewStoredProducer(file)
+		case f.Get != nil:
+			file := entity.RemoteFile{
+				Name:    filepath.Base(f.Path),
+				Path:    filepath.Dir(f.Path),
+				URL:     f.Get.URL.String(),
+				Headers: f.Get.Headers,
+			}
+			producer = proc.NewRemoteProducer(file)
+		default:
+			return nil, fmt.Errorf("create file processor: `data` or `get` must not be empty")
 		}
-		files = append(files, tmpl)
+
+		producers = append(producers, producer)
 	}
 
-	return proc.NewFileProc(files, logger), nil
+	return proc.NewFileProc(producers, logger), nil
 }

internal/factory/mkdir_proc.go

@@ -15,9 +15,9 @@ func NewMkdirProc(conf config.Config, logger entity.Logger) (proc.Proc, error) {
 	return proc.NewMkdirAllProc(dirSet, logger), nil
 }
 
-func uniqueVal(in []string) []string {
-	set := make(map[string]struct{}, len(in))
-	out := make([]string, 0, len(in))
+func uniqueVal[T comparable](in []T) []T {
+	set := make(map[T]struct{}, len(in))
+	out := make([]T, 0, len(in))
 	for _, val := range in {
 		_, ok := set[val]
 		if ok {

internal/proc/cmd.go

@@ -0,0 +1,38 @@
+package proc
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/kozmod/progen/internal/entity"
+)
+
+type RunCommandProc struct {
+	commands []entity.Command
+	logger   entity.Logger
+}
+
+func NewRunCommandProc(commands []entity.Command, logger entity.Logger) *RunCommandProc {
+	return &RunCommandProc{
+		commands: commands,
+		logger:   logger,
+	}
+}
+
+func (p *RunCommandProc) Exec() error {
+	for _, command := range p.commands {
+		cmd := exec.Command(command.Cmd, command.Args...)
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		err := cmd.Run()
+		if err != nil {
+			return fmt.Errorf("run command: %w", err)
+		}
+		p.logger.Infof("execute:\ncmd: %s\nout: %s",
+			strings.Join(append([]string{command.Cmd}, command.Args...), entity.Space),
+			out.String())
+	}
+	return nil
+}