diff --git a/Makefile b/Makefile index 6cb3b715..adf4bb2a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,9 @@ build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o _output/cmd/builder github.com/kubesphere/s2irun/cmd run: S2I_CONFIG_PATH=test/config.json go run ./cmd/main.go +run-b2i: + S2I_CONFIG_PATH=test/b2iconfig.json go run ./cmd/main.go image: docker build . -t ${IMG} push: image - docker push ${IMG} \ No newline at end of file + docker push ${IMG} diff --git a/pkg/api/describe/describer.go b/pkg/api/describe/describer.go index 5ce1bef0..a3f9befc 100644 --- a/pkg/api/describe/describer.go +++ b/pkg/api/describe/describer.go @@ -96,11 +96,11 @@ func Config(client docker.Client, config *api.Config) string { func describeBuilderImage(client docker.Client, config *api.Config, out io.Writer) { c := &api.Config{ - DockerConfig: config.DockerConfig, - PullAuthentication: config.PullAuthentication, - BuilderImage: config.BuilderImage, - BuilderPullPolicy: config.BuilderPullPolicy, - Tag: config.Tag, + DockerConfig: config.DockerConfig, + PullAuthentication: config.PullAuthentication, + BuilderImage: config.BuilderImage, + BuilderPullPolicy: config.BuilderPullPolicy, + Tag: config.Tag, IncrementalAuthentication: config.IncrementalAuthentication, } dkr := docker.New(client, c.PullAuthentication, c.PushAuthentication) diff --git a/pkg/api/types.go b/pkg/api/types.go index 8c8f5949..cbdcfdd1 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -261,6 +261,10 @@ type Config struct { // SourceURL is url of the codes such as https://github.com/a/b.git SourceURL string `json:"sourceURL,omitempty"` + // IsBinaryURL explain the type of SourceURL. + // If it is IsBinaryURL, it will download the file directly without using git. + IsBinaryURL bool `json:"isBinaryURL,omitempty"` + // The RevisionId is a branch name or a SHA-1 hash of every important thing about the commit RevisionId string `json:"revisionId,omitempty"` } diff --git a/pkg/build/config.go b/pkg/build/config.go index a7a0a0ba..b0541e3a 100644 --- a/pkg/build/config.go +++ b/pkg/build/config.go @@ -43,7 +43,7 @@ func GenerateConfigFromLabels(config *api.Config, metadata *docker.PullResult) e } if repo, ok := labels[constants.BuildSourceLocationLabel]; ok { - source, err := git.Parse(repo) + source, err := git.Parse(repo, false) if err != nil { return fmt.Errorf("couldn't parse label %q value %s: %v", constants.BuildSourceLocationLabel, repo, err) } diff --git a/pkg/build/strategies/onbuild/onbuild_test.go b/pkg/build/strategies/onbuild/onbuild_test.go index 5d8e8e56..d4e9b50f 100644 --- a/pkg/build/strategies/onbuild/onbuild_test.go +++ b/pkg/build/strategies/onbuild/onbuild_test.go @@ -7,13 +7,13 @@ import ( "strings" "testing" + "github.com/docker/docker/builder/dockerfile" "github.com/kubesphere/s2irun/pkg/api" "github.com/kubesphere/s2irun/pkg/docker" "github.com/kubesphere/s2irun/pkg/scm/git" "github.com/kubesphere/s2irun/pkg/test" testfs "github.com/kubesphere/s2irun/pkg/test/fs" "github.com/kubesphere/s2irun/pkg/utils/fs" - "github.com/docker/docker/builder/dockerfile" ) type fakeSourceHandler struct{} diff --git a/pkg/run/s2i.go b/pkg/run/s2i.go index f6e515a2..4fab4e93 100644 --- a/pkg/run/s2i.go +++ b/pkg/run/s2i.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "os" "github.com/kubesphere/s2irun/pkg/api" "github.com/kubesphere/s2irun/pkg/api/describe" "github.com/kubesphere/s2irun/pkg/api/validation" @@ -13,6 +12,7 @@ import ( s2ierr "github.com/kubesphere/s2irun/pkg/errors" "github.com/kubesphere/s2irun/pkg/scm/git" utilglog "github.com/kubesphere/s2irun/pkg/utils/glog" + "os" ) const ( @@ -107,7 +107,7 @@ func App() int { glog.Errorf("There are some errors in config file, please check the error:\n%v", err) return 1 } - apiConfig.Source, err = git.Parse(apiConfig.SourceURL) + apiConfig.Source, err = git.Parse(apiConfig.SourceURL, apiConfig.IsBinaryURL) if err != nil { glog.Errorf("SourceURL is illegal, please check the error:\n%v", err) return 1 diff --git a/pkg/scm/downloaders/binary/download.go b/pkg/scm/downloaders/binary/download.go new file mode 100644 index 00000000..4456244d --- /dev/null +++ b/pkg/scm/downloaders/binary/download.go @@ -0,0 +1,82 @@ +package binary + +import ( + "github.com/kubesphere/s2irun/pkg/api" + "github.com/kubesphere/s2irun/pkg/api/constants" + "github.com/kubesphere/s2irun/pkg/scm/git" + "github.com/kubesphere/s2irun/pkg/utils/bytefmt" + "github.com/kubesphere/s2irun/pkg/utils/fs" + utilglog "github.com/kubesphere/s2irun/pkg/utils/glog" + "io" + "net/http" + "path/filepath" + "strconv" + "strings" +) + +var glog = utilglog.StderrLog + +// File represents a simplest possible Downloader implementation where the +// sources are just copied from local directory. +type File struct { + fs.FileSystem +} + +// Download download sources from a http link into the working directory. +// Caller guarantees that config.Source.IsLocal() is true. +func (f *File) Download(config *api.Config) (*git.SourceInfo, error) { + _, filename := filepath.Split(config.Source.String()) + config.WorkingSourceDir = filepath.Join(config.WorkingDir, constants.Source) + binaryPath := filepath.Join(config.WorkingSourceDir, filename) + glog.V(0).Infof("Start Download Binary %s", filename) + resp, err := http.Get(config.Source.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + out, err := f.Create(binaryPath) + if err != nil { + return nil, err + } + + defer out.Close() + + strsize := resp.Header.Get("Content-Length") + size, _ := strconv.ParseUint(strsize, 10, 64) + + counter := &WriteCounter{Size: size} + _, err = io.Copy(out, io.TeeReader(resp.Body, counter)) + glog.V(0).Infof("Finish Download Binary %s", filename) + return &git.SourceInfo{ + Location: config.Source.String(), + ContextDir: config.ContextDir, + }, nil +} + +// write cycle. +type WriteCounter struct { + Total uint64 + Size uint64 +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.Total += uint64(n) + wc.PrintProgress() + return n, nil +} + +func (wc WriteCounter) PrintProgress() { + // Clear the line by using a character return to go back to the start and remove + // the remaining characters by filling it with spaces + glog.V(0).Infof("\r%s", strings.Repeat(" ", 35)) + + // Return again and print current status of download + // We use the humanize package to print the bytes in a meaningful way (e.g. 10 MB) + if wc.Size > 0 { + glog.V(0).Infof("\rDownloading... %s//%s", bytefmt.ByteSize(wc.Total), bytefmt.ByteSize(wc.Size)) + } else { + glog.V(0).Infof("\rDownloading... %s complete", bytefmt.ByteSize(wc.Total)) + } + +} diff --git a/pkg/scm/downloaders/binary/download_test.go b/pkg/scm/downloaders/binary/download_test.go new file mode 100644 index 00000000..f8255820 --- /dev/null +++ b/pkg/scm/downloaders/binary/download_test.go @@ -0,0 +1,28 @@ +package binary + +import ( + "github.com/kubesphere/s2irun/pkg/api" + "github.com/kubesphere/s2irun/pkg/scm/git" + testfs "github.com/kubesphere/s2irun/pkg/test/fs" + "testing" +) + +func TestDownload(t *testing.T) { + fs := &testfs.FakeFileSystem{} + f := &File{fs} + + config := &api.Config{ + Source: git.MustParse("https://kubesphere.io/etcd-operator.svg"), + IsBinaryURL: true, + } + info, err := f.Download(config) + if err != nil { + t.Errorf("Unexpected error %v", err) + } + if fs.CreateFile != "upload/src/etcd-operator.svg" { + t.Errorf("Unexpected fs.CreateFile %s", fs.CreateFile) + } + if info.Location != config.Source.String() || info.ContextDir != config.ContextDir { + t.Errorf("Unexpected info") + } +} diff --git a/pkg/scm/git/url.go b/pkg/scm/git/url.go index 15f2cc3d..ed7f4f59 100644 --- a/pkg/scm/git/url.go +++ b/pkg/scm/git/url.go @@ -42,6 +42,8 @@ const ( URLTypeSCP // URLTypeLocal is the local type (see above) URLTypeLocal + // URLTypeBinary is the URL to download file + URLTypeBinary ) // String returns a string representation of the URLType @@ -53,6 +55,8 @@ func (t URLType) String() string { return "URLTypeSCP" case URLTypeLocal: return "URLTypeLocal" + case URLTypeBinary: + return "URLTypeBinary" } panic("unknown URLType") } @@ -84,7 +88,7 @@ func splitOnByte(s string, c byte) (string, string) { } // Parse parses a "Git URL" -func Parse(rawurl string) (*URL, error) { +func Parse(rawurl string, isBinaryURL bool) (*URL, error) { if urlSchemeRegexp.MatchString(rawurl) && (runtime.GOOS != "windows" || !dosDriveRegexp.MatchString(rawurl)) { u, err := url.Parse(rawurl) @@ -99,11 +103,17 @@ func Parse(rawurl string) (*URL, error) { return nil, fmt.Errorf("file url %q has non-absolute path %q", rawurl, u.Path) } } - - return &URL{ - URL: *u, - Type: URLTypeURL, - }, nil + if isBinaryURL { + return &URL{ + URL: *u, + Type: URLTypeBinary, + }, nil + } else { + return &URL{ + URL: *u, + Type: URLTypeURL, + }, nil + } } s, fragment := splitOnByte(rawurl, '#') @@ -136,7 +146,7 @@ func Parse(rawurl string) (*URL, error) { // MustParse parses a "Git URL" and panics on failure func MustParse(rawurl string) *URL { - u, err := Parse(rawurl) + u, err := Parse(rawurl, false) if err != nil { panic(err) } @@ -149,6 +159,8 @@ func (u URL) String() string { switch u.Type { case URLTypeURL: return u.URL.String() + case URLTypeBinary: + return u.URL.String() case URLTypeSCP: if u.URL.User != nil { s = u.URL.User.Username() + "@" diff --git a/pkg/scm/git/url_test.go b/pkg/scm/git/url_test.go index 062bef6e..4cb76b86 100644 --- a/pkg/scm/git/url_test.go +++ b/pkg/scm/git/url_test.go @@ -209,7 +209,7 @@ func TestParse(t *testing.T) { ) for _, test := range tests { - parsedURL, err := Parse(test.rawurl) + parsedURL, err := Parse(test.rawurl, false) if test.expectedError != (err != nil) { t.Errorf("%s: Parse() returned err: %v", test.rawurl, err) } diff --git a/pkg/scm/scm.go b/pkg/scm/scm.go index 95b178a2..9b218353 100644 --- a/pkg/scm/scm.go +++ b/pkg/scm/scm.go @@ -3,6 +3,7 @@ package scm import ( "github.com/kubesphere/s2irun/pkg/build" "github.com/kubesphere/s2irun/pkg/errors" + "github.com/kubesphere/s2irun/pkg/scm/downloaders/binary" "github.com/kubesphere/s2irun/pkg/scm/downloaders/empty" "github.com/kubesphere/s2irun/pkg/scm/downloaders/file" gitdownloader "github.com/kubesphere/s2irun/pkg/scm/downloaders/git" @@ -22,7 +23,9 @@ func DownloaderForSource(fs fs.FileSystem, s *git.URL, forceCopy bool) (build.Do if s == nil { return &empty.Noop{}, nil } - + if s.Type == git.URLTypeBinary { + return &binary.File{FileSystem: fs}, nil + } if s.IsLocal() { if forceCopy { return &file.File{FileSystem: fs}, nil diff --git a/pkg/utils/bytefmt/bytes.go b/pkg/utils/bytefmt/bytes.go new file mode 100644 index 00000000..6808a5e2 --- /dev/null +++ b/pkg/utils/bytefmt/bytes.go @@ -0,0 +1,121 @@ +// Package bytefmt contains helper methods and constants for converting to and from a human-readable byte format. +// +// bytefmt.ByteSize(100.5*bytefmt.MEGABYTE) // "100.5M" +// bytefmt.ByteSize(uint64(1024)) // "1K" +// +package bytefmt + +import ( + "errors" + "strconv" + "strings" + "unicode" +) + +const ( + BYTE = 1 << (10 * iota) + KILOBYTE + MEGABYTE + GIGABYTE + TERABYTE + PETABYTE + EXABYTE +) + +var invalidByteQuantityError = errors.New("byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB") + +// ByteSize returns a human-readable byte string of the form 10M, 12.5K, and so forth. The following units are available: +// E: Exabyte +// P: Petabyte +// T: Terabyte +// G: Gigabyte +// M: Megabyte +// K: Kilobyte +// B: Byte +// The unit that results in the smallest number greater than or equal to 1 is always chosen. +func ByteSize(bytes uint64) string { + unit := "" + value := float64(bytes) + + switch { + case bytes >= EXABYTE: + unit = "E" + value = value / EXABYTE + case bytes >= PETABYTE: + unit = "P" + value = value / PETABYTE + case bytes >= TERABYTE: + unit = "T" + value = value / TERABYTE + case bytes >= GIGABYTE: + unit = "G" + value = value / GIGABYTE + case bytes >= MEGABYTE: + unit = "M" + value = value / MEGABYTE + case bytes >= KILOBYTE: + unit = "K" + value = value / KILOBYTE + case bytes >= BYTE: + unit = "B" + case bytes == 0: + return "0" + } + + result := strconv.FormatFloat(value, 'f', 1, 64) + result = strings.TrimSuffix(result, ".0") + return result + unit +} + +// ToMegabytes parses a string formatted by ByteSize as megabytes. +func ToMegabytes(s string) (uint64, error) { + bytes, err := ToBytes(s) + if err != nil { + return 0, err + } + + return bytes / MEGABYTE, nil +} + +// ToBytes parses a string formatted by ByteSize as bytes. Note binary-prefixed and SI prefixed units both mean a base-2 units +// KB = K = KiB = 1024 +// MB = M = MiB = 1024 * K +// GB = G = GiB = 1024 * M +// TB = T = TiB = 1024 * G +// PB = P = PiB = 1024 * T +// EB = E = EiB = 1024 * P +func ToBytes(s string) (uint64, error) { + s = strings.TrimSpace(s) + s = strings.ToUpper(s) + + i := strings.IndexFunc(s, unicode.IsLetter) + + if i == -1 { + return 0, invalidByteQuantityError + } + + bytesString, multiple := s[:i], s[i:] + bytes, err := strconv.ParseFloat(bytesString, 64) + if err != nil || bytes <= 0 { + return 0, invalidByteQuantityError + } + + switch multiple { + case "E", "EB", "EIB": + return uint64(bytes * EXABYTE), nil + case "P", "PB", "PIB": + return uint64(bytes * PETABYTE), nil + case "T", "TB", "TIB": + return uint64(bytes * TERABYTE), nil + case "G", "GB", "GIB": + return uint64(bytes * GIGABYTE), nil + case "M", "MB", "MIB": + return uint64(bytes * MEGABYTE), nil + case "K", "KB", "KIB": + return uint64(bytes * KILOBYTE), nil + case "B": + return uint64(bytes), nil + default: + return 0, invalidByteQuantityError + } +} diff --git a/pkg/utils/idutils/id_utils.go b/pkg/utils/idutils/id_utils.go index 30cd445a..a41b6b1b 100644 --- a/pkg/utils/idutils/id_utils.go +++ b/pkg/utils/idutils/id_utils.go @@ -1,38 +1,38 @@ -package idutils - -import ( - "github.com/kubesphere/s2irun/pkg/utils/stringutils" - "github.com/sony/sonyflake" - "github.com/speps/go-hashids" -) - -var sf *sonyflake.Sonyflake - -func init() { - var st sonyflake.Settings - sf = sonyflake.NewSonyflake(st) -} - -func GetIntId() uint64 { - id, err := sf.NextID() - if err != nil { - panic(err) - } - return id -} - -// format likes: B6BZVN3mOPvx -func GetUuid(prefix string) string { - id := GetIntId() - hd := hashids.NewData() - h, err := hashids.NewWithData(hd) - if err != nil { - panic(err) - } - i, err := h.Encode([]int{int(id)}) - if err != nil { - panic(err) - } - - return prefix + stringutils.Reverse(i) -} +package idutils + +import ( + "github.com/kubesphere/s2irun/pkg/utils/stringutils" + "github.com/sony/sonyflake" + "github.com/speps/go-hashids" +) + +var sf *sonyflake.Sonyflake + +func init() { + var st sonyflake.Settings + sf = sonyflake.NewSonyflake(st) +} + +func GetIntId() uint64 { + id, err := sf.NextID() + if err != nil { + panic(err) + } + return id +} + +// format likes: B6BZVN3mOPvx +func GetUuid(prefix string) string { + id := GetIntId() + hd := hashids.NewData() + h, err := hashids.NewWithData(hd) + if err != nil { + panic(err) + } + i, err := h.Encode([]int{int(id)}) + if err != nil { + panic(err) + } + + return prefix + stringutils.Reverse(i) +} diff --git a/pkg/utils/stringutils/string.go b/pkg/utils/stringutils/string.go index fd43910a..282c790b 100644 --- a/pkg/utils/stringutils/string.go +++ b/pkg/utils/stringutils/string.go @@ -1,64 +1,64 @@ -package stringutils - -import ( - "unicode/utf8" - - "github.com/asaskevich/govalidator" -) - -// Creates an slice of slice values not included in the other given slice. -func Diff(base, exclude []string) (result []string) { - excludeMap := make(map[string]bool) - for _, s := range exclude { - excludeMap[s] = true - } - for _, s := range base { - if !excludeMap[s] { - result = append(result, s) - } - } - return result -} - -func Unique(ss []string) (result []string) { - smap := make(map[string]bool) - for _, s := range ss { - smap[s] = true - } - for s := range smap { - result = append(result, s) - } - return result -} - -func CamelCaseToUnderscore(str string) string { - return govalidator.CamelCaseToUnderscore(str) -} - -func UnderscoreToCamelCase(str string) string { - return govalidator.UnderscoreToCamelCase(str) -} - -func FindString(array []string, str string) int { - for index, s := range array { - if str == s { - return index - } - } - return -1 -} - -func StringIn(str string, array []string) bool { - return FindString(array, str) > -1 -} - -func Reverse(s string) string { - size := len(s) - buf := make([]byte, size) - for start := 0; start < size; { - r, n := utf8.DecodeRuneInString(s[start:]) - start += n - utf8.EncodeRune(buf[size-start:], r) - } - return string(buf) -} +package stringutils + +import ( + "unicode/utf8" + + "github.com/asaskevich/govalidator" +) + +// Creates an slice of slice values not included in the other given slice. +func Diff(base, exclude []string) (result []string) { + excludeMap := make(map[string]bool) + for _, s := range exclude { + excludeMap[s] = true + } + for _, s := range base { + if !excludeMap[s] { + result = append(result, s) + } + } + return result +} + +func Unique(ss []string) (result []string) { + smap := make(map[string]bool) + for _, s := range ss { + smap[s] = true + } + for s := range smap { + result = append(result, s) + } + return result +} + +func CamelCaseToUnderscore(str string) string { + return govalidator.CamelCaseToUnderscore(str) +} + +func UnderscoreToCamelCase(str string) string { + return govalidator.UnderscoreToCamelCase(str) +} + +func FindString(array []string, str string) int { + for index, s := range array { + if str == s { + return index + } + } + return -1 +} + +func StringIn(str string, array []string) bool { + return FindString(array, str) > -1 +} + +func Reverse(s string) string { + size := len(s) + buf := make([]byte, size) + for start := 0; start < size; { + r, n := utf8.DecodeRuneInString(s[start:]) + start += n + utf8.EncodeRune(buf[size-start:], r) + } + return string(buf) +} diff --git a/test/b2iconfig.json b/test/b2iconfig.json new file mode 100644 index 00000000..840891a4 --- /dev/null +++ b/test/b2iconfig.json @@ -0,0 +1,13 @@ +{ + "displayName":"For Test", + "builderImage":"kubespheredev/java-8-centos7", + "tag":"runzexia/hello-java", + "export":false, + "pushAuthentication":{ + "username":"", + "password":"" + }, + "revisionId":"staging", + "sourceURL":"https://github.com/kubesphere/devops-java-sample/releases/download/v0.0.1/devops-sample-0.0.1-SNAPSHOT.jar", + "isBinaryURL": true +}