From b518074517b846bd86d013376d17eb73f00dbab9 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Sun, 1 Nov 2015 21:46:14 -0600 Subject: [PATCH 01/18] Added proper support for downloading via a Windows UNC path or a relative uri. Added proper support for validating a downloadableURL containing a UNC or relative uri. Removed the workaround for an earlier Go issue that had remained dormant in common/download.go (issue #5927). When building a .vmx file via the vmware-iso builder, transform the path to the correct os-formatted one (using filepath.FromSlash). --- builder/vmware/iso/step_create_vmx.go | 5 ++ common/config.go | 94 ++++++++++++++++++--------- common/download.go | 49 ++++++++++++++ packer/core.go | 2 +- 4 files changed, 119 insertions(+), 31 deletions(-) diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 1f4ee812f9d..62546d21208 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -43,6 +43,11 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { isoPath := state.Get("iso_path").(string) ui := state.Get("ui").(packer.Ui) + // Convert the iso_path into a path relative to the .vmx file if possible + if relativeIsoPath,err := filepath.Rel(config.VMXTemplatePath, filepath.FromSlash(isoPath)); err == nil { + isoPath = relativeIsoPath + } + ui.Say("Building and writing VMX file") vmxTemplate := DefaultVMXTemplate diff --git a/common/config.go b/common/config.go index 49055338849..93a54922191 100644 --- a/common/config.go +++ b/common/config.go @@ -52,76 +52,110 @@ func DownloadableURL(original string) (string, error) { // we're dealing with a drive letter and thus a file path. idx := strings.Index(original, ":") if idx == 1 { - original = "file:///" + original + original = "file://" + filepath.ToSlash(original) } } - url, err := url.Parse(original) + // XXX: The validation here is later re-parsed in common/download.go and + // thus any modifications here must remain consistent over there too. + uri, err := url.Parse(original) if err != nil { return "", err } - if url.Scheme == "" { - url.Scheme = "file" + if uri.Scheme == "" { + uri.Scheme = "file" } - if url.Scheme == "file" { - // Windows file handling is all sorts of tricky... + const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) + if uri.Scheme == "file" { + var ospath string // os-formatted pathname if runtime.GOOS == "windows" { - // If the path is using Windows-style slashes, URL parses - // it into the host field. - if url.Path == "" && strings.Contains(url.Host, `\`) { - url.Path = url.Host - url.Host = "" - } - - // For Windows absolute file paths, remove leading / prior to processing - // since net/url turns "C:/" into "/C:/" - if len(url.Path) > 0 && url.Path[0] == '/' { - url.Path = url.Path[1:len(url.Path)] + // Move any extra path components that were mis-parsed into the Host + // field back into the uri.Path field + if len(uri.Host) >= len(UNCPrefix) && uri.Host[:len(UNCPrefix)] == UNCPrefix { + idx := strings.Index(uri.Host[len(UNCPrefix):], string(os.PathSeparator)) + if idx > -1 { + uri.Path = filepath.ToSlash(uri.Host[idx+len(UNCPrefix):]) + uri.Path + uri.Host = uri.Host[:idx+len(UNCPrefix)] + } } + // Now all we need to do to convert the uri to a platform-specific path + // is to trade it's slashes for some os.PathSeparator ones. + ospath = uri.Host + filepath.FromSlash(uri.Path) + + } else { + // Since we're already using sane paths on a sane platform, anything in + // uri.Host can be assumed that the user is describing a relative uri. + // This means that if we concatenate it with uri.Path, the filepath + // transform will still open the file correctly. + // i.e. file://localdirectory/filename -> localdirectory/filename + ospath = uri.Host + uri.Path } // Only do the filepath transformations if the file appears - // to actually exist. - if _, err := os.Stat(url.Path); err == nil { - url.Path, err = filepath.Abs(url.Path) + // to actually exist. We don't do it on windows, because EvalSymlinks + // won't understand how to handle UNC paths and other Windows-specific minutae. + if _, err := os.Stat(ospath); err == nil && runtime.GOOS != "windows" { + ospath, err = filepath.Abs(ospath) if err != nil { return "", err } - url.Path, err = filepath.EvalSymlinks(url.Path) + ospath, err = filepath.EvalSymlinks(ospath) if err != nil { return "", err } - url.Path = filepath.Clean(url.Path) + ospath = filepath.Clean(ospath) } + // now that ospath was normalized and such.. if runtime.GOOS == "windows" { - // Also replace all backslashes with forwardslashes since Windows - // users are likely to do this but the URL should actually only - // contain forward slashes. - url.Path = strings.Replace(url.Path, `\`, `/`, -1) + uri.Host = "" + // Check to see if our ospath is unc-prefixed, and if it is then split + // the UNC host into uri.Host, leaving the rest in ospath. + // This way, our UNC-uri is protected from injury in the call to uri.String() + if len(ospath) >= len(UNCPrefix) && ospath[:len(UNCPrefix)] == UNCPrefix { + idx := strings.Index(ospath[len(UNCPrefix):], string(os.PathSeparator)) + if idx > -1 { + uri.Host = ospath[:len(UNCPrefix)+idx] + ospath = ospath[len(UNCPrefix)+idx:] + } + } + // Restore the uri by re-transforming our os-formatted path + uri.Path = filepath.ToSlash(ospath) + } else { + uri.Host = "" + uri.Path = filepath.ToSlash(ospath) } } // Make sure it is lowercased - url.Scheme = strings.ToLower(url.Scheme) + uri.Scheme = strings.ToLower(uri.Scheme) // Verify that the scheme is something we support in our common downloader. supported := []string{"file", "http", "https"} found := false for _, s := range supported { - if url.Scheme == s { + if uri.Scheme == s { found = true break } } if !found { - return "", fmt.Errorf("Unsupported URL scheme: %s", url.Scheme) + return "", fmt.Errorf("Unsupported URL scheme: %s", uri.Scheme) + } + + // explicit check to see if we need to manually replace the uri host with a UNC one + if runtime.GOOS == "windows" && uri.Scheme == "file" { + if len(uri.Host) >= len(UNCPrefix) && uri.Host[:len(UNCPrefix)] == UNCPrefix { + escapedHost := url.QueryEscape(uri.Host) + return strings.Replace(uri.String(), escapedHost, uri.Host, 1), nil + } } - return url.String(), nil + // otherwise, we can trust the url handler + return uri.String(), nil } diff --git a/common/download.go b/common/download.go index 8eaf236fcdf..5f9940056f6 100644 --- a/common/download.go +++ b/common/download.go @@ -15,6 +15,10 @@ import ( "net/http" "net/url" "os" + "runtime" + "path" + "path/filepath" + "strings" ) // DownloadConfig is the configuration given to instantiate a new @@ -129,6 +133,51 @@ func (d *DownloadClient) Get() (string, error) { // locally and we don't make a copy. Normally we would copy or download. log.Printf("[DEBUG] Using local file: %s", finalPath) + // FIXME: + // cwd should point to the path relative to client.json, but + // since this isn't exposed to us anywhere, we use os.Getwd() + // to figure it out. + cwd,err := os.Getwd() + if err != nil { + return "", fmt.Errorf("Unable to get working directory") + } + + // convert the actual file uri to a windowsy path + // (this logic must correspond to the same logic in common/config.go) + if runtime.GOOS == "windows" { + const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) + + // move any extra path components that were parsed into Host due + // to UNC into the url.Path field so that it's PathSeparators get + // normalized + if len(url.Host) >= len(UNCPrefix) && url.Host[:len(UNCPrefix)] == UNCPrefix { + idx := strings.Index(url.Host[len(UNCPrefix):], string(os.PathSeparator)) + if idx > -1 { + url.Path = filepath.ToSlash(url.Host[idx+len(UNCPrefix):]) + url.Path + url.Host = url.Host[:idx+len(UNCPrefix)] + } + } + + // clean up backward-slashes since they only matter when part of a unc path + urlPath := filepath.ToSlash(url.Path) + + // semi-absolute path (current drive letter) -- file:///absolute/path + if url.Host == "" && len(urlPath) > 0 && urlPath[0] == '/' { + finalPath = path.Join(filepath.VolumeName(cwd), urlPath) + + // relative path -- file://./relative/path + // file://relative/path + } else if url.Host == "" || (len(url.Host) > 0 && url.Host[0] == '.') { + finalPath = path.Join(filepath.ToSlash(cwd), urlPath) + + // absolute path + } else { + // UNC -- file://\\host/share/whatever + // drive -- file://c:/absolute/path + finalPath = path.Join(url.Host, urlPath) + } + } + // Keep track of the source so we can make sure not to delete this later sourcePath = finalPath if _, err = os.Stat(finalPath); err != nil { diff --git a/packer/core.go b/packer/core.go index 7da3e451006..c8ac2cfb7d8 100644 --- a/packer/core.go +++ b/packer/core.go @@ -67,7 +67,7 @@ func NewCore(c *CoreConfig) (*Core, error) { return nil, err } - // Go through and interpolate all the build names. We shuld be able + // Go through and interpolate all the build names. We should be able // to do this at this point with the variables. result.builds = make(map[string]*template.Builder) for _, b := range c.Template.Builders { From 06d65fcab00480410bd29872edc0f7b79cfc0347 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Mon, 18 Jan 2016 17:36:47 -0600 Subject: [PATCH 02/18] Added some testcases for the various file uri transforms to download_test.go Moved some of the code for normalizing a Windows file uri to a regular path into it's own function NormalizeWindowsURL --- common/download.go | 88 ++++++++++++++++++---------------- common/download_test.go | 103 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 41 deletions(-) diff --git a/common/download.go b/common/download.go index 5f9940056f6..65d72d22a54 100644 --- a/common/download.go +++ b/common/download.go @@ -101,6 +101,44 @@ func (d *DownloadClient) Cancel() { // TODO(mitchellh): Implement } +// Take a uri and convert it to a path that makes sense on the Windows platform +func NormalizeWindowsURL(basepath string, url url.URL) string { + // This logic must correspond to the same logic in the NormalizeWindowsURL + // function found in common/config.go since that function _also_ checks that + // the url actually exists in file form. + + const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) + + // move any extra path components that were parsed into Host due + // to UNC into the url.Path field so that it's PathSeparators get + // normalized + if len(url.Host) >= len(UNCPrefix) && url.Host[:len(UNCPrefix)] == UNCPrefix { + idx := strings.Index(url.Host[len(UNCPrefix):], string(os.PathSeparator)) + if idx > -1 { + url.Path = filepath.ToSlash(url.Host[idx+len(UNCPrefix):]) + url.Path + url.Host = url.Host[:idx+len(UNCPrefix)] + } + } + + // clean up backward-slashes since they only matter when part of a unc path + urlPath := filepath.ToSlash(url.Path) + + // semi-absolute path (current drive letter) -- file:///absolute/path + if url.Host == "" && len(urlPath) > 0 && urlPath[0] == '/' { + return path.Join(filepath.VolumeName(basepath), urlPath) + + // relative path -- file://./relative/path + // file://relative/path + } else if url.Host == "" || (len(url.Host) > 0 && url.Host[0] == '.') { + return path.Join(filepath.ToSlash(basepath), urlPath) + } + + // absolute path + // UNC -- file://\\host/share/whatever + // drive -- file://c:/absolute/path + return path.Join(url.Host, urlPath) +} + func (d *DownloadClient) Get() (string, error) { // If we already have the file and it matches, then just return the target path. if verify, _ := d.VerifyChecksum(d.config.TargetPath); verify { @@ -133,49 +171,17 @@ func (d *DownloadClient) Get() (string, error) { // locally and we don't make a copy. Normally we would copy or download. log.Printf("[DEBUG] Using local file: %s", finalPath) - // FIXME: - // cwd should point to the path relative to client.json, but - // since this isn't exposed to us anywhere, we use os.Getwd() - // to figure it out. - cwd,err := os.Getwd() - if err != nil { - return "", fmt.Errorf("Unable to get working directory") - } - - // convert the actual file uri to a windowsy path - // (this logic must correspond to the same logic in common/config.go) + // transform the actual file uri to a windowsy path if we're being windowsy. if runtime.GOOS == "windows" { - const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) - - // move any extra path components that were parsed into Host due - // to UNC into the url.Path field so that it's PathSeparators get - // normalized - if len(url.Host) >= len(UNCPrefix) && url.Host[:len(UNCPrefix)] == UNCPrefix { - idx := strings.Index(url.Host[len(UNCPrefix):], string(os.PathSeparator)) - if idx > -1 { - url.Path = filepath.ToSlash(url.Host[idx+len(UNCPrefix):]) + url.Path - url.Host = url.Host[:idx+len(UNCPrefix)] - } - } - - // clean up backward-slashes since they only matter when part of a unc path - urlPath := filepath.ToSlash(url.Path) - - // semi-absolute path (current drive letter) -- file:///absolute/path - if url.Host == "" && len(urlPath) > 0 && urlPath[0] == '/' { - finalPath = path.Join(filepath.VolumeName(cwd), urlPath) - - // relative path -- file://./relative/path - // file://relative/path - } else if url.Host == "" || (len(url.Host) > 0 && url.Host[0] == '.') { - finalPath = path.Join(filepath.ToSlash(cwd), urlPath) - - // absolute path - } else { - // UNC -- file://\\host/share/whatever - // drive -- file://c:/absolute/path - finalPath = path.Join(url.Host, urlPath) + // FIXME: cwd should point to a path relative to the TEMPLATE path, + // but since this isn't exposed to us anywhere, we use os.Getwd() + // and assume the user ran packer in the same directory that + // any relative files are located at. + cwd,err := os.Getwd() + if err != nil { + return "", fmt.Errorf("Unable to get working directory") } + finalPath = NormalizeWindowsURL(cwd, *url) } // Keep track of the source so we can make sure not to delete this later diff --git a/common/download_test.go b/common/download_test.go index 497a0564990..b46d81455c6 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -9,6 +9,8 @@ import ( "net/http/httptest" "os" "runtime" + "strings" + "path/filepath" "testing" ) @@ -382,5 +384,106 @@ func TestDownloadFileUrl(t *testing.T) { if _, err = os.Stat(sourcePath); err != nil { t.Errorf("Could not stat source file: %s", sourcePath) } +} +// SimulateFileUriDownload is a simple utility function that converts a uri +// into a testable file path whilst ignoring a correct checksum match, stripping +// UNC path info, and then calling stat to ensure the correct file exists. +// (used by TestFileUriTransforms) +func SimulateFileUriDownload(t *testing.T, uri string) (string,error) { + // source_path is a file path and source is a network path + source := fmt.Sprintf(uri) + t.Logf("Trying to download %s", source) + + config := &DownloadConfig{ + Url: source, + // This should be wrong. We want to make sure we don't delete + Checksum: []byte("nope"), + Hash: HashForType("sha256"), + CopyFile: false, + } + + // go go go + client := NewDownloadClient(config) + path, err := client.Get() + + // ignore any non-important checksum errors if it's not a unc path + if !strings.HasPrefix(path, "\\\\") && err.Error() != "checksums didn't match expected: 6e6f7065" { + t.Fatalf("Unexpected failure; expected checksum not to match") + } + + // if it's a unc path, then remove the host and share name so we don't have + // to force the user to enable ADMIN$ and Windows File Sharing + if strings.HasPrefix(path, "\\\\") { + res := strings.SplitN(path, "/", 3) + path = "/" + res[2] + } + + if _, err = os.Stat(path); err != nil { + t.Errorf("Could not stat source file: %s", path) + } + return path,err +} + +// TestFileUriTransforms tests the case where we use a local file uri +// for iso_url. There's a few different formats that a file uri can exist as +// and so we try to test the most useful and common ones. +func TestFileUriTransforms(t *testing.T) { + const testpath = /* have your */ "test-fixtures/fileurl/cake" /* and eat it too */ + const host = "localhost" + + var cwd string + var volume string + var share string + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("Unable to detect working directory: %s", err) + return + } + cwd = filepath.ToSlash(cwd) + volume = filepath.VolumeName(cwd) + share = volume + if share[len(share)-1] == ':' { + share = share[:len(share)-1] + "$" + } + cwd = cwd[len(volume):] + + t.Logf("TestFileUriTransforms : Running with cwd : '%s'", cwd) + t.Logf("TestFileUriTransforms : Running with volume : '%s'", volume) + + // ./relative/path -> ./relative/path + // /absolute/path -> /absolute/path + // c:/windows/absolute -> c:/windows/absolute + // \\host/sharename/file -> \\host/sharename/file + testcases := []string{ + "./%s", + cwd + "/%s", + volume + cwd + "/%s", + "\\\\" + host + "/" + share + "/" + cwd[1:] + "/%s", + } + + // all regular slashed testcases + for _,testcase := range testcases { + uri := "file://" + fmt.Sprintf(testcase, testpath) + t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) + res,err := SimulateFileUriDownload(t, uri) + if err != nil { + t.Errorf("Unable to transform uri '%s' into a path : %v", uri, err) + } + t.Errorf("TestFileUriTransforms : Result Path '%s'", res) + } + + // ...and finally the oddball windows native path + // \\host\sharename\file -> \\host/sharename/file + testpath_native := filepath.FromSlash(testpath) + testcase_native := "\\\\" + host + "\\" + share + "\\" + filepath.FromSlash(cwd[1:]) + "\\%s" + uri := "file://" + fmt.Sprintf(testcase_native, testpath_native) + t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) + res,err := SimulateFileUriDownload(t, uri) + if err != nil { + t.Errorf("Unable to transform uri '%s' into a path", uri) + return + } + t.Errorf("TestFileUriTransforms : Result Path '%s'", res) } From e64a86910bade00c8b83ce692c8e0a83c789f55a Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Tue, 5 Apr 2016 13:11:30 -0500 Subject: [PATCH 03/18] Added the file, ftp, and smb downloaders to common/download.go --- common/download.go | 353 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 313 insertions(+), 40 deletions(-) diff --git a/common/download.go b/common/download.go index 65d72d22a54..b0550927f9f 100644 --- a/common/download.go +++ b/common/download.go @@ -12,7 +12,6 @@ import ( "hash" "io" "log" - "net/http" "net/url" "os" "runtime" @@ -21,6 +20,13 @@ import ( "strings" ) +import ( + "net/http" + "github.com/jlaffeye/ftp" + "bufio" +) + + // DownloadConfig is the configuration given to instantiate a new // download instance. Once a configuration is used to instantiate // a download client, it must not be modified. @@ -78,10 +84,15 @@ func HashForType(t string) hash.Hash { // NewDownloadClient returns a new DownloadClient for the given // configuration. func NewDownloadClient(c *DownloadConfig) *DownloadClient { + const mtu = 1500 /* ethernet */ - 20 /* ipv4 */ - 20 /* tcp */ + if c.DownloaderMap == nil { c.DownloaderMap = map[string]Downloader{ + "file": &FileDownloader{bufferSize: nil}, + "ftp": &FTPDownloader{userInfo: url.Userinfo{username:"anonymous", password: "anonymous@"}, mtu: mtu}, "http": &HTTPDownloader{userAgent: c.UserAgent}, "https": &HTTPDownloader{userAgent: c.UserAgent}, + "smb": &SMBDownloader{bufferSize: nil} } } @@ -101,44 +112,6 @@ func (d *DownloadClient) Cancel() { // TODO(mitchellh): Implement } -// Take a uri and convert it to a path that makes sense on the Windows platform -func NormalizeWindowsURL(basepath string, url url.URL) string { - // This logic must correspond to the same logic in the NormalizeWindowsURL - // function found in common/config.go since that function _also_ checks that - // the url actually exists in file form. - - const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) - - // move any extra path components that were parsed into Host due - // to UNC into the url.Path field so that it's PathSeparators get - // normalized - if len(url.Host) >= len(UNCPrefix) && url.Host[:len(UNCPrefix)] == UNCPrefix { - idx := strings.Index(url.Host[len(UNCPrefix):], string(os.PathSeparator)) - if idx > -1 { - url.Path = filepath.ToSlash(url.Host[idx+len(UNCPrefix):]) + url.Path - url.Host = url.Host[:idx+len(UNCPrefix)] - } - } - - // clean up backward-slashes since they only matter when part of a unc path - urlPath := filepath.ToSlash(url.Path) - - // semi-absolute path (current drive letter) -- file:///absolute/path - if url.Host == "" && len(urlPath) > 0 && urlPath[0] == '/' { - return path.Join(filepath.VolumeName(basepath), urlPath) - - // relative path -- file://./relative/path - // file://relative/path - } else if url.Host == "" || (len(url.Host) > 0 && url.Host[0] == '.') { - return path.Join(filepath.ToSlash(basepath), urlPath) - } - - // absolute path - // UNC -- file://\\host/share/whatever - // drive -- file://c:/absolute/path - return path.Join(url.Host, urlPath) -} - func (d *DownloadClient) Get() (string, error) { // If we already have the file and it matches, then just return the target path. if verify, _ := d.VerifyChecksum(d.config.TargetPath); verify { @@ -153,6 +126,11 @@ func (d *DownloadClient) Get() (string, error) { log.Printf("Parsed URL: %#v", u) + /* FIXME: + handle the special case of d.config.CopyFile which returns the path + in an os-specific format. + */ + // Files when we don't copy the file are special cased. var f *os.File var finalPath string @@ -271,7 +249,7 @@ func (*HTTPDownloader) Cancel() { } func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { - log.Printf("Starting download: %s", src.String()) + log.Printf("Starting download over HTTP: %s", src.String()) // Seek to the beginning by default if _, err := dst.Seek(0, 0); err != nil { @@ -350,3 +328,298 @@ func (d *HTTPDownloader) Progress() uint { func (d *HTTPDownloader) Total() uint { return d.total } + +// FTPDownloader is an implementation of Downloader that downloads +// files over FTP. +type FTPDownloader struct { + userInfo url.UserInfo + mtu uint + + active bool + progress uint + total uint +} + +func (*FTPDownloader) Cancel() { + d.active = false +} + +func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { + var userinfo *url.Userinfo + + userinfo = d.userInfo + d.active = false + + // check the uri is correct + uri, err := url.Parse(src) + if err != nil { return err } + + if uri.Scheme != "ftp" { + return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) + } + + // connect to ftp server + var cli *ftp.ServerConn + + log.Printf("Starting download over FTP: %s : %s\n", uri.Host, Uri.Path) + cli,err := ftp.Dial(uri.Host) + if err != nil { return nil } + defer cli.Close() + + // handle authentication + if uri.User != nil { userinfo = uri.User } + + log.Printf("Authenticating to FTP server: %s : %s\n", uri.User.username, uri.User.password) + err = cli.Login(userinfo.username, userinfo.password) + if err != nil { return err } + + // locate specified path + path := path.Dir(uri.Path) + + log.Printf("Changing to FTP directory : %s\n", path) + err = cli.ChangeDir(path) + if err != nil { return nil } + + curpath,err := cli.CurrentDir() + if err != nil { return err } + log.Printf("Current FTP directory : %s\n", curpath) + + // collect stats about the specified file + var name string + var entry *ftp.Entry + + _,name = path.Split(uri.Path) + entry = nil + + entries,err := cli.List(curpath) + for _,e := range entries { + if e.Type == ftp.EntryTypeFile && e.Name == name { + entry = e + break + } + } + + if entry == nil { + return fmt.Errorf("Unable to find file: %s", uri.Path) + } + log.Printf("Found file : %s : %v bytes\n", entry.Name, entry.Size) + + d.progress = 0 + d.total = entry.Size + + // download specified file + d.active = true + reader,err := cli.RetrFrom(uri.Path, d.progress) + if err != nil { return nil } + + // do it in a goro so that if someone wants to cancel it, they can + errch := make(chan error) + go func(d *FTPDownloader, r *io.Reader, w *bufio.Writer, e chan error) { + defer w.Flush() + for ; d.active { + n,err := io.CopyN(writer, reader, d.mtu) + if err != nil { break } + d.progress += n + } + d.active = false + e <- err + }(d, reader, bufio.NewWriter(dst), errch) + + // spin until it's done + err = <-errch + reader.Close() + + if err == nil && d.progress != d.total { + err = fmt.Errorf("FTP total transfer size was %d when %d was expected", d.progress, d.total) + } + + // log out and quit + cli.Logout() + cli.Quit() + return err +} + +func (d *FTPDownloader) Progress() uint { + return d.progress +} + +func (d *FTPDownloader) Total() uint { + return d.total +} + +// FileDownloader is an implementation of Downloader that downloads +// files using the regular filesystem. +type FileDownloader struct { + bufferSize *uint + + active bool + progress uint + total uint +} + +func (*FileDownloader) Cancel() { + d.active = false +} + +func (d *FileDownloader) Progress() uint { + return d.progress +} + +func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { + d.active = false + + /* parse the uri using the net/url module */ + uri, err := url.Parse(src) + if uri.Scheme != "file" { + return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) + } + + /* use the current working directory as the base for relative uri's */ + cwd,err := os.Getwd() + if err != nil { + return "", fmt.Errorf("Unable to get working directory") + } + + /* determine which uri format is being used and convert to a real path */ + var realpath string, basepath string + basepath = filepath.ToSlash(cwd) + + // absolute path -- file://c:/absolute/path + if strings.HasSuffix(uri.Host, ":") { + realpath = path.Join(uri.Host, uri.Path) + + // semi-absolute path (current drive letter) -- file:///absolute/path + } else if uri.Host == "" && strings.HasPrefix(uri.Path, "/") { + realpath = path.Join(filepath.VolumeName(basepath), uri.Path) + + // relative path -- file://./relative/path + } else if uri.Host == "." { + realpath = path.Join(basepath, uri.Path) + + // relative path -- file://relative/path + } else { + realpath = path.Join(basepath, uri.Host, uri.Path) + } + + /* download the file using the operating system's facilities */ + d.progress = 0 + d.active = true + + f, err = os.Open(realpath) + if err != nil { return err } + defer f.Close() + + // get the file size + fi, err := f.Stat() + if err != nil { return err } + d.total = fi.Size() + + // no bufferSize specified, so copy synchronously. + if d.bufferSize == nil { + n,err := io.Copy(dst, f) + d.active = false + d.progress += n + + // use a goro in case someone else wants to enable cancel/resume + } else { + errch := make(chan error) + go func(d* FileDownloader, r *bufio.Reader, w *bufio.Writer, e chan error) { + defer w.Flush() + for ; d.active { + n,err := io.CopyN(writer, reader, d.bufferSize) + if err != nil { break } + d.progress += n + } + d.active = false + e <- err + }(d, f, bufio.NewWriter(dst), errch) + + // ...and we spin until it's done + err = <-errch + } + f.Close() + return err +} + +func (d *FileDownloader) Total() uint { + return d.total +} + +// SMBDownloader is an implementation of Downloader that downloads +// files using the "\\" path format on Windows +type SMBDownloader struct { + bufferSize *uint + + active bool + progress uint + total uint +} + +func (*SMBDownloader) Cancel() { + d.active = false +} + +func (d *SMBDownloader) Progress() uint { + return d.progress +} + +func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { + const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) + d.active = false + + if runtime.GOOS != "windows" { + return fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) + } + + /* convert the uri using the net/url module to a UNC path */ + var realpath string + uri, err := url.Parse(src) + if uri.Scheme != "smb" { + return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) + } + + realpath = UNCPrefix + filepath.ToSlash(path.Join(uri.Host, uri.Path)) + + /* Open up the "\\"-prefixed path using the Windows filesystem */ + d.progress = 0 + d.active = true + + f, err = os.Open(realpath) + if err != nil { return err } + defer f.Close() + + // get the file size (at the risk of performance) + fi, err := f.Stat() + if err != nil { return err } + d.total = fi.Size() + + // no bufferSize specified, so copy synchronously. + if d.bufferSize == nil { + n,err := io.Copy(dst, f) + d.active = false + d.progress += n + + // use a goro in case someone else wants to enable cancel/resume + } else { + errch := make(chan error) + go func(d* SMBDownloader, r *bufio.Reader, w *bufio.Writer, e chan error) { + defer w.Flush() + for ; d.active { + n,err := io.CopyN(writer, reader, d.bufferSize) + if err != nil { break } + d.progress += n + } + d.active = false + e <- err + }(d, f, bufio.NewWriter(dst), errch) + + // ...and as usual we spin until it's done + err = <-errch + } + f.Close() + return err +} + +func (d *SMBDownloader) Total() uint { + return d.total +} From aa325249f3c3d2b1715053b0ca9dc04c23dc74dd Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Tue, 5 Apr 2016 15:01:06 -0500 Subject: [PATCH 04/18] Refactored the code a bit to move the CopyFile hack out of DownloadClient and instead into each protocol. config.go: Removed all of the windows-specific net/url hackery since it's now handled mostly by download.go Removed the replacement of '\' with '/' since url.Parse does it now. Added knowledge of the other protocols implemented in download.go (ftp, smb) Removed some modules that were unused in this commit. download.go: Moved the file-path conversions for the different protocols into their own internally callable functions. Shuffled some of the functions around in case someone wants to implement the ability to resume. Modified DownloadClient.Get to remove the CopyFile special case and trust the protocol implementations if a user doesn't want to copy the file. Since all the protocols except for HTTPDownloader implement Cancel, added a Resume method as a placeholder for another developer to implement. Added a few missing names from their function definitions. Fixed the syntax in a few lines due to my suckage at go. Adjusted the types for progress and total so that they support 64-bit sizes. Removed the usage of the bufio library since it wasn't really being used. --- common/config.go | 121 +++----------- common/download.go | 383 +++++++++++++++++++++++---------------------- 2 files changed, 220 insertions(+), 284 deletions(-) diff --git a/common/config.go b/common/config.go index 93a54922191..6b6d635d803 100644 --- a/common/config.go +++ b/common/config.go @@ -2,10 +2,8 @@ package common import ( "fmt" - "net/url" "os" "path/filepath" - "runtime" "strings" "time" ) @@ -47,115 +45,38 @@ func ChooseString(vals ...string) string { // a completely valid URL. For example, the original URL might be "local/file.iso" // which isn't a valid URL. DownloadableURL will return "file:///local/file.iso" func DownloadableURL(original string) (string, error) { - if runtime.GOOS == "windows" { - // If the distance to the first ":" is just one character, assume - // we're dealing with a drive letter and thus a file path. - idx := strings.Index(original, ":") - if idx == 1 { - original = "file://" + filepath.ToSlash(original) - } - } - - // XXX: The validation here is later re-parsed in common/download.go and - // thus any modifications here must remain consistent over there too. - uri, err := url.Parse(original) - if err != nil { - return "", err - } - - if uri.Scheme == "" { - uri.Scheme = "file" - } - - const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) - if uri.Scheme == "file" { - var ospath string // os-formatted pathname - if runtime.GOOS == "windows" { - // Move any extra path components that were mis-parsed into the Host - // field back into the uri.Path field - if len(uri.Host) >= len(UNCPrefix) && uri.Host[:len(UNCPrefix)] == UNCPrefix { - idx := strings.Index(uri.Host[len(UNCPrefix):], string(os.PathSeparator)) - if idx > -1 { - uri.Path = filepath.ToSlash(uri.Host[idx+len(UNCPrefix):]) + uri.Path - uri.Host = uri.Host[:idx+len(UNCPrefix)] - } - } - // Now all we need to do to convert the uri to a platform-specific path - // is to trade it's slashes for some os.PathSeparator ones. - ospath = uri.Host + filepath.FromSlash(uri.Path) - - } else { - // Since we're already using sane paths on a sane platform, anything in - // uri.Host can be assumed that the user is describing a relative uri. - // This means that if we concatenate it with uri.Path, the filepath - // transform will still open the file correctly. - // i.e. file://localdirectory/filename -> localdirectory/filename - ospath = uri.Host + uri.Path - } - - // Only do the filepath transformations if the file appears - // to actually exist. We don't do it on windows, because EvalSymlinks - // won't understand how to handle UNC paths and other Windows-specific minutae. - if _, err := os.Stat(ospath); err == nil && runtime.GOOS != "windows" { - ospath, err = filepath.Abs(ospath) - if err != nil { - return "", err - } - - ospath, err = filepath.EvalSymlinks(ospath) - if err != nil { - return "", err - } - - ospath = filepath.Clean(ospath) - } - - // now that ospath was normalized and such.. - if runtime.GOOS == "windows" { - uri.Host = "" - // Check to see if our ospath is unc-prefixed, and if it is then split - // the UNC host into uri.Host, leaving the rest in ospath. - // This way, our UNC-uri is protected from injury in the call to uri.String() - if len(ospath) >= len(UNCPrefix) && ospath[:len(UNCPrefix)] == UNCPrefix { - idx := strings.Index(ospath[len(UNCPrefix):], string(os.PathSeparator)) - if idx > -1 { - uri.Host = ospath[:len(UNCPrefix)+idx] - ospath = ospath[len(UNCPrefix)+idx:] - } - } - // Restore the uri by re-transforming our os-formatted path - uri.Path = filepath.ToSlash(ospath) - } else { - uri.Host = "" - uri.Path = filepath.ToSlash(ospath) - } - } - - // Make sure it is lowercased - uri.Scheme = strings.ToLower(uri.Scheme) // Verify that the scheme is something we support in our common downloader. - supported := []string{"file", "http", "https"} + supported := []string{"file", "http", "https", "ftp", "smb"} found := false for _, s := range supported { - if uri.Scheme == s { + if strings.HasPrefix(original, s + "://") { found = true break } } - if !found { - return "", fmt.Errorf("Unsupported URL scheme: %s", uri.Scheme) + // If it's properly prefixed with something we support, then we don't need + // to make it a uri. + if found { + return original, nil } - // explicit check to see if we need to manually replace the uri host with a UNC one - if runtime.GOOS == "windows" && uri.Scheme == "file" { - if len(uri.Host) >= len(UNCPrefix) && uri.Host[:len(UNCPrefix)] == UNCPrefix { - escapedHost := url.QueryEscape(uri.Host) - return strings.Replace(uri.String(), escapedHost, uri.Host, 1), nil - } + // If the file exists, then make it an absolute path + _,err := os.Stat(original) + if err == nil { + original, err = filepath.Abs(filepath.FromSlash(original)) + if err != nil { return "", err } + + original, err = filepath.EvalSymlinks(original) + if err != nil { return "", err } + + original = filepath.Clean(original) + original = filepath.ToSlash(original) } - // otherwise, we can trust the url handler - return uri.String(), nil + // Since it wasn't properly prefixed, let's make it into a well-formed + // file:// uri. + + return "file://" + original, nil } diff --git a/common/download.go b/common/download.go index b0550927f9f..c06f385e8af 100644 --- a/common/download.go +++ b/common/download.go @@ -10,20 +10,20 @@ import ( "errors" "fmt" "hash" - "io" "log" "net/url" "os" "runtime" "path" - "path/filepath" "strings" ) +// imports related to each Downloader implementation import ( + "io" + "path/filepath" "net/http" - "github.com/jlaffeye/ftp" - "bufio" + "github.com/jlaffaye/ftp" ) @@ -89,23 +89,35 @@ func NewDownloadClient(c *DownloadConfig) *DownloadClient { if c.DownloaderMap == nil { c.DownloaderMap = map[string]Downloader{ "file": &FileDownloader{bufferSize: nil}, - "ftp": &FTPDownloader{userInfo: url.Userinfo{username:"anonymous", password: "anonymous@"}, mtu: mtu}, + "ftp": &FTPDownloader{userInfo: url.UserPassword("anonymous", "anonymous@"), mtu: mtu}, "http": &HTTPDownloader{userAgent: c.UserAgent}, "https": &HTTPDownloader{userAgent: c.UserAgent}, - "smb": &SMBDownloader{bufferSize: nil} + "smb": &SMBDownloader{bufferSize: nil}, } } return &DownloadClient{config: c} } -// A downloader is responsible for actually taking a remote URL and -// downloading it. +// A downloader implements the ability to transfer a file, and cancel or resume +// it. type Downloader interface { + Resume() Cancel() + Progress() uint64 + Total() uint64 +} + +// A LocalDownloader is responsible for converting a uri to a local path +// that the platform can open directly. +type LocalDownloader interface { + toPath(string, url.URL) (string,error) +} + +// A RemoteDownloader is responsible for actually taking a remote URL and +// downloading it. +type RemoteDownloader interface { Download(*os.File, *url.URL) error - Progress() uint - Total() uint } func (d *DownloadClient) Cancel() { @@ -119,75 +131,54 @@ func (d *DownloadClient) Get() (string, error) { return d.config.TargetPath, nil } + /* parse the configuration url into a net/url object */ u, err := url.Parse(d.config.Url) - if err != nil { - return "", err - } - + if err != nil { return "", err } log.Printf("Parsed URL: %#v", u) - /* FIXME: - handle the special case of d.config.CopyFile which returns the path - in an os-specific format. - */ + /* use the current working directory as the base for relative uri's */ + cwd,err := os.Getwd() + if err != nil { return "", err } - // Files when we don't copy the file are special cased. - var f *os.File + // Determine which is the correct downloader to use var finalPath string - sourcePath := "" - if u.Scheme == "file" && !d.config.CopyFile { - // This is special case for relative path in this case user specify - // file:../ and after parse destination goes to Opaque - if u.Path != "" { - // If url.Path is set just use this - finalPath = u.Path - } else if u.Opaque != "" { - // otherwise try url.Opaque - finalPath = u.Opaque - } - // This is a special case where we use a source file that already exists - // locally and we don't make a copy. Normally we would copy or download. - log.Printf("[DEBUG] Using local file: %s", finalPath) - // transform the actual file uri to a windowsy path if we're being windowsy. - if runtime.GOOS == "windows" { - // FIXME: cwd should point to a path relative to the TEMPLATE path, - // but since this isn't exposed to us anywhere, we use os.Getwd() - // and assume the user ran packer in the same directory that - // any relative files are located at. - cwd,err := os.Getwd() - if err != nil { - return "", fmt.Errorf("Unable to get working directory") - } - finalPath = NormalizeWindowsURL(cwd, *url) - } + var ok bool + d.downloader, ok = d.config.DownloaderMap[u.Scheme] + if !ok { + return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme) + } - // Keep track of the source so we can make sure not to delete this later - sourcePath = finalPath - if _, err = os.Stat(finalPath); err != nil { - return "", err - } - } else { - finalPath = d.config.TargetPath + remote,ok := d.downloader.(RemoteDownloader) + if !ok { + return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader : %T", u.Scheme, d.downloader) + } - var ok bool - d.downloader, ok = d.config.DownloaderMap[u.Scheme] - if !ok { - return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme) - } + local,ok := d.downloader.(LocalDownloader) + if !ok && !d.config.CopyFile{ + return "", fmt.Errorf("Not allowed to use uri scheme %s in no copy file mode : %T", u.Scheme, d.downloader) + } + + // If we're copying the file, then just use the actual downloader + if d.config.CopyFile { + var f *os.File + finalPath = d.config.TargetPath - // Otherwise, download using the downloader. f, err = os.OpenFile(finalPath, os.O_RDWR|os.O_CREATE, os.FileMode(0666)) - if err != nil { - return "", err - } + if err != nil { return "", err } log.Printf("[DEBUG] Downloading: %s", u.String()) - err = d.downloader.Download(f, u) + err = remote.Download(f, u) f.Close() - if err != nil { - return "", err - } + if err != nil { return "", err } + + // Otherwise if our Downloader is a LocalDownloader we can just use the + // path after transforming it. + } else { + finalPath,err = local.toPath(cwd, *u) + if err != nil { return "", err } + + log.Printf("[DEBUG] Using local file: %s", finalPath) } if d.config.Hash != nil { @@ -195,9 +186,7 @@ func (d *DownloadClient) Get() (string, error) { verify, err = d.VerifyChecksum(finalPath) if err == nil && !verify { // Only delete the file if we made a copy or downloaded it - if sourcePath != finalPath { - os.Remove(finalPath) - } + if d.config.CopyFile { os.Remove(finalPath) } err = fmt.Errorf( "checksums didn't match expected: %s", @@ -210,10 +199,7 @@ func (d *DownloadClient) Get() (string, error) { // PercentProgress returns the download progress as a percentage. func (d *DownloadClient) PercentProgress() int { - if d.downloader == nil { - return -1 - } - + if d.downloader == nil { return -1 } return int((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100) } @@ -239,12 +225,16 @@ func (d *DownloadClient) VerifyChecksum(path string) (bool, error) { // HTTPDownloader is an implementation of Downloader that downloads // files over HTTP. type HTTPDownloader struct { - progress uint - total uint + progress uint64 + total uint64 userAgent string } -func (*HTTPDownloader) Cancel() { +func (d *HTTPDownloader) Cancel() { + // TODO(mitchellh): Implement +} + +func (d *HTTPDownloader) Resume() { // TODO(mitchellh): Implement } @@ -285,7 +275,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { if fi, err := dst.Stat(); err == nil { if _, err = dst.Seek(0, os.SEEK_END); err == nil { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) - d.progress = uint(fi.Size()) + d.progress = uint64(fi.Size()) } } } @@ -299,7 +289,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return err } - d.total = d.progress + uint(resp.ContentLength) + d.total = d.progress + uint64(resp.ContentLength) var buffer [4096]byte for { n, err := resp.Body.Read(buffer[:]) @@ -307,7 +297,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return err } - d.progress += uint(n) + d.progress += uint64(n) if _, werr := dst.Write(buffer[:n]); werr != nil { return werr @@ -321,29 +311,41 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return nil } -func (d *HTTPDownloader) Progress() uint { +func (d *HTTPDownloader) Progress() uint64 { return d.progress } -func (d *HTTPDownloader) Total() uint { +func (d *HTTPDownloader) Total() uint64 { return d.total } // FTPDownloader is an implementation of Downloader that downloads // files over FTP. type FTPDownloader struct { - userInfo url.UserInfo + userInfo *url.Userinfo mtu uint active bool - progress uint - total uint + progress uint64 + total uint64 +} + +func (d *FTPDownloader) Progress() uint64 { + return d.progress } -func (*FTPDownloader) Cancel() { +func (d *FTPDownloader) Total() uint64 { + return d.total +} + +func (d *FTPDownloader) Cancel() { d.active = false } +func (d *FTPDownloader) Resume() { + // TODO: Implement +} + func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { var userinfo *url.Userinfo @@ -351,33 +353,34 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { d.active = false // check the uri is correct - uri, err := url.Parse(src) - if err != nil { return err } - - if uri.Scheme != "ftp" { - return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) + if src == nil || src.Scheme != "ftp" { + return fmt.Errorf("Unexpected uri scheme: %s", src.Scheme) } + uri := src // connect to ftp server var cli *ftp.ServerConn - log.Printf("Starting download over FTP: %s : %s\n", uri.Host, Uri.Path) + log.Printf("Starting download over FTP: %s : %s\n", uri.Host, uri.Path) cli,err := ftp.Dial(uri.Host) if err != nil { return nil } - defer cli.Close() + defer cli.Quit() // handle authentication if uri.User != nil { userinfo = uri.User } - log.Printf("Authenticating to FTP server: %s : %s\n", uri.User.username, uri.User.password) - err = cli.Login(userinfo.username, userinfo.password) + pass,ok := userinfo.Password() + if !ok { pass = "ftp@" } + + log.Printf("Authenticating to FTP server: %s : %s\n", userinfo.Username(), pass) + err = cli.Login(userinfo.Username(), pass) if err != nil { return err } // locate specified path - path := path.Dir(uri.Path) + p := path.Dir(uri.Path) - log.Printf("Changing to FTP directory : %s\n", path) - err = cli.ChangeDir(path) + log.Printf("Changing to FTP directory : %s\n", p) + err = cli.ChangeDir(p) if err != nil { return nil } curpath,err := cli.CurrentDir() @@ -393,7 +396,7 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { entries,err := cli.List(curpath) for _,e := range entries { - if e.Type == ftp.EntryTypeFile && e.Name == name { + if e.Type == ftp.EntryTypeFile && e.Name == name { entry = e break } @@ -414,16 +417,15 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { // do it in a goro so that if someone wants to cancel it, they can errch := make(chan error) - go func(d *FTPDownloader, r *io.Reader, w *bufio.Writer, e chan error) { - defer w.Flush() - for ; d.active { - n,err := io.CopyN(writer, reader, d.mtu) + go func(d *FTPDownloader, r io.Reader, w io.Writer, e chan error) { + for ; d.active; { + n,err := io.CopyN(w, r, int64(d.mtu)) if err != nil { break } - d.progress += n + d.progress += uint64(n) } d.active = false e <- err - }(d, reader, bufio.NewWriter(dst), errch) + }(d, reader, dst, errch) // spin until it's done err = <-errch @@ -433,106 +435,109 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { err = fmt.Errorf("FTP total transfer size was %d when %d was expected", d.progress, d.total) } - // log out and quit + // log out cli.Logout() - cli.Quit() return err } -func (d *FTPDownloader) Progress() uint { - return d.progress -} - -func (d *FTPDownloader) Total() uint { - return d.total -} - // FileDownloader is an implementation of Downloader that downloads // files using the regular filesystem. type FileDownloader struct { bufferSize *uint active bool - progress uint - total uint + progress uint64 + total uint64 } -func (*FileDownloader) Cancel() { - d.active = false +func (d *FileDownloader) Progress() uint64 { + return d.progress } -func (d *FileDownloader) Progress() uint { - return d.progress +func (d *FileDownloader) Total() uint64 { + return d.total } -func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { +func (d *FileDownloader) Cancel() { d.active = false +} - /* parse the uri using the net/url module */ - uri, err := url.Parse(src) - if uri.Scheme != "file" { - return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) - } +func (d *FileDownloader) Resume() { + // TODO: Implement +} - /* use the current working directory as the base for relative uri's */ - cwd,err := os.Getwd() - if err != nil { - return "", fmt.Errorf("Unable to get working directory") - } +func (d *FileDownloader) toPath(base string, uri url.URL) (string,error) { + var result string - /* determine which uri format is being used and convert to a real path */ - var realpath string, basepath string - basepath = filepath.ToSlash(cwd) - - // absolute path -- file://c:/absolute/path + // absolute path -- file://c:/absolute/path -> c:/absolute/path if strings.HasSuffix(uri.Host, ":") { - realpath = path.Join(uri.Host, uri.Path) + result = path.Join(uri.Host, uri.Path) - // semi-absolute path (current drive letter) -- file:///absolute/path + // semi-absolute path (current drive letter) + // -- file:///absolute/path -> /absolute/path } else if uri.Host == "" && strings.HasPrefix(uri.Path, "/") { - realpath = path.Join(filepath.VolumeName(basepath), uri.Path) + result = path.Join(filepath.VolumeName(base), uri.Path) - // relative path -- file://./relative/path + // relative path -- file://./relative/path -> ./relative/path } else if uri.Host == "." { - realpath = path.Join(basepath, uri.Path) + result = path.Join(base, uri.Path) - // relative path -- file://relative/path + // relative path -- file://relative/path -> ./relative/path } else { - realpath = path.Join(basepath, uri.Host, uri.Path) + result = path.Join(base, uri.Host, uri.Path) + } + return filepath.ToSlash(result),nil +} + +func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { + d.active = false + + /* check the uri's scheme to make sure it matches */ + if src == nil || src.Scheme != "file" { + return fmt.Errorf("Unexpected uri scheme: %s", src.Scheme) } + uri := src + + /* use the current working directory as the base for relative uri's */ + cwd,err := os.Getwd() + if err != nil { return err } + + /* determine which uri format is being used and convert to a real path */ + realpath,err := d.toPath(cwd, *uri) + if err != nil { return err } /* download the file using the operating system's facilities */ d.progress = 0 d.active = true - f, err = os.Open(realpath) + f, err := os.Open(realpath) if err != nil { return err } defer f.Close() // get the file size fi, err := f.Stat() if err != nil { return err } - d.total = fi.Size() + d.total = uint64(fi.Size()) // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { - n,err := io.Copy(dst, f) + var n int64 + n,err = io.Copy(dst, f) d.active = false - d.progress += n + d.progress += uint64(n) // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) - go func(d* FileDownloader, r *bufio.Reader, w *bufio.Writer, e chan error) { - defer w.Flush() - for ; d.active { - n,err := io.CopyN(writer, reader, d.bufferSize) + go func(d* FileDownloader, r io.Reader, w io.Writer, e chan error) { + for ; d.active; { + n,err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } - d.progress += n + d.progress += uint64(n) } d.active = false e <- err - }(d, f, bufio.NewWriter(dst), errch) + }(d, f, dst, errch) // ...and we spin until it's done err = <-errch @@ -541,77 +546,91 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { return err } -func (d *FileDownloader) Total() uint { - return d.total -} - // SMBDownloader is an implementation of Downloader that downloads // files using the "\\" path format on Windows type SMBDownloader struct { bufferSize *uint active bool - progress uint - total uint + progress uint64 + total uint64 +} + +func (d *SMBDownloader) Progress() uint64 { + return d.progress } -func (*SMBDownloader) Cancel() { +func (d *SMBDownloader) Total() uint64 { + return d.total +} + +func (d *SMBDownloader) Cancel() { d.active = false } -func (d *SMBDownloader) Progress() uint { - return d.progress +func (d *SMBDownloader) Resume() { + // TODO: Implement } -func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { +func (d *SMBDownloader) toPath(base string, uri url.URL) (string,error) { const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) - d.active = false if runtime.GOOS != "windows" { - return fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) + return "",fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) } + return UNCPrefix + filepath.ToSlash(path.Join(uri.Host, uri.Path)), nil +} + +func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { + d.active = false + /* convert the uri using the net/url module to a UNC path */ - var realpath string - uri, err := url.Parse(src) - if uri.Scheme != "smb" { - return fmt.Errorf("Unexpected uri scheme: %s", uri.Scheme) + if src == nil || src.Scheme != "smb" { + return fmt.Errorf("Unexpected uri scheme: %s", src.Scheme) } + uri := src - realpath = UNCPrefix + filepath.ToSlash(path.Join(uri.Host, uri.Path)) + /* use the current working directory as the base for relative uri's */ + cwd,err := os.Getwd() + if err != nil { return err } + + /* convert uri to an smb-path */ + realpath,err := d.toPath(cwd, *uri) + if err != nil { return err } /* Open up the "\\"-prefixed path using the Windows filesystem */ d.progress = 0 d.active = true - f, err = os.Open(realpath) + f, err := os.Open(realpath) if err != nil { return err } defer f.Close() // get the file size (at the risk of performance) fi, err := f.Stat() if err != nil { return err } - d.total = fi.Size() + d.total = uint64(fi.Size()) // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { - n,err := io.Copy(dst, f) + var n int64 + n,err = io.Copy(dst, f) d.active = false - d.progress += n + d.progress += uint64(n) // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) - go func(d* SMBDownloader, r *bufio.Reader, w *bufio.Writer, e chan error) { - defer w.Flush() - for ; d.active { - n,err := io.CopyN(writer, reader, d.bufferSize) + go func(d* SMBDownloader, r io.Reader, w io.Writer, e chan error) { + for ; d.active; { + n,err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } - d.progress += n + d.progress += uint64(n) } d.active = false e <- err - }(d, f, bufio.NewWriter(dst), errch) + }(d, f, dst, errch) // ...and as usual we spin until it's done err = <-errch @@ -619,7 +638,3 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { f.Close() return err } - -func (d *SMBDownloader) Total() uint { - return d.total -} From 6fa8700e06a34604f48bde3cc6cad2784ab7cfee Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Tue, 5 Apr 2016 16:51:17 -0500 Subject: [PATCH 05/18] Fixed some of the unit-tests in common/ due to the changes made in {config,download}.go config.go: Fixed some issues related to the url scheme not being lowercased which broke some of the tests. config_test.go: Removed the UNC share test for \\host\share\file since SMB support has been moved to a different uri scheme. download_test.go: Explicitly set the CopyFile configuration option for all the unit-tests that test file copying capability. Removed the UNC share testcase since it's under a different uri scheme now. Modified the file:// UNC share testcase to explicitly test the smb:// uri. Changed the incorrect t.Errorf calls to t.Logf so that the tests can pass. --- common/config.go | 13 +++++++++++-- common/config_test.go | 17 ++--------------- common/download_test.go | 21 +++++++++++++-------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/common/config.go b/common/config.go index 6b6d635d803..312e7ddc825 100644 --- a/common/config.go +++ b/common/config.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "time" + "net/url" ) // PackerKeyEnv is used to specify the key interval (delay) between keystrokes @@ -50,7 +51,7 @@ func DownloadableURL(original string) (string, error) { supported := []string{"file", "http", "https", "ftp", "smb"} found := false for _, s := range supported { - if strings.HasPrefix(original, s + "://") { + if strings.HasPrefix(strings.ToLower(original), s + "://") { found = true break } @@ -59,7 +60,15 @@ func DownloadableURL(original string) (string, error) { // If it's properly prefixed with something we support, then we don't need // to make it a uri. if found { - return original, nil + original = filepath.ToSlash(original) + + // make sure that it can be parsed though.. + uri,err := url.Parse(original) + if err != nil { return "", err } + + uri.Scheme = strings.ToLower(uri.Scheme) + + return uri.String(), nil } // If the file exists, then make it an absolute path diff --git a/common/config_test.go b/common/config_test.go index 8bd68473919..1fc3a79c23a 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" "path/filepath" - "runtime" "strings" "testing" ) @@ -41,13 +40,7 @@ func TestDownloadableURL(t *testing.T) { // Invalid URL: has hex code in host _, err := DownloadableURL("http://what%20.com") if err == nil { - t.Fatal("expected err") - } - - // Invalid: unsupported scheme - _, err = DownloadableURL("ftp://host.com/path") - if err == nil { - t.Fatal("expected err") + t.Fatalf("expected err : %s", err) } // Valid: http @@ -85,11 +78,7 @@ func TestDownloadableURL_FilePaths(t *testing.T) { } tfPath = filepath.Clean(tfPath) - filePrefix := "file://" - if runtime.GOOS == "windows" { - filePrefix += "/" - } // Relative filepath. We run this test in a func so that // the defers run right away. @@ -111,9 +100,7 @@ func TestDownloadableURL_FilePaths(t *testing.T) { t.Fatalf("err: %s", err) } - expected := fmt.Sprintf("%s%s", - filePrefix, - strings.Replace(tfPath, `\`, `/`, -1)) + expected := "file://" + strings.Replace(tfPath, `\`, `/`, -1) if u != expected { t.Fatalf("unexpected: %#v != %#v", u, expected) } diff --git a/common/download_test.go b/common/download_test.go index b46d81455c6..091d9c7d79c 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -58,6 +58,7 @@ func TestDownloadClient_basic(t *testing.T) { client := NewDownloadClient(&DownloadConfig{ Url: ts.URL + "/basic.txt", TargetPath: tf.Name(), + CopyFile: true, }) path, err := client.Get() @@ -93,6 +94,7 @@ func TestDownloadClient_checksumBad(t *testing.T) { TargetPath: tf.Name(), Hash: HashForType("md5"), Checksum: checksum, + CopyFile: true, }) if _, err := client.Get(); err == nil { t.Fatal("should error") @@ -117,6 +119,7 @@ func TestDownloadClient_checksumGood(t *testing.T) { TargetPath: tf.Name(), Hash: HashForType("md5"), Checksum: checksum, + CopyFile: true, }) path, err := client.Get() if err != nil { @@ -147,6 +150,7 @@ func TestDownloadClient_checksumNoDownload(t *testing.T) { TargetPath: "./test-fixtures/root/another.txt", Hash: HashForType("md5"), Checksum: checksum, + CopyFile: true, }) path, err := client.Get() if err != nil { @@ -185,6 +189,7 @@ func TestDownloadClient_resume(t *testing.T) { client := NewDownloadClient(&DownloadConfig{ Url: ts.URL, TargetPath: tf.Name(), + CopyFile: true, }) path, err := client.Get() if err != nil { @@ -242,6 +247,7 @@ func TestDownloadClient_usesDefaultUserAgent(t *testing.T) { config := &DownloadConfig{ Url: server.URL, TargetPath: tf.Name(), + CopyFile: true, } client := NewDownloadClient(config) @@ -273,6 +279,7 @@ func TestDownloadClient_setsUserAgent(t *testing.T) { Url: server.URL, TargetPath: tf.Name(), UserAgent: "fancy user agent", + CopyFile: true, } client := NewDownloadClient(config) @@ -353,6 +360,7 @@ func TestDownloadFileUrl(t *testing.T) { if err != nil { t.Fatalf("Unable to detect working directory: %s", err) } + cwd = filepath.ToSlash(cwd) // source_path is a file path and source is a network path sourcePath := fmt.Sprintf("%s/test-fixtures/fileurl/%s", cwd, "cake") @@ -455,12 +463,10 @@ func TestFileUriTransforms(t *testing.T) { // ./relative/path -> ./relative/path // /absolute/path -> /absolute/path // c:/windows/absolute -> c:/windows/absolute - // \\host/sharename/file -> \\host/sharename/file testcases := []string{ "./%s", cwd + "/%s", volume + cwd + "/%s", - "\\\\" + host + "/" + share + "/" + cwd[1:] + "/%s", } // all regular slashed testcases @@ -471,19 +477,18 @@ func TestFileUriTransforms(t *testing.T) { if err != nil { t.Errorf("Unable to transform uri '%s' into a path : %v", uri, err) } - t.Errorf("TestFileUriTransforms : Result Path '%s'", res) + t.Logf("TestFileUriTransforms : Result Path '%s'", res) } // ...and finally the oddball windows native path - // \\host\sharename\file -> \\host/sharename/file - testpath_native := filepath.FromSlash(testpath) - testcase_native := "\\\\" + host + "\\" + share + "\\" + filepath.FromSlash(cwd[1:]) + "\\%s" - uri := "file://" + fmt.Sprintf(testcase_native, testpath_native) + // smb://host/sharename/file -> \\host\sharename\file + testcase := host + "/" + share + "/" + cwd[1:] + "/%s" + uri := "smb://" + fmt.Sprintf(testcase, testpath) t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) res,err := SimulateFileUriDownload(t, uri) if err != nil { t.Errorf("Unable to transform uri '%s' into a path", uri) return } - t.Errorf("TestFileUriTransforms : Result Path '%s'", res) + t.Logf("TestFileUriTransforms : Result Path '%s'", res) } From fb6a6cfcb50a03dc858a5864f3b24d145f5c91df Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Sun, 12 Feb 2017 17:00:47 -0600 Subject: [PATCH 06/18] Added a progressbar using gopkg.in/cheggaaa/pb.v1 as per #3578 for all the DownloadClients in common/download.go. --- common/download.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/common/download.go b/common/download.go index c06f385e8af..96a40d4889c 100644 --- a/common/download.go +++ b/common/download.go @@ -23,6 +23,7 @@ import ( "io" "path/filepath" "net/http" + "gopkg.in/cheggaaa/pb.v1" "github.com/jlaffaye/ftp" ) @@ -157,6 +158,7 @@ func (d *DownloadClient) Get() (string, error) { local,ok := d.downloader.(LocalDownloader) if !ok && !d.config.CopyFile{ return "", fmt.Errorf("Not allowed to use uri scheme %s in no copy file mode : %T", u.Scheme, d.downloader) + d.config.CopyFile = true } // If we're copying the file, then just use the actual downloader @@ -275,7 +277,9 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { if fi, err := dst.Stat(); err == nil { if _, err = dst.Seek(0, os.SEEK_END); err == nil { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) + d.progress = uint64(fi.Size()) + progressBar.Set64(int64(d.Progress())) } } } @@ -290,6 +294,8 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { } d.total = d.progress + uint64(resp.ContentLength) + progressBar := pb.New64(int64(d.Total())).Start() + var buffer [4096]byte for { n, err := resp.Body.Read(buffer[:]) @@ -298,6 +304,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { } d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) if _, werr := dst.Write(buffer[:n]); werr != nil { return werr @@ -409,6 +416,7 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { d.progress = 0 d.total = entry.Size + progressBar := pb.New64(int64(d.Total())).Start() // download specified file d.active = true @@ -421,7 +429,9 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { for ; d.active; { n,err := io.CopyN(w, r, int64(d.mtu)) if err != nil { break } + d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) } d.active = false e <- err @@ -518,13 +528,16 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { fi, err := f.Stat() if err != nil { return err } d.total = uint64(fi.Size()) + progressBar := pb.New64(int64(d.Total())).Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { var n int64 n,err = io.Copy(dst, f) d.active = false + d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) // use a goro in case someone else wants to enable cancel/resume } else { @@ -533,7 +546,9 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { for ; d.active; { n,err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } + d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) } d.active = false e <- err @@ -611,13 +626,16 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { fi, err := f.Stat() if err != nil { return err } d.total = uint64(fi.Size()) + progressBar := pb.New64(int64(d.Total())).Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { var n int64 n,err = io.Copy(dst, f) d.active = false + d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) // use a goro in case someone else wants to enable cancel/resume } else { @@ -626,7 +644,9 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { for ; d.active; { n,err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } + d.progress += uint64(n) + progressBar.Set64(int64(d.Progress())) } d.active = false e <- err From 48d944f9a26971bfa77fc2e631cbc3b5cc618660 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 1 Mar 2017 14:35:00 -0600 Subject: [PATCH 07/18] Moved the setting of HTTPDownloader's current progress to after the object actually gets instantiated. ;) --- common/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/download.go b/common/download.go index 96a40d4889c..f8b273e8df6 100644 --- a/common/download.go +++ b/common/download.go @@ -279,7 +279,6 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) d.progress = uint64(fi.Size()) - progressBar.Set64(int64(d.Progress())) } } } @@ -295,6 +294,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { d.total = d.progress + uint64(resp.ContentLength) progressBar := pb.New64(int64(d.Total())).Start() + progressBar.Set64(int64(d.Progress())) var buffer [4096]byte for { From e21ebd5cdf2fd0217a14dcad7cc0771cca63ca51 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 1 Mar 2017 14:37:05 -0600 Subject: [PATCH 08/18] go fmt --- common/config.go | 22 +++-- common/download.go | 203 +++++++++++++++++++++++++--------------- common/download_test.go | 14 +-- 3 files changed, 147 insertions(+), 92 deletions(-) diff --git a/common/config.go b/common/config.go index 312e7ddc825..e3df84ccc93 100644 --- a/common/config.go +++ b/common/config.go @@ -2,11 +2,11 @@ package common import ( "fmt" + "net/url" "os" "path/filepath" "strings" "time" - "net/url" ) // PackerKeyEnv is used to specify the key interval (delay) between keystrokes @@ -51,7 +51,7 @@ func DownloadableURL(original string) (string, error) { supported := []string{"file", "http", "https", "ftp", "smb"} found := false for _, s := range supported { - if strings.HasPrefix(strings.ToLower(original), s + "://") { + if strings.HasPrefix(strings.ToLower(original), s+"://") { found = true break } @@ -63,8 +63,10 @@ func DownloadableURL(original string) (string, error) { original = filepath.ToSlash(original) // make sure that it can be parsed though.. - uri,err := url.Parse(original) - if err != nil { return "", err } + uri, err := url.Parse(original) + if err != nil { + return "", err + } uri.Scheme = strings.ToLower(uri.Scheme) @@ -72,16 +74,20 @@ func DownloadableURL(original string) (string, error) { } // If the file exists, then make it an absolute path - _,err := os.Stat(original) + _, err := os.Stat(original) if err == nil { original, err = filepath.Abs(filepath.FromSlash(original)) - if err != nil { return "", err } + if err != nil { + return "", err + } original, err = filepath.EvalSymlinks(original) - if err != nil { return "", err } + if err != nil { + return "", err + } original = filepath.Clean(original) - original = filepath.ToSlash(original) + original = filepath.ToSlash(original) } // Since it wasn't properly prefixed, let's make it into a well-formed diff --git a/common/download.go b/common/download.go index f8b273e8df6..110cb15365e 100644 --- a/common/download.go +++ b/common/download.go @@ -13,21 +13,20 @@ import ( "log" "net/url" "os" - "runtime" "path" + "runtime" "strings" ) // imports related to each Downloader implementation import ( + "github.com/jlaffaye/ftp" + "gopkg.in/cheggaaa/pb.v1" "io" - "path/filepath" "net/http" - "gopkg.in/cheggaaa/pb.v1" - "github.com/jlaffaye/ftp" + "path/filepath" ) - // DownloadConfig is the configuration given to instantiate a new // download instance. Once a configuration is used to instantiate // a download client, it must not be modified. @@ -112,7 +111,7 @@ type Downloader interface { // A LocalDownloader is responsible for converting a uri to a local path // that the platform can open directly. type LocalDownloader interface { - toPath(string, url.URL) (string,error) + toPath(string, url.URL) (string, error) } // A RemoteDownloader is responsible for actually taking a remote URL and @@ -134,12 +133,16 @@ func (d *DownloadClient) Get() (string, error) { /* parse the configuration url into a net/url object */ u, err := url.Parse(d.config.Url) - if err != nil { return "", err } + if err != nil { + return "", err + } log.Printf("Parsed URL: %#v", u) /* use the current working directory as the base for relative uri's */ - cwd,err := os.Getwd() - if err != nil { return "", err } + cwd, err := os.Getwd() + if err != nil { + return "", err + } // Determine which is the correct downloader to use var finalPath string @@ -150,13 +153,13 @@ func (d *DownloadClient) Get() (string, error) { return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme) } - remote,ok := d.downloader.(RemoteDownloader) + remote, ok := d.downloader.(RemoteDownloader) if !ok { return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader : %T", u.Scheme, d.downloader) } - local,ok := d.downloader.(LocalDownloader) - if !ok && !d.config.CopyFile{ + local, ok := d.downloader.(LocalDownloader) + if !ok && !d.config.CopyFile { return "", fmt.Errorf("Not allowed to use uri scheme %s in no copy file mode : %T", u.Scheme, d.downloader) d.config.CopyFile = true } @@ -167,18 +170,24 @@ func (d *DownloadClient) Get() (string, error) { finalPath = d.config.TargetPath f, err = os.OpenFile(finalPath, os.O_RDWR|os.O_CREATE, os.FileMode(0666)) - if err != nil { return "", err } + if err != nil { + return "", err + } log.Printf("[DEBUG] Downloading: %s", u.String()) err = remote.Download(f, u) f.Close() - if err != nil { return "", err } + if err != nil { + return "", err + } - // Otherwise if our Downloader is a LocalDownloader we can just use the - // path after transforming it. + // Otherwise if our Downloader is a LocalDownloader we can just use the + // path after transforming it. } else { - finalPath,err = local.toPath(cwd, *u) - if err != nil { return "", err } + finalPath, err = local.toPath(cwd, *u) + if err != nil { + return "", err + } log.Printf("[DEBUG] Using local file: %s", finalPath) } @@ -188,7 +197,9 @@ func (d *DownloadClient) Get() (string, error) { verify, err = d.VerifyChecksum(finalPath) if err == nil && !verify { // Only delete the file if we made a copy or downloaded it - if d.config.CopyFile { os.Remove(finalPath) } + if d.config.CopyFile { + os.Remove(finalPath) + } err = fmt.Errorf( "checksums didn't match expected: %s", @@ -201,7 +212,9 @@ func (d *DownloadClient) Get() (string, error) { // PercentProgress returns the download progress as a percentage. func (d *DownloadClient) PercentProgress() int { - if d.downloader == nil { return -1 } + if d.downloader == nil { + return -1 + } return int((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100) } @@ -330,11 +343,11 @@ func (d *HTTPDownloader) Total() uint64 { // files over FTP. type FTPDownloader struct { userInfo *url.Userinfo - mtu uint + mtu uint - active bool + active bool progress uint64 - total uint64 + total uint64 } func (d *FTPDownloader) Progress() uint64 { @@ -369,40 +382,52 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { var cli *ftp.ServerConn log.Printf("Starting download over FTP: %s : %s\n", uri.Host, uri.Path) - cli,err := ftp.Dial(uri.Host) - if err != nil { return nil } + cli, err := ftp.Dial(uri.Host) + if err != nil { + return nil + } defer cli.Quit() // handle authentication - if uri.User != nil { userinfo = uri.User } + if uri.User != nil { + userinfo = uri.User + } - pass,ok := userinfo.Password() - if !ok { pass = "ftp@" } + pass, ok := userinfo.Password() + if !ok { + pass = "ftp@" + } log.Printf("Authenticating to FTP server: %s : %s\n", userinfo.Username(), pass) err = cli.Login(userinfo.Username(), pass) - if err != nil { return err } + if err != nil { + return err + } // locate specified path p := path.Dir(uri.Path) log.Printf("Changing to FTP directory : %s\n", p) err = cli.ChangeDir(p) - if err != nil { return nil } + if err != nil { + return nil + } - curpath,err := cli.CurrentDir() - if err != nil { return err } + curpath, err := cli.CurrentDir() + if err != nil { + return err + } log.Printf("Current FTP directory : %s\n", curpath) // collect stats about the specified file var name string var entry *ftp.Entry - _,name = path.Split(uri.Path) + _, name = path.Split(uri.Path) entry = nil - entries,err := cli.List(curpath) - for _,e := range entries { + entries, err := cli.List(curpath) + for _, e := range entries { if e.Type == ftp.EntryTypeFile && e.Name == name { entry = e break @@ -420,15 +445,19 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { // download specified file d.active = true - reader,err := cli.RetrFrom(uri.Path, d.progress) - if err != nil { return nil } + reader, err := cli.RetrFrom(uri.Path, d.progress) + if err != nil { + return nil + } // do it in a goro so that if someone wants to cancel it, they can errch := make(chan error) go func(d *FTPDownloader, r io.Reader, w io.Writer, e chan error) { - for ; d.active; { - n,err := io.CopyN(w, r, int64(d.mtu)) - if err != nil { break } + for d.active { + n, err := io.CopyN(w, r, int64(d.mtu)) + if err != nil { + break + } d.progress += uint64(n) progressBar.Set64(int64(d.Progress())) @@ -455,9 +484,9 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { type FileDownloader struct { bufferSize *uint - active bool + active bool progress uint64 - total uint64 + total uint64 } func (d *FileDownloader) Progress() uint64 { @@ -476,27 +505,27 @@ func (d *FileDownloader) Resume() { // TODO: Implement } -func (d *FileDownloader) toPath(base string, uri url.URL) (string,error) { +func (d *FileDownloader) toPath(base string, uri url.URL) (string, error) { var result string // absolute path -- file://c:/absolute/path -> c:/absolute/path if strings.HasSuffix(uri.Host, ":") { result = path.Join(uri.Host, uri.Path) - // semi-absolute path (current drive letter) - // -- file:///absolute/path -> /absolute/path + // semi-absolute path (current drive letter) + // -- file:///absolute/path -> /absolute/path } else if uri.Host == "" && strings.HasPrefix(uri.Path, "/") { result = path.Join(filepath.VolumeName(base), uri.Path) - // relative path -- file://./relative/path -> ./relative/path + // relative path -- file://./relative/path -> ./relative/path } else if uri.Host == "." { result = path.Join(base, uri.Path) - // relative path -- file://relative/path -> ./relative/path + // relative path -- file://relative/path -> ./relative/path } else { result = path.Join(base, uri.Host, uri.Path) } - return filepath.ToSlash(result),nil + return filepath.ToSlash(result), nil } func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { @@ -509,43 +538,53 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { uri := src /* use the current working directory as the base for relative uri's */ - cwd,err := os.Getwd() - if err != nil { return err } + cwd, err := os.Getwd() + if err != nil { + return err + } /* determine which uri format is being used and convert to a real path */ - realpath,err := d.toPath(cwd, *uri) - if err != nil { return err } + realpath, err := d.toPath(cwd, *uri) + if err != nil { + return err + } /* download the file using the operating system's facilities */ d.progress = 0 d.active = true f, err := os.Open(realpath) - if err != nil { return err } + if err != nil { + return err + } defer f.Close() // get the file size fi, err := f.Stat() - if err != nil { return err } + if err != nil { + return err + } d.total = uint64(fi.Size()) progressBar := pb.New64(int64(d.Total())).Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { var n int64 - n,err = io.Copy(dst, f) + n, err = io.Copy(dst, f) d.active = false d.progress += uint64(n) progressBar.Set64(int64(d.Progress())) - // use a goro in case someone else wants to enable cancel/resume + // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) - go func(d* FileDownloader, r io.Reader, w io.Writer, e chan error) { - for ; d.active; { - n,err := io.CopyN(w, r, int64(*d.bufferSize)) - if err != nil { break } + go func(d *FileDownloader, r io.Reader, w io.Writer, e chan error) { + for d.active { + n, err := io.CopyN(w, r, int64(*d.bufferSize)) + if err != nil { + break + } d.progress += uint64(n) progressBar.Set64(int64(d.Progress())) @@ -566,9 +605,9 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { type SMBDownloader struct { bufferSize *uint - active bool + active bool progress uint64 - total uint64 + total uint64 } func (d *SMBDownloader) Progress() uint64 { @@ -587,11 +626,11 @@ func (d *SMBDownloader) Resume() { // TODO: Implement } -func (d *SMBDownloader) toPath(base string, uri url.URL) (string,error) { - const UNCPrefix = string(os.PathSeparator)+string(os.PathSeparator) +func (d *SMBDownloader) toPath(base string, uri url.URL) (string, error) { + const UNCPrefix = string(os.PathSeparator) + string(os.PathSeparator) if runtime.GOOS != "windows" { - return "",fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) + return "", fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) } return UNCPrefix + filepath.ToSlash(path.Join(uri.Host, uri.Path)), nil @@ -607,43 +646,53 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { uri := src /* use the current working directory as the base for relative uri's */ - cwd,err := os.Getwd() - if err != nil { return err } + cwd, err := os.Getwd() + if err != nil { + return err + } /* convert uri to an smb-path */ - realpath,err := d.toPath(cwd, *uri) - if err != nil { return err } + realpath, err := d.toPath(cwd, *uri) + if err != nil { + return err + } /* Open up the "\\"-prefixed path using the Windows filesystem */ d.progress = 0 d.active = true f, err := os.Open(realpath) - if err != nil { return err } + if err != nil { + return err + } defer f.Close() // get the file size (at the risk of performance) fi, err := f.Stat() - if err != nil { return err } + if err != nil { + return err + } d.total = uint64(fi.Size()) progressBar := pb.New64(int64(d.Total())).Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { var n int64 - n,err = io.Copy(dst, f) + n, err = io.Copy(dst, f) d.active = false d.progress += uint64(n) progressBar.Set64(int64(d.Progress())) - // use a goro in case someone else wants to enable cancel/resume + // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) - go func(d* SMBDownloader, r io.Reader, w io.Writer, e chan error) { - for ; d.active; { - n,err := io.CopyN(w, r, int64(*d.bufferSize)) - if err != nil { break } + go func(d *SMBDownloader, r io.Reader, w io.Writer, e chan error) { + for d.active { + n, err := io.CopyN(w, r, int64(*d.bufferSize)) + if err != nil { + break + } d.progress += uint64(n) progressBar.Set64(int64(d.Progress())) diff --git a/common/download_test.go b/common/download_test.go index 091d9c7d79c..81cc3244f9c 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -8,9 +8,9 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "runtime" "strings" - "path/filepath" "testing" ) @@ -119,7 +119,7 @@ func TestDownloadClient_checksumGood(t *testing.T) { TargetPath: tf.Name(), Hash: HashForType("md5"), Checksum: checksum, - CopyFile: true, + CopyFile: true, }) path, err := client.Get() if err != nil { @@ -398,7 +398,7 @@ func TestDownloadFileUrl(t *testing.T) { // into a testable file path whilst ignoring a correct checksum match, stripping // UNC path info, and then calling stat to ensure the correct file exists. // (used by TestFileUriTransforms) -func SimulateFileUriDownload(t *testing.T, uri string) (string,error) { +func SimulateFileUriDownload(t *testing.T, uri string) (string, error) { // source_path is a file path and source is a network path source := fmt.Sprintf(uri) t.Logf("Trying to download %s", source) @@ -430,7 +430,7 @@ func SimulateFileUriDownload(t *testing.T, uri string) (string,error) { if _, err = os.Stat(path); err != nil { t.Errorf("Could not stat source file: %s", path) } - return path,err + return path, err } // TestFileUriTransforms tests the case where we use a local file uri @@ -470,10 +470,10 @@ func TestFileUriTransforms(t *testing.T) { } // all regular slashed testcases - for _,testcase := range testcases { + for _, testcase := range testcases { uri := "file://" + fmt.Sprintf(testcase, testpath) t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) - res,err := SimulateFileUriDownload(t, uri) + res, err := SimulateFileUriDownload(t, uri) if err != nil { t.Errorf("Unable to transform uri '%s' into a path : %v", uri, err) } @@ -485,7 +485,7 @@ func TestFileUriTransforms(t *testing.T) { testcase := host + "/" + share + "/" + cwd[1:] + "/%s" uri := "smb://" + fmt.Sprintf(testcase, testpath) t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) - res,err := SimulateFileUriDownload(t, uri) + res, err := SimulateFileUriDownload(t, uri) if err != nil { t.Errorf("Unable to transform uri '%s' into a path", uri) return From 2cd6387bdc8b9ca22438ba263c34740fd5799230 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 1 Mar 2017 14:38:01 -0600 Subject: [PATCH 09/18] Added gopkg.in/cheggaaa/pb.v1 to vendor/vendor.json via govendor. --- vendor/gopkg.in/cheggaaa/pb.v1/LICENSE | 12 + vendor/gopkg.in/cheggaaa/pb.v1/README.md | 176 +++++++ vendor/gopkg.in/cheggaaa/pb.v1/format.go | 87 ++++ vendor/gopkg.in/cheggaaa/pb.v1/pb.go | 444 ++++++++++++++++++ vendor/gopkg.in/cheggaaa/pb.v1/pb_nix.go | 8 + vendor/gopkg.in/cheggaaa/pb.v1/pb_solaris.go | 6 + vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go | 141 ++++++ vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go | 110 +++++ vendor/gopkg.in/cheggaaa/pb.v1/pool.go | 76 +++ vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go | 35 ++ vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go | 22 + vendor/gopkg.in/cheggaaa/pb.v1/reader.go | 25 + vendor/gopkg.in/cheggaaa/pb.v1/runecount.go | 17 + vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go | 9 + vendor/gopkg.in/cheggaaa/pb.v1/termios_nix.go | 7 + vendor/vendor.json | 16 + 16 files changed, 1191 insertions(+) create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/LICENSE create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/README.md create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/format.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pb.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pb_nix.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pb_solaris.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pool.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/reader.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/runecount.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go create mode 100644 vendor/gopkg.in/cheggaaa/pb.v1/termios_nix.go diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE b/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE new file mode 100644 index 00000000000..51197033392 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2012-2015, Sergey Cherepanov +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/README.md b/vendor/gopkg.in/cheggaaa/pb.v1/README.md new file mode 100644 index 00000000000..31babb30543 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/README.md @@ -0,0 +1,176 @@ +# Terminal progress bar for Go + +Simple progress bar for console programs. + + +## Installation + +``` +go get gopkg.in/cheggaaa/pb.v1 +``` + +## Usage + +```Go +package main + +import ( + "gopkg.in/cheggaaa/pb.v1" + "time" +) + +func main() { + count := 100000 + bar := pb.StartNew(count) + for i := 0; i < count; i++ { + bar.Increment() + time.Sleep(time.Millisecond) + } + bar.FinishPrint("The End!") +} + +``` + +Result will be like this: + +``` +> go run test.go +37158 / 100000 [================>_______________________________] 37.16% 1m11s +``` + +## Customization + +```Go +// create bar +bar := pb.New(count) + +// refresh info every second (default 200ms) +bar.SetRefreshRate(time.Second) + +// show percents (by default already true) +bar.ShowPercent = true + +// show bar (by default already true) +bar.ShowBar = true + +// no counters +bar.ShowCounters = false + +// show "time left" +bar.ShowTimeLeft = true + +// show average speed +bar.ShowSpeed = true + +// sets the width of the progress bar +bar.SetWidth(80) + +// sets the width of the progress bar, but if terminal size smaller will be ignored +bar.SetMaxWidth(80) + +// convert output to readable format (like KB, MB) +bar.SetUnits(pb.U_BYTES) + +// and start +bar.Start() +``` + +## Progress bar for IO Operations + +```go +// create and start bar +bar := pb.New(myDataLen).SetUnits(pb.U_BYTES) +bar.Start() + +// my io.Reader +r := myReader + +// my io.Writer +w := myWriter + +// create proxy reader +reader := bar.NewProxyReader(r) + +// and copy from pb reader +io.Copy(w, reader) + +``` + +```go +// create and start bar +bar := pb.New(myDataLen).SetUnits(pb.U_BYTES) +bar.Start() + +// my io.Reader +r := myReader + +// my io.Writer +w := myWriter + +// create multi writer +writer := io.MultiWriter(w, bar) + +// and copy +io.Copy(writer, r) + +bar.Finish() +``` + +## Custom Progress Bar Look-and-feel + +```go +bar.Format("<.- >") +``` + +## Multiple Progress Bars (experimental and unstable) + +Do not print to terminal while pool is active. + +```go +package main + +import ( + "math/rand" + "sync" + "time" + + "gopkg.in/cheggaaa/pb.v1" +) + +func main() { + // create bars + first := pb.New(200).Prefix("First ") + second := pb.New(200).Prefix("Second ") + third := pb.New(200).Prefix("Third ") + // start pool + pool, err := pb.StartPool(first, second, third) + if err != nil { + panic(err) + } + // update bars + wg := new(sync.WaitGroup) + for _, bar := range []*pb.ProgressBar{first, second, third} { + wg.Add(1) + go func(cb *pb.ProgressBar) { + for n := 0; n < 200; n++ { + cb.Increment() + time.Sleep(time.Millisecond * time.Duration(rand.Intn(100))) + } + cb.Finish() + wg.Done() + }(bar) + } + wg.Wait() + // close pool + pool.Stop() +} +``` + +The result will be as follows: + +``` +$ go run example/multiple.go +First 141 / 1000 [===============>---------------------------------------] 14.10 % 44s +Second 139 / 1000 [==============>---------------------------------------] 13.90 % 44s +Third 152 / 1000 [================>--------------------------------------] 15.20 % 40s +``` diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/format.go b/vendor/gopkg.in/cheggaaa/pb.v1/format.go new file mode 100644 index 00000000000..d5aeff793d8 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/format.go @@ -0,0 +1,87 @@ +package pb + +import ( + "fmt" + "strings" + "time" +) + +type Units int + +const ( + // U_NO are default units, they represent a simple value and are not formatted at all. + U_NO Units = iota + // U_BYTES units are formatted in a human readable way (b, Bb, Mb, ...) + U_BYTES + // U_DURATION units are formatted in a human readable way (3h14m15s) + U_DURATION +) + +func Format(i int64) *formatter { + return &formatter{n: i} +} + +type formatter struct { + n int64 + unit Units + width int + perSec bool +} + +func (f *formatter) Value(n int64) *formatter { + f.n = n + return f +} + +func (f *formatter) To(unit Units) *formatter { + f.unit = unit + return f +} + +func (f *formatter) Width(width int) *formatter { + f.width = width + return f +} + +func (f *formatter) PerSec() *formatter { + f.perSec = true + return f +} + +func (f *formatter) String() (out string) { + switch f.unit { + case U_BYTES: + out = formatBytes(f.n) + case U_DURATION: + d := time.Duration(f.n) + if d > time.Hour*24 { + out = fmt.Sprintf("%dd", d/24/time.Hour) + d -= (d / time.Hour / 24) * (time.Hour * 24) + } + out = fmt.Sprintf("%s%v", out, d) + default: + out = fmt.Sprintf(fmt.Sprintf("%%%dd", f.width), f.n) + } + if f.perSec { + out += "/s" + } + return +} + +// Convert bytes to human readable string. Like a 2 MB, 64.2 KB, 52 B +func formatBytes(i int64) (result string) { + switch { + case i > (1024 * 1024 * 1024 * 1024): + result = fmt.Sprintf("%.02f TB", float64(i)/1024/1024/1024/1024) + case i > (1024 * 1024 * 1024): + result = fmt.Sprintf("%.02f GB", float64(i)/1024/1024/1024) + case i > (1024 * 1024): + result = fmt.Sprintf("%.02f MB", float64(i)/1024/1024) + case i > 1024: + result = fmt.Sprintf("%.02f KB", float64(i)/1024) + default: + result = fmt.Sprintf("%d B", i) + } + result = strings.Trim(result, " ") + return +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb.go new file mode 100644 index 00000000000..f1f15bff393 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pb.go @@ -0,0 +1,444 @@ +// Simple console progress bars +package pb + +import ( + "fmt" + "io" + "math" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf8" +) + +// Current version +const Version = "1.0.6" + +const ( + // Default refresh rate - 200ms + DEFAULT_REFRESH_RATE = time.Millisecond * 200 + FORMAT = "[=>-]" +) + +// DEPRECATED +// variables for backward compatibility, from now do not work +// use pb.Format and pb.SetRefreshRate +var ( + DefaultRefreshRate = DEFAULT_REFRESH_RATE + BarStart, BarEnd, Empty, Current, CurrentN string +) + +// Create new progress bar object +func New(total int) *ProgressBar { + return New64(int64(total)) +} + +// Create new progress bar object using int64 as total +func New64(total int64) *ProgressBar { + pb := &ProgressBar{ + Total: total, + RefreshRate: DEFAULT_REFRESH_RATE, + ShowPercent: true, + ShowCounters: true, + ShowBar: true, + ShowTimeLeft: true, + ShowFinalTime: true, + Units: U_NO, + ManualUpdate: false, + finish: make(chan struct{}), + currentValue: -1, + mu: new(sync.Mutex), + } + return pb.Format(FORMAT) +} + +// Create new object and start +func StartNew(total int) *ProgressBar { + return New(total).Start() +} + +// Callback for custom output +// For example: +// bar.Callback = func(s string) { +// mySuperPrint(s) +// } +// +type Callback func(out string) + +type ProgressBar struct { + current int64 // current must be first member of struct (https://code.google.com/p/go/issues/detail?id=5278) + + Total int64 + RefreshRate time.Duration + ShowPercent, ShowCounters bool + ShowSpeed, ShowTimeLeft, ShowBar bool + ShowFinalTime bool + Output io.Writer + Callback Callback + NotPrint bool + Units Units + Width int + ForceWidth bool + ManualUpdate bool + AutoStat bool + + // Default width for the time box. + UnitsWidth int + TimeBoxWidth int + + finishOnce sync.Once //Guards isFinish + finish chan struct{} + isFinish bool + + startTime time.Time + startValue int64 + currentValue int64 + + prefix, postfix string + + mu *sync.Mutex + lastPrint string + + BarStart string + BarEnd string + Empty string + Current string + CurrentN string + + AlwaysUpdate bool +} + +// Start print +func (pb *ProgressBar) Start() *ProgressBar { + pb.startTime = time.Now() + pb.startValue = pb.current + if pb.Total == 0 { + pb.ShowTimeLeft = false + pb.ShowPercent = false + pb.AutoStat = false + } + if !pb.ManualUpdate { + pb.Update() // Initial printing of the bar before running the bar refresher. + go pb.refresher() + } + return pb +} + +// Increment current value +func (pb *ProgressBar) Increment() int { + return pb.Add(1) +} + +// Get current value +func (pb *ProgressBar) Get() int64 { + c := atomic.LoadInt64(&pb.current) + return c +} + +// Set current value +func (pb *ProgressBar) Set(current int) *ProgressBar { + return pb.Set64(int64(current)) +} + +// Set64 sets the current value as int64 +func (pb *ProgressBar) Set64(current int64) *ProgressBar { + atomic.StoreInt64(&pb.current, current) + return pb +} + +// Add to current value +func (pb *ProgressBar) Add(add int) int { + return int(pb.Add64(int64(add))) +} + +func (pb *ProgressBar) Add64(add int64) int64 { + return atomic.AddInt64(&pb.current, add) +} + +// Set prefix string +func (pb *ProgressBar) Prefix(prefix string) *ProgressBar { + pb.prefix = prefix + return pb +} + +// Set postfix string +func (pb *ProgressBar) Postfix(postfix string) *ProgressBar { + pb.postfix = postfix + return pb +} + +// Set custom format for bar +// Example: bar.Format("[=>_]") +// Example: bar.Format("[\x00=\x00>\x00-\x00]") // \x00 is the delimiter +func (pb *ProgressBar) Format(format string) *ProgressBar { + var formatEntries []string + if len(format) == 5 { + formatEntries = strings.Split(format, "") + } else { + formatEntries = strings.Split(format, "\x00") + } + if len(formatEntries) == 5 { + pb.BarStart = formatEntries[0] + pb.BarEnd = formatEntries[4] + pb.Empty = formatEntries[3] + pb.Current = formatEntries[1] + pb.CurrentN = formatEntries[2] + } + return pb +} + +// Set bar refresh rate +func (pb *ProgressBar) SetRefreshRate(rate time.Duration) *ProgressBar { + pb.RefreshRate = rate + return pb +} + +// Set units +// bar.SetUnits(U_NO) - by default +// bar.SetUnits(U_BYTES) - for Mb, Kb, etc +func (pb *ProgressBar) SetUnits(units Units) *ProgressBar { + pb.Units = units + return pb +} + +// Set max width, if width is bigger than terminal width, will be ignored +func (pb *ProgressBar) SetMaxWidth(width int) *ProgressBar { + pb.Width = width + pb.ForceWidth = false + return pb +} + +// Set bar width +func (pb *ProgressBar) SetWidth(width int) *ProgressBar { + pb.Width = width + pb.ForceWidth = true + return pb +} + +// End print +func (pb *ProgressBar) Finish() { + //Protect multiple calls + pb.finishOnce.Do(func() { + close(pb.finish) + pb.write(atomic.LoadInt64(&pb.current)) + switch { + case pb.Output != nil: + fmt.Fprintln(pb.Output) + case !pb.NotPrint: + fmt.Println() + } + pb.isFinish = true + }) +} + +// End print and write string 'str' +func (pb *ProgressBar) FinishPrint(str string) { + pb.Finish() + if pb.Output != nil { + fmt.Fprintln(pb.Output, str) + } else { + fmt.Println(str) + } +} + +// implement io.Writer +func (pb *ProgressBar) Write(p []byte) (n int, err error) { + n = len(p) + pb.Add(n) + return +} + +// implement io.Reader +func (pb *ProgressBar) Read(p []byte) (n int, err error) { + n = len(p) + pb.Add(n) + return +} + +// Create new proxy reader over bar +// Takes io.Reader or io.ReadCloser +func (pb *ProgressBar) NewProxyReader(r io.Reader) *Reader { + return &Reader{r, pb} +} + +func (pb *ProgressBar) write(current int64) { + width := pb.GetWidth() + + var percentBox, countersBox, timeLeftBox, speedBox, barBox, end, out string + + // percents + if pb.ShowPercent { + var percent float64 + if pb.Total > 0 { + percent = float64(current) / (float64(pb.Total) / float64(100)) + } else { + percent = float64(current) / float64(100) + } + percentBox = fmt.Sprintf(" %6.02f%%", percent) + } + + // counters + if pb.ShowCounters { + current := Format(current).To(pb.Units).Width(pb.UnitsWidth) + if pb.Total > 0 { + total := Format(pb.Total).To(pb.Units).Width(pb.UnitsWidth) + countersBox = fmt.Sprintf(" %s / %s ", current, total) + } else { + countersBox = fmt.Sprintf(" %s / ? ", current) + } + } + + // time left + fromStart := time.Now().Sub(pb.startTime) + currentFromStart := current - pb.startValue + select { + case <-pb.finish: + if pb.ShowFinalTime { + var left time.Duration + left = (fromStart / time.Second) * time.Second + timeLeftBox = fmt.Sprintf(" %s", left.String()) + } + default: + if pb.ShowTimeLeft && currentFromStart > 0 { + perEntry := fromStart / time.Duration(currentFromStart) + var left time.Duration + if pb.Total > 0 { + left = time.Duration(pb.Total-currentFromStart) * perEntry + left = (left / time.Second) * time.Second + } else { + left = time.Duration(currentFromStart) * perEntry + left = (left / time.Second) * time.Second + } + timeLeft := Format(int64(left)).To(U_DURATION).String() + timeLeftBox = fmt.Sprintf(" %s", timeLeft) + } + } + + if len(timeLeftBox) < pb.TimeBoxWidth { + timeLeftBox = fmt.Sprintf("%s%s", strings.Repeat(" ", pb.TimeBoxWidth-len(timeLeftBox)), timeLeftBox) + } + + // speed + if pb.ShowSpeed && currentFromStart > 0 { + fromStart := time.Now().Sub(pb.startTime) + speed := float64(currentFromStart) / (float64(fromStart) / float64(time.Second)) + speedBox = " " + Format(int64(speed)).To(pb.Units).Width(pb.UnitsWidth).PerSec().String() + } + + barWidth := escapeAwareRuneCountInString(countersBox + pb.BarStart + pb.BarEnd + percentBox + timeLeftBox + speedBox + pb.prefix + pb.postfix) + // bar + if pb.ShowBar { + size := width - barWidth + if size > 0 { + if pb.Total > 0 { + curCount := int(math.Ceil((float64(current) / float64(pb.Total)) * float64(size))) + emptCount := size - curCount + barBox = pb.BarStart + if emptCount < 0 { + emptCount = 0 + } + if curCount > size { + curCount = size + } + if emptCount <= 0 { + barBox += strings.Repeat(pb.Current, curCount) + } else if curCount > 0 { + barBox += strings.Repeat(pb.Current, curCount-1) + pb.CurrentN + } + barBox += strings.Repeat(pb.Empty, emptCount) + pb.BarEnd + } else { + barBox = pb.BarStart + pos := size - int(current)%int(size) + if pos-1 > 0 { + barBox += strings.Repeat(pb.Empty, pos-1) + } + barBox += pb.Current + if size-pos-1 > 0 { + barBox += strings.Repeat(pb.Empty, size-pos-1) + } + barBox += pb.BarEnd + } + } + } + + // check len + out = pb.prefix + countersBox + barBox + percentBox + speedBox + timeLeftBox + pb.postfix + if escapeAwareRuneCountInString(out) < width { + end = strings.Repeat(" ", width-utf8.RuneCountInString(out)) + } + + // and print! + pb.mu.Lock() + pb.lastPrint = out + end + pb.mu.Unlock() + switch { + case pb.isFinish: + return + case pb.Output != nil: + fmt.Fprint(pb.Output, "\r"+out+end) + case pb.Callback != nil: + pb.Callback(out + end) + case !pb.NotPrint: + fmt.Print("\r" + out + end) + } +} + +// GetTerminalWidth - returns terminal width for all platforms. +func GetTerminalWidth() (int, error) { + return terminalWidth() +} + +func (pb *ProgressBar) GetWidth() int { + if pb.ForceWidth { + return pb.Width + } + + width := pb.Width + termWidth, _ := terminalWidth() + if width == 0 || termWidth <= width { + width = termWidth + } + + return width +} + +// Write the current state of the progressbar +func (pb *ProgressBar) Update() { + c := atomic.LoadInt64(&pb.current) + if pb.AlwaysUpdate || c != pb.currentValue { + pb.write(c) + pb.currentValue = c + } + if pb.AutoStat { + if c == 0 { + pb.startTime = time.Now() + pb.startValue = 0 + } else if c >= pb.Total && pb.isFinish != true { + pb.Finish() + } + } +} + +func (pb *ProgressBar) String() string { + return pb.lastPrint +} + +// Internal loop for refreshing the progressbar +func (pb *ProgressBar) refresher() { + for { + select { + case <-pb.finish: + return + case <-time.After(pb.RefreshRate): + pb.Update() + } + } +} + +type window struct { + Row uint16 + Col uint16 + Xpixel uint16 + Ypixel uint16 +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_nix.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_nix.go new file mode 100644 index 00000000000..c06097b4345 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pb_nix.go @@ -0,0 +1,8 @@ +// +build linux darwin freebsd netbsd openbsd dragonfly +// +build !appengine + +package pb + +import "syscall" + +const sysIoctl = syscall.SYS_IOCTL diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_solaris.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_solaris.go new file mode 100644 index 00000000000..b7d461e1742 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pb_solaris.go @@ -0,0 +1,6 @@ +// +build solaris +// +build !appengine + +package pb + +const sysIoctl = 54 diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go new file mode 100644 index 00000000000..72f68283544 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go @@ -0,0 +1,141 @@ +// +build windows + +package pb + +import ( + "errors" + "fmt" + "os" + "sync" + "syscall" + "unsafe" +) + +var tty = os.Stdin + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + + // GetConsoleScreenBufferInfo retrieves information about the + // specified console screen buffer. + // http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx + procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") + + // GetConsoleMode retrieves the current input mode of a console's + // input buffer or the current output mode of a console screen buffer. + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms683167(v=vs.85).aspx + getConsoleMode = kernel32.NewProc("GetConsoleMode") + + // SetConsoleMode sets the input mode of a console's input buffer + // or the output mode of a console screen buffer. + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx + setConsoleMode = kernel32.NewProc("SetConsoleMode") + + // SetConsoleCursorPosition sets the cursor position in the + // specified console screen buffer. + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms686025(v=vs.85).aspx + setConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") +) + +type ( + // Defines the coordinates of the upper left and lower right corners + // of a rectangle. + // See + // http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx + smallRect struct { + Left, Top, Right, Bottom int16 + } + + // Defines the coordinates of a character cell in a console screen + // buffer. The origin of the coordinate system (0,0) is at the top, left cell + // of the buffer. + // See + // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx + coordinates struct { + X, Y int16 + } + + word int16 + + // Contains information about a console screen buffer. + // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx + consoleScreenBufferInfo struct { + dwSize coordinates + dwCursorPosition coordinates + wAttributes word + srWindow smallRect + dwMaximumWindowSize coordinates + } +) + +// terminalWidth returns width of the terminal. +func terminalWidth() (width int, err error) { + var info consoleScreenBufferInfo + _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0) + if e != 0 { + return 0, error(e) + } + return int(info.dwSize.X) - 1, nil +} + +func getCursorPos() (pos coordinates, err error) { + var info consoleScreenBufferInfo + _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0) + if e != 0 { + return info.dwCursorPosition, error(e) + } + return info.dwCursorPosition, nil +} + +func setCursorPos(pos coordinates) error { + _, _, e := syscall.Syscall(setConsoleCursorPosition.Addr(), 2, uintptr(syscall.Stdout), uintptr(uint32(uint16(pos.Y))<<16|uint32(uint16(pos.X))), 0) + if e != 0 { + return error(e) + } + return nil +} + +var ErrPoolWasStarted = errors.New("Bar pool was started") + +var echoLocked bool +var echoLockMutex sync.Mutex + +var oldState word + +func lockEcho() (quit chan int, err error) { + echoLockMutex.Lock() + defer echoLockMutex.Unlock() + if echoLocked { + err = ErrPoolWasStarted + return + } + echoLocked = true + + if _, _, e := syscall.Syscall(getConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&oldState)), 0); e != 0 { + err = fmt.Errorf("Can't get terminal settings: %v", e) + return + } + + newState := oldState + const ENABLE_ECHO_INPUT = 0x0004 + const ENABLE_LINE_INPUT = 0x0002 + newState = newState & (^(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT)) + if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(newState), 0); e != 0 { + err = fmt.Errorf("Can't set terminal settings: %v", e) + return + } + return +} + +func unlockEcho() (err error) { + echoLockMutex.Lock() + defer echoLockMutex.Unlock() + if !echoLocked { + return + } + echoLocked = false + if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(oldState), 0); e != 0 { + err = fmt.Errorf("Can't set terminal settings") + } + return +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go new file mode 100644 index 00000000000..12e6fe6e2a0 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go @@ -0,0 +1,110 @@ +// +build linux darwin freebsd netbsd openbsd solaris dragonfly +// +build !appengine + +package pb + +import ( + "errors" + "fmt" + "os" + "os/signal" + "runtime" + "sync" + "syscall" + "unsafe" +) + +const ( + TIOCGWINSZ = 0x5413 + TIOCGWINSZ_OSX = 1074295912 +) + +var tty *os.File + +var ErrPoolWasStarted = errors.New("Bar pool was started") + +var echoLocked bool +var echoLockMutex sync.Mutex + +func init() { + var err error + tty, err = os.Open("/dev/tty") + if err != nil { + tty = os.Stdin + } +} + +// terminalWidth returns width of the terminal. +func terminalWidth() (int, error) { + w := new(window) + tio := syscall.TIOCGWINSZ + if runtime.GOOS == "darwin" { + tio = TIOCGWINSZ_OSX + } + res, _, err := syscall.Syscall(sysIoctl, + tty.Fd(), + uintptr(tio), + uintptr(unsafe.Pointer(w)), + ) + if int(res) == -1 { + return 0, err + } + return int(w.Col), nil +} + +var oldState syscall.Termios + +func lockEcho() (quit chan int, err error) { + echoLockMutex.Lock() + defer echoLockMutex.Unlock() + if echoLocked { + err = ErrPoolWasStarted + return + } + echoLocked = true + + fd := tty.Fd() + if _, _, e := syscall.Syscall6(sysIoctl, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); e != 0 { + err = fmt.Errorf("Can't get terminal settings: %v", e) + return + } + + newState := oldState + newState.Lflag &^= syscall.ECHO + newState.Lflag |= syscall.ICANON | syscall.ISIG + newState.Iflag |= syscall.ICRNL + if _, _, e := syscall.Syscall6(sysIoctl, fd, ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); e != 0 { + err = fmt.Errorf("Can't set terminal settings: %v", e) + return + } + quit = make(chan int, 1) + go catchTerminate(quit) + return +} + +func unlockEcho() (err error) { + echoLockMutex.Lock() + defer echoLockMutex.Unlock() + if !echoLocked { + return + } + echoLocked = false + fd := tty.Fd() + if _, _, e := syscall.Syscall6(sysIoctl, fd, ioctlWriteTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); e != 0 { + err = fmt.Errorf("Can't set terminal settings") + } + return +} + +// listen exit signals and restore terminal state +func catchTerminate(quit chan int) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL) + defer signal.Stop(sig) + select { + case <-quit: + unlockEcho() + case <-sig: + unlockEcho() + } +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool.go new file mode 100644 index 00000000000..0b4a4afa885 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pool.go @@ -0,0 +1,76 @@ +// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows + +package pb + +import ( + "sync" + "time" +) + +// Create and start new pool with given bars +// You need call pool.Stop() after work +func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) { + pool = new(Pool) + if err = pool.start(); err != nil { + return + } + pool.Add(pbs...) + return +} + +type Pool struct { + RefreshRate time.Duration + bars []*ProgressBar + quit chan int + finishOnce sync.Once +} + +// Add progress bars. +func (p *Pool) Add(pbs ...*ProgressBar) { + for _, bar := range pbs { + bar.ManualUpdate = true + bar.NotPrint = true + bar.Start() + p.bars = append(p.bars, bar) + } +} + +func (p *Pool) start() (err error) { + p.RefreshRate = DefaultRefreshRate + quit, err := lockEcho() + if err != nil { + return + } + p.quit = make(chan int) + go p.writer(quit) + return +} + +func (p *Pool) writer(finish chan int) { + var first = true + for { + select { + case <-time.After(p.RefreshRate): + if p.print(first) { + p.print(false) + finish <- 1 + return + } + first = false + case <-p.quit: + finish <- 1 + return + } + } +} + +// Restore terminal state and close pool +func (p *Pool) Stop() error { + // Wait until one final refresh has passed. + time.Sleep(p.RefreshRate) + + p.finishOnce.Do(func() { + close(p.quit) + }) + return unlockEcho() +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go new file mode 100644 index 00000000000..d7a5ace41f4 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go @@ -0,0 +1,35 @@ +// +build windows + +package pb + +import ( + "fmt" + "log" +) + +func (p *Pool) print(first bool) bool { + var out string + if !first { + coords, err := getCursorPos() + if err != nil { + log.Panic(err) + } + coords.Y -= int16(len(p.bars)) + coords.X = 0 + + err = setCursorPos(coords) + if err != nil { + log.Panic(err) + } + } + isFinished := true + for _, bar := range p.bars { + if !bar.isFinish { + isFinished = false + } + bar.Update() + out += fmt.Sprintf("\r%s\n", bar.String()) + } + fmt.Print(out) + return isFinished +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go new file mode 100644 index 00000000000..d95b71d8762 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go @@ -0,0 +1,22 @@ +// +build linux darwin freebsd netbsd openbsd solaris dragonfly + +package pb + +import "fmt" + +func (p *Pool) print(first bool) bool { + var out string + if !first { + out = fmt.Sprintf("\033[%dA", len(p.bars)) + } + isFinished := true + for _, bar := range p.bars { + if !bar.isFinish { + isFinished = false + } + bar.Update() + out += fmt.Sprintf("\r%s\n", bar.String()) + } + fmt.Print(out) + return isFinished +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/reader.go b/vendor/gopkg.in/cheggaaa/pb.v1/reader.go new file mode 100644 index 00000000000..9f3148b5464 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/reader.go @@ -0,0 +1,25 @@ +package pb + +import ( + "io" +) + +// It's proxy reader, implement io.Reader +type Reader struct { + io.Reader + bar *ProgressBar +} + +func (r *Reader) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.bar.Add(n) + return +} + +// Close the reader when it implements io.Closer +func (r *Reader) Close() (err error) { + if closer, ok := r.Reader.(io.Closer); ok { + return closer.Close() + } + return +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go b/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go new file mode 100644 index 00000000000..d52edd36523 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go @@ -0,0 +1,17 @@ +package pb + +import ( + "github.com/mattn/go-runewidth" + "regexp" +) + +// Finds the control character sequences (like colors) +var ctrlFinder = regexp.MustCompile("\x1b\x5b[0-9]+\x6d") + +func escapeAwareRuneCountInString(s string) int { + n := runewidth.StringWidth(s) + for _, sm := range ctrlFinder.FindAllString(s, -1) { + n -= len(sm) + } + return n +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go b/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go new file mode 100644 index 00000000000..517ea8ed72f --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go @@ -0,0 +1,9 @@ +// +build darwin freebsd netbsd openbsd dragonfly +// +build !appengine + +package pb + +import "syscall" + +const ioctlReadTermios = syscall.TIOCGETA +const ioctlWriteTermios = syscall.TIOCSETA diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/termios_nix.go b/vendor/gopkg.in/cheggaaa/pb.v1/termios_nix.go new file mode 100644 index 00000000000..ebb3fe87c83 --- /dev/null +++ b/vendor/gopkg.in/cheggaaa/pb.v1/termios_nix.go @@ -0,0 +1,7 @@ +// +build linux solaris +// +build !appengine + +package pb + +const ioctlReadTermios = 0x5401 // syscall.TCGETS +const ioctlWriteTermios = 0x5402 // syscall.TCSETS diff --git a/vendor/vendor.json b/vendor/vendor.json index 35704c41907..4581d05af6c 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1480,6 +1480,22 @@ "checksumSHA1": "U7dGDNwEHORvJFMoNSXErKE7ITg=", "path": "google.golang.org/cloud/internal", "revision": "5a3b06f8b5da3b7c3a93da43163b872c86c509ef" + }, + { + "checksumSHA1": "w3z2xM4+RT/OWvVusDgNXZZEEkE=", + "path": "gopkg.in/cheggaaa/pb.v1", + "revision": "d7e6ca3010b6f084d8056847f55d7f572f180678", + "revisionTime": "2016-11-21T09:29:06Z" + }, + { + "checksumSHA1": "gY2M/3SCxqgrPz+P/N6JQ+m/wnA=", + "path": "gopkg.in/xmlpath.v2", + "revision": "860cbeca3ebcc600db0b213c0e83ad6ce91f5739" + }, + { + "checksumSHA1": "0dDWcU08d0FS4d99NCgCkGLjjqw=", + "path": "gopkg.in/xmlpath.v2/cmd/webpath", + "revision": "860cbeca3ebcc600db0b213c0e83ad6ce91f5739" } ], "rootPath": "github.com/hashicorp/packer" From b307312af837b99b031136e6be3d2ffd41170456 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 1 Mar 2017 14:45:49 -0600 Subject: [PATCH 10/18] Ugh..added dependency for gopkg.in/cheggaaa/pb.v1: github.com/matn/go-runewidth. Also added github.com/jlaffaye/ftp for ftp support. --- vendor/github.com/jlaffaye/ftp/LICENSE | 13 + vendor/github.com/jlaffaye/ftp/README.md | 17 + vendor/github.com/jlaffaye/ftp/ftp.go | 671 +++++++++ vendor/github.com/jlaffaye/ftp/status.go | 106 ++ vendor/github.com/mattn/go-runewidth/LICENSE | 21 + .../github.com/mattn/go-runewidth/README.mkd | 27 + .../mattn/go-runewidth/runewidth.go | 1223 +++++++++++++++++ .../mattn/go-runewidth/runewidth_js.go | 8 + .../mattn/go-runewidth/runewidth_posix.go | 77 ++ .../mattn/go-runewidth/runewidth_windows.go | 25 + vendor/vendor.json | 12 + 11 files changed, 2200 insertions(+) create mode 100644 vendor/github.com/jlaffaye/ftp/LICENSE create mode 100644 vendor/github.com/jlaffaye/ftp/README.md create mode 100644 vendor/github.com/jlaffaye/ftp/ftp.go create mode 100644 vendor/github.com/jlaffaye/ftp/status.go create mode 100644 vendor/github.com/mattn/go-runewidth/LICENSE create mode 100644 vendor/github.com/mattn/go-runewidth/README.mkd create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_js.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_posix.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_windows.go diff --git a/vendor/github.com/jlaffaye/ftp/LICENSE b/vendor/github.com/jlaffaye/ftp/LICENSE new file mode 100644 index 00000000000..9ab085c5146 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2011-2013, Julien Laffaye + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/jlaffaye/ftp/README.md b/vendor/github.com/jlaffaye/ftp/README.md new file mode 100644 index 00000000000..b711be7ad36 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/README.md @@ -0,0 +1,17 @@ +# goftp # + +[![Build Status](https://travis-ci.org/jlaffaye/ftp.svg?branch=master)](https://travis-ci.org/jlaffaye/ftp) +[![Coverage Status](https://coveralls.io/repos/jlaffaye/ftp/badge.svg?branch=master&service=github)](https://coveralls.io/github/jlaffaye/ftp?branch=master) +[![Go ReportCard](http://goreportcard.com/badge/jlaffaye/ftp)](http://goreportcard.com/report/jlaffaye/ftp) + +A FTP client package for Go + +## Install ## + +``` +go get -u github.com/jlaffaye/ftp +``` + +## Documentation ## + +http://godoc.org/github.com/jlaffaye/ftp diff --git a/vendor/github.com/jlaffaye/ftp/ftp.go b/vendor/github.com/jlaffaye/ftp/ftp.go new file mode 100644 index 00000000000..db7ac92771f --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/ftp.go @@ -0,0 +1,671 @@ +// Package ftp implements a FTP client as described in RFC 959. +package ftp + +import ( + "bufio" + "errors" + "io" + "net" + "net/textproto" + "strconv" + "strings" + "time" +) + +// EntryType describes the different types of an Entry. +type EntryType int + +// The differents types of an Entry +const ( + EntryTypeFile EntryType = iota + EntryTypeFolder + EntryTypeLink +) + +// ServerConn represents the connection to a remote FTP server. +type ServerConn struct { + conn *textproto.Conn + host string + timeout time.Duration + features map[string]string +} + +// Entry describes a file and is returned by List(). +type Entry struct { + Name string + Type EntryType + Size uint64 + Time time.Time +} + +// response represent a data-connection +type response struct { + conn net.Conn + c *ServerConn +} + +// Connect is an alias to Dial, for backward compatibility +func Connect(addr string) (*ServerConn, error) { + return Dial(addr) +} + +// Dial is like DialTimeout with no timeout +func Dial(addr string) (*ServerConn, error) { + return DialTimeout(addr, 0) +} + +// DialTimeout initializes the connection to the specified ftp server address. +// +// It is generally followed by a call to Login() as most FTP commands require +// an authenticated user. +func DialTimeout(addr string, timeout time.Duration) (*ServerConn, error) { + tconn, err := net.DialTimeout("tcp", addr, timeout) + if err != nil { + return nil, err + } + + // Use the resolved IP address in case addr contains a domain name + // If we use the domain name, we might not resolve to the same IP. + remoteAddr := tconn.RemoteAddr().String() + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return nil, err + } + + conn := textproto.NewConn(tconn) + + c := &ServerConn{ + conn: conn, + host: host, + timeout: timeout, + features: make(map[string]string), + } + + _, _, err = c.conn.ReadResponse(StatusReady) + if err != nil { + c.Quit() + return nil, err + } + + err = c.feat() + if err != nil { + c.Quit() + return nil, err + } + + return c, nil +} + +// Login authenticates the client with specified user and password. +// +// "anonymous"/"anonymous" is a common user/password scheme for FTP servers +// that allows anonymous read-only accounts. +func (c *ServerConn) Login(user, password string) error { + code, message, err := c.cmd(-1, "USER %s", user) + if err != nil { + return err + } + + switch code { + case StatusLoggedIn: + case StatusUserOK: + _, _, err = c.cmd(StatusLoggedIn, "PASS %s", password) + if err != nil { + return err + } + default: + return errors.New(message) + } + + // Switch to binary mode + _, _, err = c.cmd(StatusCommandOK, "TYPE I") + if err != nil { + return err + } + + return nil +} + +// feat issues a FEAT FTP command to list the additional commands supported by +// the remote FTP server. +// FEAT is described in RFC 2389 +func (c *ServerConn) feat() error { + code, message, err := c.cmd(-1, "FEAT") + if err != nil { + return err + } + + if code != StatusSystem { + // The server does not support the FEAT command. This is not an + // error: we consider that there is no additional feature. + return nil + } + + lines := strings.Split(message, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, " ") { + continue + } + + line = strings.TrimSpace(line) + featureElements := strings.SplitN(line, " ", 2) + + command := featureElements[0] + + var commandDesc string + if len(featureElements) == 2 { + commandDesc = featureElements[1] + } + + c.features[command] = commandDesc + } + + return nil +} + +// epsv issues an "EPSV" command to get a port number for a data connection. +func (c *ServerConn) epsv() (port int, err error) { + _, line, err := c.cmd(StatusExtendedPassiveMode, "EPSV") + if err != nil { + return + } + + start := strings.Index(line, "|||") + end := strings.LastIndex(line, "|") + if start == -1 || end == -1 { + err = errors.New("Invalid EPSV response format") + return + } + port, err = strconv.Atoi(line[start+3 : end]) + return +} + +// pasv issues a "PASV" command to get a port number for a data connection. +func (c *ServerConn) pasv() (port int, err error) { + _, line, err := c.cmd(StatusPassiveMode, "PASV") + if err != nil { + return + } + + // PASV response format : 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). + start := strings.Index(line, "(") + end := strings.LastIndex(line, ")") + if start == -1 || end == -1 { + return 0, errors.New("Invalid PASV response format") + } + + // We have to split the response string + pasvData := strings.Split(line[start+1:end], ",") + + if len(pasvData) < 6 { + return 0, errors.New("Invalid PASV response format") + } + + // Let's compute the port number + portPart1, err1 := strconv.Atoi(pasvData[4]) + if err1 != nil { + err = err1 + return + } + + portPart2, err2 := strconv.Atoi(pasvData[5]) + if err2 != nil { + err = err2 + return + } + + // Recompose port + port = portPart1*256 + portPart2 + return +} + +// openDataConn creates a new FTP data connection. +func (c *ServerConn) openDataConn() (net.Conn, error) { + var ( + port int + err error + ) + + if port, err = c.epsv(); err != nil { + if port, err = c.pasv(); err != nil { + return nil, err + } + } + + return net.DialTimeout("tcp", net.JoinHostPort(c.host, strconv.Itoa(port)), c.timeout) +} + +// cmd is a helper function to execute a command and check for the expected FTP +// return code +func (c *ServerConn) cmd(expected int, format string, args ...interface{}) (int, string, error) { + _, err := c.conn.Cmd(format, args...) + if err != nil { + return 0, "", err + } + + return c.conn.ReadResponse(expected) +} + +// cmdDataConnFrom executes a command which require a FTP data connection. +// Issues a REST FTP command to specify the number of bytes to skip for the transfer. +func (c *ServerConn) cmdDataConnFrom(offset uint64, format string, args ...interface{}) (net.Conn, error) { + conn, err := c.openDataConn() + if err != nil { + return nil, err + } + + if offset != 0 { + _, _, err := c.cmd(StatusRequestFilePending, "REST %d", offset) + if err != nil { + return nil, err + } + } + + _, err = c.conn.Cmd(format, args...) + if err != nil { + conn.Close() + return nil, err + } + + code, msg, err := c.conn.ReadResponse(-1) + if err != nil { + conn.Close() + return nil, err + } + if code != StatusAlreadyOpen && code != StatusAboutToSend { + conn.Close() + return nil, &textproto.Error{Code: code, Msg: msg} + } + + return conn, nil +} + +var errUnsupportedListLine = errors.New("Unsupported LIST line") + +// parseRFC3659ListLine parses the style of directory line defined in RFC 3659. +func parseRFC3659ListLine(line string) (*Entry, error) { + iSemicolon := strings.Index(line, ";") + iWhitespace := strings.Index(line, " ") + + if iSemicolon < 0 || iSemicolon > iWhitespace { + return nil, errUnsupportedListLine + } + + e := &Entry{ + Name: line[iWhitespace+1:], + } + + for _, field := range strings.Split(line[:iWhitespace-1], ";") { + i := strings.Index(field, "=") + if i < 1 { + return nil, errUnsupportedListLine + } + + key := field[:i] + value := field[i+1:] + + switch key { + case "modify": + var err error + e.Time, err = time.Parse("20060102150405", value) + if err != nil { + return nil, err + } + case "type": + switch value { + case "dir", "cdir", "pdir": + e.Type = EntryTypeFolder + case "file": + e.Type = EntryTypeFile + } + case "size": + e.setSize(value) + } + } + return e, nil +} + +// parseLsListLine parses a directory line in a format based on the output of +// the UNIX ls command. +func parseLsListLine(line string) (*Entry, error) { + fields := strings.Fields(line) + if len(fields) >= 7 && fields[1] == "folder" && fields[2] == "0" { + e := &Entry{ + Type: EntryTypeFolder, + Name: strings.Join(fields[6:], " "), + } + if err := e.setTime(fields[3:6]); err != nil { + return nil, err + } + + return e, nil + } + + if len(fields) < 8 { + return nil, errUnsupportedListLine + } + + if fields[1] == "0" { + e := &Entry{ + Type: EntryTypeFile, + Name: strings.Join(fields[7:], " "), + } + + if err := e.setSize(fields[2]); err != nil { + return nil, err + } + if err := e.setTime(fields[4:7]); err != nil { + return nil, err + } + + return e, nil + } + + if len(fields) < 9 { + return nil, errUnsupportedListLine + } + + e := &Entry{} + switch fields[0][0] { + case '-': + e.Type = EntryTypeFile + if err := e.setSize(fields[4]); err != nil { + return nil, err + } + case 'd': + e.Type = EntryTypeFolder + case 'l': + e.Type = EntryTypeLink + default: + return nil, errors.New("Unknown entry type") + } + + if err := e.setTime(fields[5:8]); err != nil { + return nil, err + } + + e.Name = strings.Join(fields[8:], " ") + return e, nil +} + +var dirTimeFormats = []string{ + "01-02-06 03:04PM", + "2006-01-02 15:04", +} + +// parseDirListLine parses a directory line in a format based on the output of +// the MS-DOS DIR command. +func parseDirListLine(line string) (*Entry, error) { + e := &Entry{} + var err error + + // Try various time formats that DIR might use, and stop when one works. + for _, format := range dirTimeFormats { + if len(line) > len(format) { + e.Time, err = time.Parse(format, line[:len(format)]) + if err == nil { + line = line[len(format):] + break + } + } + } + if err != nil { + // None of the time formats worked. + return nil, errUnsupportedListLine + } + + line = strings.TrimLeft(line, " ") + if strings.HasPrefix(line, "") { + e.Type = EntryTypeFolder + line = strings.TrimPrefix(line, "") + } else { + space := strings.Index(line, " ") + if space == -1 { + return nil, errUnsupportedListLine + } + e.Size, err = strconv.ParseUint(line[:space], 10, 64) + if err != nil { + return nil, errUnsupportedListLine + } + e.Type = EntryTypeFile + line = line[space:] + } + + e.Name = strings.TrimLeft(line, " ") + return e, nil +} + +var listLineParsers = []func(line string) (*Entry, error){ + parseRFC3659ListLine, + parseLsListLine, + parseDirListLine, +} + +// parseListLine parses the various non-standard format returned by the LIST +// FTP command. +func parseListLine(line string) (*Entry, error) { + for _, f := range listLineParsers { + e, err := f(line) + if err == errUnsupportedListLine { + // Try another format. + continue + } + return e, err + } + return nil, errUnsupportedListLine +} + +func (e *Entry) setSize(str string) (err error) { + e.Size, err = strconv.ParseUint(str, 0, 64) + return +} + +func (e *Entry) setTime(fields []string) (err error) { + var timeStr string + if strings.Contains(fields[2], ":") { // this year + thisYear, _, _ := time.Now().Date() + timeStr = fields[1] + " " + fields[0] + " " + strconv.Itoa(thisYear)[2:4] + " " + fields[2] + " GMT" + } else { // not this year + if len(fields[2]) != 4 { + return errors.New("Invalid year format in time string") + } + timeStr = fields[1] + " " + fields[0] + " " + fields[2][2:4] + " 00:00 GMT" + } + e.Time, err = time.Parse("_2 Jan 06 15:04 MST", timeStr) + return +} + +// NameList issues an NLST FTP command. +func (c *ServerConn) NameList(path string) (entries []string, err error) { + conn, err := c.cmdDataConnFrom(0, "NLST %s", path) + if err != nil { + return + } + + r := &response{conn, c} + defer r.Close() + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + entries = append(entries, scanner.Text()) + } + if err = scanner.Err(); err != nil { + return entries, err + } + return +} + +// List issues a LIST FTP command. +func (c *ServerConn) List(path string) (entries []*Entry, err error) { + conn, err := c.cmdDataConnFrom(0, "LIST %s", path) + if err != nil { + return + } + + r := &response{conn, c} + defer r.Close() + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + entry, err := parseListLine(line) + if err == nil { + entries = append(entries, entry) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return +} + +// ChangeDir issues a CWD FTP command, which changes the current directory to +// the specified path. +func (c *ServerConn) ChangeDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CWD %s", path) + return err +} + +// ChangeDirToParent issues a CDUP FTP command, which changes the current +// directory to the parent directory. This is similar to a call to ChangeDir +// with a path set to "..". +func (c *ServerConn) ChangeDirToParent() error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CDUP") + return err +} + +// CurrentDir issues a PWD FTP command, which Returns the path of the current +// directory. +func (c *ServerConn) CurrentDir() (string, error) { + _, msg, err := c.cmd(StatusPathCreated, "PWD") + if err != nil { + return "", err + } + + start := strings.Index(msg, "\"") + end := strings.LastIndex(msg, "\"") + + if start == -1 || end == -1 { + return "", errors.New("Unsuported PWD response format") + } + + return msg[start+1 : end], nil +} + +// Retr issues a RETR FTP command to fetch the specified file from the remote +// FTP server. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) Retr(path string) (io.ReadCloser, error) { + return c.RetrFrom(path, 0) +} + +// RetrFrom issues a RETR FTP command to fetch the specified file from the remote +// FTP server, the server will not send the offset first bytes of the file. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) RetrFrom(path string, offset uint64) (io.ReadCloser, error) { + conn, err := c.cmdDataConnFrom(offset, "RETR %s", path) + if err != nil { + return nil, err + } + + return &response{conn, c}, nil +} + +// Stor issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) Stor(path string, r io.Reader) error { + return c.StorFrom(path, r, 0) +} + +// StorFrom issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader, writing +// on the server will start at the given file offset. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) StorFrom(path string, r io.Reader, offset uint64) error { + conn, err := c.cmdDataConnFrom(offset, "STOR %s", path) + if err != nil { + return err + } + + _, err = io.Copy(conn, r) + conn.Close() + if err != nil { + return err + } + + _, _, err = c.conn.ReadResponse(StatusClosingDataConnection) + return err +} + +// Rename renames a file on the remote FTP server. +func (c *ServerConn) Rename(from, to string) error { + _, _, err := c.cmd(StatusRequestFilePending, "RNFR %s", from) + if err != nil { + return err + } + + _, _, err = c.cmd(StatusRequestedFileActionOK, "RNTO %s", to) + return err +} + +// Delete issues a DELE FTP command to delete the specified file from the +// remote FTP server. +func (c *ServerConn) Delete(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "DELE %s", path) + return err +} + +// MakeDir issues a MKD FTP command to create the specified directory on the +// remote FTP server. +func (c *ServerConn) MakeDir(path string) error { + _, _, err := c.cmd(StatusPathCreated, "MKD %s", path) + return err +} + +// RemoveDir issues a RMD FTP command to remove the specified directory from +// the remote FTP server. +func (c *ServerConn) RemoveDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "RMD %s", path) + return err +} + +// NoOp issues a NOOP FTP command. +// NOOP has no effects and is usually used to prevent the remote FTP server to +// close the otherwise idle connection. +func (c *ServerConn) NoOp() error { + _, _, err := c.cmd(StatusCommandOK, "NOOP") + return err +} + +// Logout issues a REIN FTP command to logout the current user. +func (c *ServerConn) Logout() error { + _, _, err := c.cmd(StatusReady, "REIN") + return err +} + +// Quit issues a QUIT FTP command to properly close the connection from the +// remote FTP server. +func (c *ServerConn) Quit() error { + c.conn.Cmd("QUIT") + return c.conn.Close() +} + +// Read implements the io.Reader interface on a FTP data connection. +func (r *response) Read(buf []byte) (int, error) { + return r.conn.Read(buf) +} + +// Close implements the io.Closer interface on a FTP data connection. +func (r *response) Close() error { + err := r.conn.Close() + _, _, err2 := r.c.conn.ReadResponse(StatusClosingDataConnection) + if err2 != nil { + err = err2 + } + return err +} diff --git a/vendor/github.com/jlaffaye/ftp/status.go b/vendor/github.com/jlaffaye/ftp/status.go new file mode 100644 index 00000000000..e90ca621122 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/status.go @@ -0,0 +1,106 @@ +package ftp + +// FTP status codes, defined in RFC 959 +const ( + StatusInitiating = 100 + StatusRestartMarker = 110 + StatusReadyMinute = 120 + StatusAlreadyOpen = 125 + StatusAboutToSend = 150 + + StatusCommandOK = 200 + StatusCommandNotImplemented = 202 + StatusSystem = 211 + StatusDirectory = 212 + StatusFile = 213 + StatusHelp = 214 + StatusName = 215 + StatusReady = 220 + StatusClosing = 221 + StatusDataConnectionOpen = 225 + StatusClosingDataConnection = 226 + StatusPassiveMode = 227 + StatusLongPassiveMode = 228 + StatusExtendedPassiveMode = 229 + StatusLoggedIn = 230 + StatusLoggedOut = 231 + StatusLogoutAck = 232 + StatusRequestedFileActionOK = 250 + StatusPathCreated = 257 + + StatusUserOK = 331 + StatusLoginNeedAccount = 332 + StatusRequestFilePending = 350 + + StatusNotAvailable = 421 + StatusCanNotOpenDataConnection = 425 + StatusTransfertAborted = 426 + StatusInvalidCredentials = 430 + StatusHostUnavailable = 434 + StatusFileActionIgnored = 450 + StatusActionAborted = 451 + Status452 = 452 + + StatusBadCommand = 500 + StatusBadArguments = 501 + StatusNotImplemented = 502 + StatusBadSequence = 503 + StatusNotImplementedParameter = 504 + StatusNotLoggedIn = 530 + StatusStorNeedAccount = 532 + StatusFileUnavailable = 550 + StatusPageTypeUnknown = 551 + StatusExceededStorage = 552 + StatusBadFileName = 553 +) + +var statusText = map[int]string{ + // 200 + StatusCommandOK: "Command okay.", + StatusCommandNotImplemented: "Command not implemented, superfluous at this site.", + StatusSystem: "System status, or system help reply.", + StatusDirectory: "Directory status.", + StatusFile: "File status.", + StatusHelp: "Help message.", + StatusName: "", + StatusReady: "Service ready for new user.", + StatusClosing: "Service closing control connection.", + StatusDataConnectionOpen: "Data connection open; no transfer in progress.", + StatusClosingDataConnection: "Closing data connection. Requested file action successful.", + StatusPassiveMode: "Entering Passive Mode.", + StatusLongPassiveMode: "Entering Long Passive Mode.", + StatusExtendedPassiveMode: "Entering Extended Passive Mode.", + StatusLoggedIn: "User logged in, proceed.", + StatusLoggedOut: "User logged out; service terminated.", + StatusLogoutAck: "Logout command noted, will complete when transfer done.", + StatusRequestedFileActionOK: "Requested file action okay, completed.", + StatusPathCreated: "Path created.", + + // 300 + StatusUserOK: "User name okay, need password.", + StatusLoginNeedAccount: "Need account for login.", + StatusRequestFilePending: "Requested file action pending further information.", + + // 400 + StatusNotAvailable: "Service not available, closing control connection.", + StatusCanNotOpenDataConnection: "Can't open data connection.", + StatusTransfertAborted: "Connection closed; transfer aborted.", + StatusInvalidCredentials: "Invalid username or password.", + StatusHostUnavailable: "Requested host unavailable.", + StatusFileActionIgnored: "Requested file action not taken.", + StatusActionAborted: "Requested action aborted. Local error in processing.", + Status452: "Insufficient storage space in system.", + + // 500 + StatusBadCommand: "Command unrecognized.", + StatusBadArguments: "Syntax error in parameters or arguments.", + StatusNotImplemented: "Command not implemented.", + StatusBadSequence: "Bad sequence of commands.", + StatusNotImplementedParameter: "Command not implemented for that parameter.", + StatusNotLoggedIn: "Not logged in.", + StatusStorNeedAccount: "Need account for storing files.", + StatusFileUnavailable: "File unavailable.", + StatusPageTypeUnknown: "Page type unknown.", + StatusExceededStorage: "Exceeded storage allocation.", + StatusBadFileName: "File name not allowed.", +} diff --git a/vendor/github.com/mattn/go-runewidth/LICENSE b/vendor/github.com/mattn/go-runewidth/LICENSE new file mode 100644 index 00000000000..91b5cef30eb --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/mattn/go-runewidth/README.mkd b/vendor/github.com/mattn/go-runewidth/README.mkd new file mode 100644 index 00000000000..66663a94b0b --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/README.mkd @@ -0,0 +1,27 @@ +go-runewidth +============ + +[![Build Status](https://travis-ci.org/mattn/go-runewidth.png?branch=master)](https://travis-ci.org/mattn/go-runewidth) +[![Coverage Status](https://coveralls.io/repos/mattn/go-runewidth/badge.png?branch=HEAD)](https://coveralls.io/r/mattn/go-runewidth?branch=HEAD) +[![GoDoc](https://godoc.org/github.com/mattn/go-runewidth?status.svg)](http://godoc.org/github.com/mattn/go-runewidth) +[![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-runewidth)](https://goreportcard.com/report/github.com/mattn/go-runewidth) + +Provides functions to get fixed width of the character or string. + +Usage +----- + +```go +runewidth.StringWidth("つのだ☆HIRO") == 12 +``` + + +Author +------ + +Yasuhiro Matsumoto + +License +------- + +under the MIT License: http://mattn.mit-license.org/2013 diff --git a/vendor/github.com/mattn/go-runewidth/runewidth.go b/vendor/github.com/mattn/go-runewidth/runewidth.go new file mode 100644 index 00000000000..2164497ad95 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth.go @@ -0,0 +1,1223 @@ +package runewidth + +var ( + // EastAsianWidth will be set true if the current locale is CJK + EastAsianWidth = IsEastAsian() + + // DefaultCondition is a condition in current locale + DefaultCondition = &Condition{EastAsianWidth} +) + +type interval struct { + first rune + last rune +} + +type table []interval + +func inTables(r rune, ts ...table) bool { + for _, t := range ts { + if inTable(r, t) { + return true + } + } + return false +} + +func inTable(r rune, t table) bool { + // func (t table) IncludesRune(r rune) bool { + if r < t[0].first { + return false + } + + bot := 0 + top := len(t) - 1 + for top >= bot { + mid := (bot + top) / 2 + + switch { + case t[mid].last < r: + bot = mid + 1 + case t[mid].first > r: + top = mid - 1 + default: + return true + } + } + + return false +} + +var private = table{ + {0x00E000, 0x00F8FF}, {0x0F0000, 0x0FFFFD}, {0x100000, 0x10FFFD}, +} + +var nonprint = table{ + {0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD}, + {0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F}, + {0x202A, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF}, +} + +var combining = table{ + {0x0300, 0x036F}, {0x0483, 0x0489}, {0x0591, 0x05BD}, + {0x05BF, 0x05BF}, {0x05C1, 0x05C2}, {0x05C4, 0x05C5}, + {0x05C7, 0x05C7}, {0x0610, 0x061A}, {0x064B, 0x065F}, + {0x0670, 0x0670}, {0x06D6, 0x06DC}, {0x06DF, 0x06E4}, + {0x06E7, 0x06E8}, {0x06EA, 0x06ED}, {0x0711, 0x0711}, + {0x0730, 0x074A}, {0x07A6, 0x07B0}, {0x07EB, 0x07F3}, + {0x0816, 0x0819}, {0x081B, 0x0823}, {0x0825, 0x0827}, + {0x0829, 0x082D}, {0x0859, 0x085B}, {0x08D4, 0x08E1}, + {0x08E3, 0x0903}, {0x093A, 0x093C}, {0x093E, 0x094F}, + {0x0951, 0x0957}, {0x0962, 0x0963}, {0x0981, 0x0983}, + {0x09BC, 0x09BC}, {0x09BE, 0x09C4}, {0x09C7, 0x09C8}, + {0x09CB, 0x09CD}, {0x09D7, 0x09D7}, {0x09E2, 0x09E3}, + {0x0A01, 0x0A03}, {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, + {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, + {0x0A70, 0x0A71}, {0x0A75, 0x0A75}, {0x0A81, 0x0A83}, + {0x0ABC, 0x0ABC}, {0x0ABE, 0x0AC5}, {0x0AC7, 0x0AC9}, + {0x0ACB, 0x0ACD}, {0x0AE2, 0x0AE3}, {0x0B01, 0x0B03}, + {0x0B3C, 0x0B3C}, {0x0B3E, 0x0B44}, {0x0B47, 0x0B48}, + {0x0B4B, 0x0B4D}, {0x0B56, 0x0B57}, {0x0B62, 0x0B63}, + {0x0B82, 0x0B82}, {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, + {0x0BCA, 0x0BCD}, {0x0BD7, 0x0BD7}, {0x0C00, 0x0C03}, + {0x0C3E, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, + {0x0C55, 0x0C56}, {0x0C62, 0x0C63}, {0x0C81, 0x0C83}, + {0x0CBC, 0x0CBC}, {0x0CBE, 0x0CC4}, {0x0CC6, 0x0CC8}, + {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, {0x0CE2, 0x0CE3}, + {0x0D01, 0x0D03}, {0x0D3E, 0x0D44}, {0x0D46, 0x0D48}, + {0x0D4A, 0x0D4D}, {0x0D57, 0x0D57}, {0x0D62, 0x0D63}, + {0x0D82, 0x0D83}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, + {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, {0x0DF2, 0x0DF3}, + {0x0E31, 0x0E31}, {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, + {0x0EB1, 0x0EB1}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, + {0x0EC8, 0x0ECD}, {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, + {0x0F37, 0x0F37}, {0x0F39, 0x0F39}, {0x0F3E, 0x0F3F}, + {0x0F71, 0x0F84}, {0x0F86, 0x0F87}, {0x0F8D, 0x0F97}, + {0x0F99, 0x0FBC}, {0x0FC6, 0x0FC6}, {0x102B, 0x103E}, + {0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, + {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, + {0x108F, 0x108F}, {0x109A, 0x109D}, {0x135D, 0x135F}, + {0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753}, + {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, + {0x180B, 0x180D}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, + {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1A17, 0x1A1B}, + {0x1A55, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, + {0x1AB0, 0x1ABE}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, + {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, + {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2}, + {0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF2, 0x1CF4}, + {0x1CF8, 0x1CF9}, {0x1DC0, 0x1DF5}, {0x1DFB, 0x1DFF}, + {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, + {0x2DE0, 0x2DFF}, {0x302A, 0x302F}, {0x3099, 0x309A}, + {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, + {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, + {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA880, 0xA881}, + {0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA926, 0xA92D}, + {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, + {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, + {0xAA4C, 0xAA4D}, {0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, + {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, + {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, + {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xFB1E, 0xFB1E}, + {0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, {0x10AE5, 0x10AE6}, {0x11000, 0x11002}, + {0x11038, 0x11046}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, + {0x11100, 0x11102}, {0x11127, 0x11134}, {0x11173, 0x11173}, + {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111CA, 0x111CC}, + {0x1122C, 0x11237}, {0x1123E, 0x1123E}, {0x112DF, 0x112EA}, + {0x11300, 0x11303}, {0x1133C, 0x1133C}, {0x1133E, 0x11344}, + {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, + {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x11435, 0x11446}, {0x114B0, 0x114C3}, {0x115AF, 0x115B5}, + {0x115B8, 0x115C0}, {0x115DC, 0x115DD}, {0x11630, 0x11640}, + {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x11C2F, 0x11C36}, + {0x11C38, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, + {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x16F51, 0x16F7E}, + {0x16F8F, 0x16F92}, {0x1BC9D, 0x1BC9E}, {0x1D165, 0x1D169}, + {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, + {0xE0100, 0xE01EF}, +} + +var doublewidth = table{ + {0x1100, 0x115F}, {0x231A, 0x231B}, {0x2329, 0x232A}, + {0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3}, + {0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653}, + {0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1}, + {0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, + {0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA}, + {0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA}, + {0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B}, + {0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E}, + {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, + {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x2E80, 0x2E99}, + {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, + {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, + {0x3105, 0x312D}, {0x3131, 0x318E}, {0x3190, 0x31BA}, + {0x31C0, 0x31E3}, {0x31F0, 0x321E}, {0x3220, 0x3247}, + {0x3250, 0x32FE}, {0x3300, 0x4DBF}, {0x4E00, 0xA48C}, + {0xA490, 0xA4C6}, {0xA960, 0xA97C}, {0xAC00, 0xD7A3}, + {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, + {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, + {0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE0}, {0x17000, 0x187EC}, + {0x18800, 0x18AF2}, {0x1B000, 0x1B001}, {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, + {0x1F200, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, + {0x1F250, 0x1F251}, {0x1F300, 0x1F320}, {0x1F32D, 0x1F335}, + {0x1F337, 0x1F37C}, {0x1F37E, 0x1F393}, {0x1F3A0, 0x1F3CA}, + {0x1F3CF, 0x1F3D3}, {0x1F3E0, 0x1F3F0}, {0x1F3F4, 0x1F3F4}, + {0x1F3F8, 0x1F43E}, {0x1F440, 0x1F440}, {0x1F442, 0x1F4FC}, + {0x1F4FF, 0x1F53D}, {0x1F54B, 0x1F54E}, {0x1F550, 0x1F567}, + {0x1F57A, 0x1F57A}, {0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A4}, + {0x1F5FB, 0x1F64F}, {0x1F680, 0x1F6C5}, {0x1F6CC, 0x1F6CC}, + {0x1F6D0, 0x1F6D2}, {0x1F6EB, 0x1F6EC}, {0x1F6F4, 0x1F6F6}, + {0x1F910, 0x1F91E}, {0x1F920, 0x1F927}, {0x1F930, 0x1F930}, + {0x1F933, 0x1F93E}, {0x1F940, 0x1F94B}, {0x1F950, 0x1F95E}, + {0x1F980, 0x1F991}, {0x1F9C0, 0x1F9C0}, {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD}, +} + +var ambiguous = table{ + {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, + {0x00AA, 0x00AA}, {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, + {0x00B6, 0x00BA}, {0x00BC, 0x00BF}, {0x00C6, 0x00C6}, + {0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1}, + {0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED}, + {0x00F0, 0x00F0}, {0x00F2, 0x00F3}, {0x00F7, 0x00FA}, + {0x00FC, 0x00FC}, {0x00FE, 0x00FE}, {0x0101, 0x0101}, + {0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B}, + {0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133}, + {0x0138, 0x0138}, {0x013F, 0x0142}, {0x0144, 0x0144}, + {0x0148, 0x014B}, {0x014D, 0x014D}, {0x0152, 0x0153}, + {0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE}, + {0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4}, + {0x01D6, 0x01D6}, {0x01D8, 0x01D8}, {0x01DA, 0x01DA}, + {0x01DC, 0x01DC}, {0x0251, 0x0251}, {0x0261, 0x0261}, + {0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB}, + {0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB}, + {0x02DD, 0x02DD}, {0x02DF, 0x02DF}, {0x0300, 0x036F}, + {0x0391, 0x03A1}, {0x03A3, 0x03A9}, {0x03B1, 0x03C1}, + {0x03C3, 0x03C9}, {0x0401, 0x0401}, {0x0410, 0x044F}, + {0x0451, 0x0451}, {0x2010, 0x2010}, {0x2013, 0x2016}, + {0x2018, 0x2019}, {0x201C, 0x201D}, {0x2020, 0x2022}, + {0x2024, 0x2027}, {0x2030, 0x2030}, {0x2032, 0x2033}, + {0x2035, 0x2035}, {0x203B, 0x203B}, {0x203E, 0x203E}, + {0x2074, 0x2074}, {0x207F, 0x207F}, {0x2081, 0x2084}, + {0x20AC, 0x20AC}, {0x2103, 0x2103}, {0x2105, 0x2105}, + {0x2109, 0x2109}, {0x2113, 0x2113}, {0x2116, 0x2116}, + {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B}, + {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, + {0x2170, 0x2179}, {0x2189, 0x2189}, {0x2190, 0x2199}, + {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4}, + {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, + {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, + {0x2211, 0x2211}, {0x2215, 0x2215}, {0x221A, 0x221A}, + {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225}, + {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, + {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, + {0x2252, 0x2252}, {0x2260, 0x2261}, {0x2264, 0x2267}, + {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283}, + {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, + {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, + {0x2460, 0x24E9}, {0x24EB, 0x254B}, {0x2550, 0x2573}, + {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1}, + {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, + {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, + {0x25CB, 0x25CB}, {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, + {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609}, + {0x260E, 0x260F}, {0x261C, 0x261C}, {0x261E, 0x261E}, + {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, + {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, + {0x266F, 0x266F}, {0x269E, 0x269F}, {0x26BF, 0x26BF}, + {0x26C6, 0x26CD}, {0x26CF, 0x26D3}, {0x26D5, 0x26E1}, + {0x26E3, 0x26E3}, {0x26E8, 0x26E9}, {0x26EB, 0x26F1}, + {0x26F4, 0x26F4}, {0x26F6, 0x26F9}, {0x26FB, 0x26FC}, + {0x26FE, 0x26FF}, {0x273D, 0x273D}, {0x2776, 0x277F}, + {0x2B56, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF}, + {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, {0x1F19B, 0x1F1AC}, {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, +} + +var emoji = table{ + {0x1F1E6, 0x1F1FF}, {0x1F321, 0x1F321}, {0x1F324, 0x1F32C}, + {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, {0x1F396, 0x1F397}, + {0x1F399, 0x1F39B}, {0x1F39E, 0x1F39F}, {0x1F3CB, 0x1F3CE}, + {0x1F3D4, 0x1F3DF}, {0x1F3F3, 0x1F3F5}, {0x1F3F7, 0x1F3F7}, + {0x1F43F, 0x1F43F}, {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FD}, + {0x1F549, 0x1F54A}, {0x1F56F, 0x1F570}, {0x1F573, 0x1F579}, + {0x1F587, 0x1F587}, {0x1F58A, 0x1F58D}, {0x1F590, 0x1F590}, + {0x1F5A5, 0x1F5A5}, {0x1F5A8, 0x1F5A8}, {0x1F5B1, 0x1F5B2}, + {0x1F5BC, 0x1F5BC}, {0x1F5C2, 0x1F5C4}, {0x1F5D1, 0x1F5D3}, + {0x1F5DC, 0x1F5DE}, {0x1F5E1, 0x1F5E1}, {0x1F5E3, 0x1F5E3}, + {0x1F5E8, 0x1F5E8}, {0x1F5EF, 0x1F5EF}, {0x1F5F3, 0x1F5F3}, + {0x1F5FA, 0x1F5FA}, {0x1F6CB, 0x1F6CF}, {0x1F6E0, 0x1F6E5}, + {0x1F6E9, 0x1F6E9}, {0x1F6F0, 0x1F6F0}, {0x1F6F3, 0x1F6F3}, +} + +var notassigned = table{ + {0x0378, 0x0379}, {0x0380, 0x0383}, {0x038B, 0x038B}, + {0x038D, 0x038D}, {0x03A2, 0x03A2}, {0x0530, 0x0530}, + {0x0557, 0x0558}, {0x0560, 0x0560}, {0x0588, 0x0588}, + {0x058B, 0x058C}, {0x0590, 0x0590}, {0x05C8, 0x05CF}, + {0x05EB, 0x05EF}, {0x05F5, 0x05FF}, {0x061D, 0x061D}, + {0x070E, 0x070E}, {0x074B, 0x074C}, {0x07B2, 0x07BF}, + {0x07FB, 0x07FF}, {0x082E, 0x082F}, {0x083F, 0x083F}, + {0x085C, 0x085D}, {0x085F, 0x089F}, {0x08B5, 0x08B5}, + {0x08BE, 0x08D3}, {0x0984, 0x0984}, {0x098D, 0x098E}, + {0x0991, 0x0992}, {0x09A9, 0x09A9}, {0x09B1, 0x09B1}, + {0x09B3, 0x09B5}, {0x09BA, 0x09BB}, {0x09C5, 0x09C6}, + {0x09C9, 0x09CA}, {0x09CF, 0x09D6}, {0x09D8, 0x09DB}, + {0x09DE, 0x09DE}, {0x09E4, 0x09E5}, {0x09FC, 0x0A00}, + {0x0A04, 0x0A04}, {0x0A0B, 0x0A0E}, {0x0A11, 0x0A12}, + {0x0A29, 0x0A29}, {0x0A31, 0x0A31}, {0x0A34, 0x0A34}, + {0x0A37, 0x0A37}, {0x0A3A, 0x0A3B}, {0x0A3D, 0x0A3D}, + {0x0A43, 0x0A46}, {0x0A49, 0x0A4A}, {0x0A4E, 0x0A50}, + {0x0A52, 0x0A58}, {0x0A5D, 0x0A5D}, {0x0A5F, 0x0A65}, + {0x0A76, 0x0A80}, {0x0A84, 0x0A84}, {0x0A8E, 0x0A8E}, + {0x0A92, 0x0A92}, {0x0AA9, 0x0AA9}, {0x0AB1, 0x0AB1}, + {0x0AB4, 0x0AB4}, {0x0ABA, 0x0ABB}, {0x0AC6, 0x0AC6}, + {0x0ACA, 0x0ACA}, {0x0ACE, 0x0ACF}, {0x0AD1, 0x0ADF}, + {0x0AE4, 0x0AE5}, {0x0AF2, 0x0AF8}, {0x0AFA, 0x0B00}, + {0x0B04, 0x0B04}, {0x0B0D, 0x0B0E}, {0x0B11, 0x0B12}, + {0x0B29, 0x0B29}, {0x0B31, 0x0B31}, {0x0B34, 0x0B34}, + {0x0B3A, 0x0B3B}, {0x0B45, 0x0B46}, {0x0B49, 0x0B4A}, + {0x0B4E, 0x0B55}, {0x0B58, 0x0B5B}, {0x0B5E, 0x0B5E}, + {0x0B64, 0x0B65}, {0x0B78, 0x0B81}, {0x0B84, 0x0B84}, + {0x0B8B, 0x0B8D}, {0x0B91, 0x0B91}, {0x0B96, 0x0B98}, + {0x0B9B, 0x0B9B}, {0x0B9D, 0x0B9D}, {0x0BA0, 0x0BA2}, + {0x0BA5, 0x0BA7}, {0x0BAB, 0x0BAD}, {0x0BBA, 0x0BBD}, + {0x0BC3, 0x0BC5}, {0x0BC9, 0x0BC9}, {0x0BCE, 0x0BCF}, + {0x0BD1, 0x0BD6}, {0x0BD8, 0x0BE5}, {0x0BFB, 0x0BFF}, + {0x0C04, 0x0C04}, {0x0C0D, 0x0C0D}, {0x0C11, 0x0C11}, + {0x0C29, 0x0C29}, {0x0C3A, 0x0C3C}, {0x0C45, 0x0C45}, + {0x0C49, 0x0C49}, {0x0C4E, 0x0C54}, {0x0C57, 0x0C57}, + {0x0C5B, 0x0C5F}, {0x0C64, 0x0C65}, {0x0C70, 0x0C77}, + {0x0C84, 0x0C84}, {0x0C8D, 0x0C8D}, {0x0C91, 0x0C91}, + {0x0CA9, 0x0CA9}, {0x0CB4, 0x0CB4}, {0x0CBA, 0x0CBB}, + {0x0CC5, 0x0CC5}, {0x0CC9, 0x0CC9}, {0x0CCE, 0x0CD4}, + {0x0CD7, 0x0CDD}, {0x0CDF, 0x0CDF}, {0x0CE4, 0x0CE5}, + {0x0CF0, 0x0CF0}, {0x0CF3, 0x0D00}, {0x0D04, 0x0D04}, + {0x0D0D, 0x0D0D}, {0x0D11, 0x0D11}, {0x0D3B, 0x0D3C}, + {0x0D45, 0x0D45}, {0x0D49, 0x0D49}, {0x0D50, 0x0D53}, + {0x0D64, 0x0D65}, {0x0D80, 0x0D81}, {0x0D84, 0x0D84}, + {0x0D97, 0x0D99}, {0x0DB2, 0x0DB2}, {0x0DBC, 0x0DBC}, + {0x0DBE, 0x0DBF}, {0x0DC7, 0x0DC9}, {0x0DCB, 0x0DCE}, + {0x0DD5, 0x0DD5}, {0x0DD7, 0x0DD7}, {0x0DE0, 0x0DE5}, + {0x0DF0, 0x0DF1}, {0x0DF5, 0x0E00}, {0x0E3B, 0x0E3E}, + {0x0E5C, 0x0E80}, {0x0E83, 0x0E83}, {0x0E85, 0x0E86}, + {0x0E89, 0x0E89}, {0x0E8B, 0x0E8C}, {0x0E8E, 0x0E93}, + {0x0E98, 0x0E98}, {0x0EA0, 0x0EA0}, {0x0EA4, 0x0EA4}, + {0x0EA6, 0x0EA6}, {0x0EA8, 0x0EA9}, {0x0EAC, 0x0EAC}, + {0x0EBA, 0x0EBA}, {0x0EBE, 0x0EBF}, {0x0EC5, 0x0EC5}, + {0x0EC7, 0x0EC7}, {0x0ECE, 0x0ECF}, {0x0EDA, 0x0EDB}, + {0x0EE0, 0x0EFF}, {0x0F48, 0x0F48}, {0x0F6D, 0x0F70}, + {0x0F98, 0x0F98}, {0x0FBD, 0x0FBD}, {0x0FCD, 0x0FCD}, + {0x0FDB, 0x0FFF}, {0x10C6, 0x10C6}, {0x10C8, 0x10CC}, + {0x10CE, 0x10CF}, {0x1249, 0x1249}, {0x124E, 0x124F}, + {0x1257, 0x1257}, {0x1259, 0x1259}, {0x125E, 0x125F}, + {0x1289, 0x1289}, {0x128E, 0x128F}, {0x12B1, 0x12B1}, + {0x12B6, 0x12B7}, {0x12BF, 0x12BF}, {0x12C1, 0x12C1}, + {0x12C6, 0x12C7}, {0x12D7, 0x12D7}, {0x1311, 0x1311}, + {0x1316, 0x1317}, {0x135B, 0x135C}, {0x137D, 0x137F}, + {0x139A, 0x139F}, {0x13F6, 0x13F7}, {0x13FE, 0x13FF}, + {0x169D, 0x169F}, {0x16F9, 0x16FF}, {0x170D, 0x170D}, + {0x1715, 0x171F}, {0x1737, 0x173F}, {0x1754, 0x175F}, + {0x176D, 0x176D}, {0x1771, 0x1771}, {0x1774, 0x177F}, + {0x17DE, 0x17DF}, {0x17EA, 0x17EF}, {0x17FA, 0x17FF}, + {0x180F, 0x180F}, {0x181A, 0x181F}, {0x1878, 0x187F}, + {0x18AB, 0x18AF}, {0x18F6, 0x18FF}, {0x191F, 0x191F}, + {0x192C, 0x192F}, {0x193C, 0x193F}, {0x1941, 0x1943}, + {0x196E, 0x196F}, {0x1975, 0x197F}, {0x19AC, 0x19AF}, + {0x19CA, 0x19CF}, {0x19DB, 0x19DD}, {0x1A1C, 0x1A1D}, + {0x1A5F, 0x1A5F}, {0x1A7D, 0x1A7E}, {0x1A8A, 0x1A8F}, + {0x1A9A, 0x1A9F}, {0x1AAE, 0x1AAF}, {0x1ABF, 0x1AFF}, + {0x1B4C, 0x1B4F}, {0x1B7D, 0x1B7F}, {0x1BF4, 0x1BFB}, + {0x1C38, 0x1C3A}, {0x1C4A, 0x1C4C}, {0x1C89, 0x1CBF}, + {0x1CC8, 0x1CCF}, {0x1CF7, 0x1CF7}, {0x1CFA, 0x1CFF}, + {0x1DF6, 0x1DFA}, {0x1F16, 0x1F17}, {0x1F1E, 0x1F1F}, + {0x1F46, 0x1F47}, {0x1F4E, 0x1F4F}, {0x1F58, 0x1F58}, + {0x1F5A, 0x1F5A}, {0x1F5C, 0x1F5C}, {0x1F5E, 0x1F5E}, + {0x1F7E, 0x1F7F}, {0x1FB5, 0x1FB5}, {0x1FC5, 0x1FC5}, + {0x1FD4, 0x1FD5}, {0x1FDC, 0x1FDC}, {0x1FF0, 0x1FF1}, + {0x1FF5, 0x1FF5}, {0x1FFF, 0x1FFF}, {0x2065, 0x2065}, + {0x2072, 0x2073}, {0x208F, 0x208F}, {0x209D, 0x209F}, + {0x20BF, 0x20CF}, {0x20F1, 0x20FF}, {0x218C, 0x218F}, + {0x23FF, 0x23FF}, {0x2427, 0x243F}, {0x244B, 0x245F}, + {0x2B74, 0x2B75}, {0x2B96, 0x2B97}, {0x2BBA, 0x2BBC}, + {0x2BC9, 0x2BC9}, {0x2BD2, 0x2BEB}, {0x2BF0, 0x2BFF}, + {0x2C2F, 0x2C2F}, {0x2C5F, 0x2C5F}, {0x2CF4, 0x2CF8}, + {0x2D26, 0x2D26}, {0x2D28, 0x2D2C}, {0x2D2E, 0x2D2F}, + {0x2D68, 0x2D6E}, {0x2D71, 0x2D7E}, {0x2D97, 0x2D9F}, + {0x2DA7, 0x2DA7}, {0x2DAF, 0x2DAF}, {0x2DB7, 0x2DB7}, + {0x2DBF, 0x2DBF}, {0x2DC7, 0x2DC7}, {0x2DCF, 0x2DCF}, + {0x2DD7, 0x2DD7}, {0x2DDF, 0x2DDF}, {0x2E45, 0x2E7F}, + {0x2E9A, 0x2E9A}, {0x2EF4, 0x2EFF}, {0x2FD6, 0x2FEF}, + {0x2FFC, 0x2FFF}, {0x3040, 0x3040}, {0x3097, 0x3098}, + {0x3100, 0x3104}, {0x312E, 0x3130}, {0x318F, 0x318F}, + {0x31BB, 0x31BF}, {0x31E4, 0x31EF}, {0x321F, 0x321F}, + {0x32FF, 0x32FF}, {0x4DB6, 0x4DBF}, {0x9FD6, 0x9FFF}, + {0xA48D, 0xA48F}, {0xA4C7, 0xA4CF}, {0xA62C, 0xA63F}, + {0xA6F8, 0xA6FF}, {0xA7AF, 0xA7AF}, {0xA7B8, 0xA7F6}, + {0xA82C, 0xA82F}, {0xA83A, 0xA83F}, {0xA878, 0xA87F}, + {0xA8C6, 0xA8CD}, {0xA8DA, 0xA8DF}, {0xA8FE, 0xA8FF}, + {0xA954, 0xA95E}, {0xA97D, 0xA97F}, {0xA9CE, 0xA9CE}, + {0xA9DA, 0xA9DD}, {0xA9FF, 0xA9FF}, {0xAA37, 0xAA3F}, + {0xAA4E, 0xAA4F}, {0xAA5A, 0xAA5B}, {0xAAC3, 0xAADA}, + {0xAAF7, 0xAB00}, {0xAB07, 0xAB08}, {0xAB0F, 0xAB10}, + {0xAB17, 0xAB1F}, {0xAB27, 0xAB27}, {0xAB2F, 0xAB2F}, + {0xAB66, 0xAB6F}, {0xABEE, 0xABEF}, {0xABFA, 0xABFF}, + {0xD7A4, 0xD7AF}, {0xD7C7, 0xD7CA}, {0xD7FC, 0xD7FF}, + {0xFA6E, 0xFA6F}, {0xFADA, 0xFAFF}, {0xFB07, 0xFB12}, + {0xFB18, 0xFB1C}, {0xFB37, 0xFB37}, {0xFB3D, 0xFB3D}, + {0xFB3F, 0xFB3F}, {0xFB42, 0xFB42}, {0xFB45, 0xFB45}, + {0xFBC2, 0xFBD2}, {0xFD40, 0xFD4F}, {0xFD90, 0xFD91}, + {0xFDC8, 0xFDEF}, {0xFDFE, 0xFDFF}, {0xFE1A, 0xFE1F}, + {0xFE53, 0xFE53}, {0xFE67, 0xFE67}, {0xFE6C, 0xFE6F}, + {0xFE75, 0xFE75}, {0xFEFD, 0xFEFE}, {0xFF00, 0xFF00}, + {0xFFBF, 0xFFC1}, {0xFFC8, 0xFFC9}, {0xFFD0, 0xFFD1}, + {0xFFD8, 0xFFD9}, {0xFFDD, 0xFFDF}, {0xFFE7, 0xFFE7}, + {0xFFEF, 0xFFF8}, {0xFFFE, 0xFFFF}, {0x1000C, 0x1000C}, + {0x10027, 0x10027}, {0x1003B, 0x1003B}, {0x1003E, 0x1003E}, + {0x1004E, 0x1004F}, {0x1005E, 0x1007F}, {0x100FB, 0x100FF}, + {0x10103, 0x10106}, {0x10134, 0x10136}, {0x1018F, 0x1018F}, + {0x1019C, 0x1019F}, {0x101A1, 0x101CF}, {0x101FE, 0x1027F}, + {0x1029D, 0x1029F}, {0x102D1, 0x102DF}, {0x102FC, 0x102FF}, + {0x10324, 0x1032F}, {0x1034B, 0x1034F}, {0x1037B, 0x1037F}, + {0x1039E, 0x1039E}, {0x103C4, 0x103C7}, {0x103D6, 0x103FF}, + {0x1049E, 0x1049F}, {0x104AA, 0x104AF}, {0x104D4, 0x104D7}, + {0x104FC, 0x104FF}, {0x10528, 0x1052F}, {0x10564, 0x1056E}, + {0x10570, 0x105FF}, {0x10737, 0x1073F}, {0x10756, 0x1075F}, + {0x10768, 0x107FF}, {0x10806, 0x10807}, {0x10809, 0x10809}, + {0x10836, 0x10836}, {0x10839, 0x1083B}, {0x1083D, 0x1083E}, + {0x10856, 0x10856}, {0x1089F, 0x108A6}, {0x108B0, 0x108DF}, + {0x108F3, 0x108F3}, {0x108F6, 0x108FA}, {0x1091C, 0x1091E}, + {0x1093A, 0x1093E}, {0x10940, 0x1097F}, {0x109B8, 0x109BB}, + {0x109D0, 0x109D1}, {0x10A04, 0x10A04}, {0x10A07, 0x10A0B}, + {0x10A14, 0x10A14}, {0x10A18, 0x10A18}, {0x10A34, 0x10A37}, + {0x10A3B, 0x10A3E}, {0x10A48, 0x10A4F}, {0x10A59, 0x10A5F}, + {0x10AA0, 0x10ABF}, {0x10AE7, 0x10AEA}, {0x10AF7, 0x10AFF}, + {0x10B36, 0x10B38}, {0x10B56, 0x10B57}, {0x10B73, 0x10B77}, + {0x10B92, 0x10B98}, {0x10B9D, 0x10BA8}, {0x10BB0, 0x10BFF}, + {0x10C49, 0x10C7F}, {0x10CB3, 0x10CBF}, {0x10CF3, 0x10CF9}, + {0x10D00, 0x10E5F}, {0x10E7F, 0x10FFF}, {0x1104E, 0x11051}, + {0x11070, 0x1107E}, {0x110C2, 0x110CF}, {0x110E9, 0x110EF}, + {0x110FA, 0x110FF}, {0x11135, 0x11135}, {0x11144, 0x1114F}, + {0x11177, 0x1117F}, {0x111CE, 0x111CF}, {0x111E0, 0x111E0}, + {0x111F5, 0x111FF}, {0x11212, 0x11212}, {0x1123F, 0x1127F}, + {0x11287, 0x11287}, {0x11289, 0x11289}, {0x1128E, 0x1128E}, + {0x1129E, 0x1129E}, {0x112AA, 0x112AF}, {0x112EB, 0x112EF}, + {0x112FA, 0x112FF}, {0x11304, 0x11304}, {0x1130D, 0x1130E}, + {0x11311, 0x11312}, {0x11329, 0x11329}, {0x11331, 0x11331}, + {0x11334, 0x11334}, {0x1133A, 0x1133B}, {0x11345, 0x11346}, + {0x11349, 0x1134A}, {0x1134E, 0x1134F}, {0x11351, 0x11356}, + {0x11358, 0x1135C}, {0x11364, 0x11365}, {0x1136D, 0x1136F}, + {0x11375, 0x113FF}, {0x1145A, 0x1145A}, {0x1145C, 0x1145C}, + {0x1145E, 0x1147F}, {0x114C8, 0x114CF}, {0x114DA, 0x1157F}, + {0x115B6, 0x115B7}, {0x115DE, 0x115FF}, {0x11645, 0x1164F}, + {0x1165A, 0x1165F}, {0x1166D, 0x1167F}, {0x116B8, 0x116BF}, + {0x116CA, 0x116FF}, {0x1171A, 0x1171C}, {0x1172C, 0x1172F}, + {0x11740, 0x1189F}, {0x118F3, 0x118FE}, {0x11900, 0x11ABF}, + {0x11AF9, 0x11BFF}, {0x11C09, 0x11C09}, {0x11C37, 0x11C37}, + {0x11C46, 0x11C4F}, {0x11C6D, 0x11C6F}, {0x11C90, 0x11C91}, + {0x11CA8, 0x11CA8}, {0x11CB7, 0x11FFF}, {0x1239A, 0x123FF}, + {0x1246F, 0x1246F}, {0x12475, 0x1247F}, {0x12544, 0x12FFF}, + {0x1342F, 0x143FF}, {0x14647, 0x167FF}, {0x16A39, 0x16A3F}, + {0x16A5F, 0x16A5F}, {0x16A6A, 0x16A6D}, {0x16A70, 0x16ACF}, + {0x16AEE, 0x16AEF}, {0x16AF6, 0x16AFF}, {0x16B46, 0x16B4F}, + {0x16B5A, 0x16B5A}, {0x16B62, 0x16B62}, {0x16B78, 0x16B7C}, + {0x16B90, 0x16EFF}, {0x16F45, 0x16F4F}, {0x16F7F, 0x16F8E}, + {0x16FA0, 0x16FDF}, {0x16FE1, 0x16FFF}, {0x187ED, 0x187FF}, + {0x18AF3, 0x1AFFF}, {0x1B002, 0x1BBFF}, {0x1BC6B, 0x1BC6F}, + {0x1BC7D, 0x1BC7F}, {0x1BC89, 0x1BC8F}, {0x1BC9A, 0x1BC9B}, + {0x1BCA4, 0x1CFFF}, {0x1D0F6, 0x1D0FF}, {0x1D127, 0x1D128}, + {0x1D1E9, 0x1D1FF}, {0x1D246, 0x1D2FF}, {0x1D357, 0x1D35F}, + {0x1D372, 0x1D3FF}, {0x1D455, 0x1D455}, {0x1D49D, 0x1D49D}, + {0x1D4A0, 0x1D4A1}, {0x1D4A3, 0x1D4A4}, {0x1D4A7, 0x1D4A8}, + {0x1D4AD, 0x1D4AD}, {0x1D4BA, 0x1D4BA}, {0x1D4BC, 0x1D4BC}, + {0x1D4C4, 0x1D4C4}, {0x1D506, 0x1D506}, {0x1D50B, 0x1D50C}, + {0x1D515, 0x1D515}, {0x1D51D, 0x1D51D}, {0x1D53A, 0x1D53A}, + {0x1D53F, 0x1D53F}, {0x1D545, 0x1D545}, {0x1D547, 0x1D549}, + {0x1D551, 0x1D551}, {0x1D6A6, 0x1D6A7}, {0x1D7CC, 0x1D7CD}, + {0x1DA8C, 0x1DA9A}, {0x1DAA0, 0x1DAA0}, {0x1DAB0, 0x1DFFF}, + {0x1E007, 0x1E007}, {0x1E019, 0x1E01A}, {0x1E022, 0x1E022}, + {0x1E025, 0x1E025}, {0x1E02B, 0x1E7FF}, {0x1E8C5, 0x1E8C6}, + {0x1E8D7, 0x1E8FF}, {0x1E94B, 0x1E94F}, {0x1E95A, 0x1E95D}, + {0x1E960, 0x1EDFF}, {0x1EE04, 0x1EE04}, {0x1EE20, 0x1EE20}, + {0x1EE23, 0x1EE23}, {0x1EE25, 0x1EE26}, {0x1EE28, 0x1EE28}, + {0x1EE33, 0x1EE33}, {0x1EE38, 0x1EE38}, {0x1EE3A, 0x1EE3A}, + {0x1EE3C, 0x1EE41}, {0x1EE43, 0x1EE46}, {0x1EE48, 0x1EE48}, + {0x1EE4A, 0x1EE4A}, {0x1EE4C, 0x1EE4C}, {0x1EE50, 0x1EE50}, + {0x1EE53, 0x1EE53}, {0x1EE55, 0x1EE56}, {0x1EE58, 0x1EE58}, + {0x1EE5A, 0x1EE5A}, {0x1EE5C, 0x1EE5C}, {0x1EE5E, 0x1EE5E}, + {0x1EE60, 0x1EE60}, {0x1EE63, 0x1EE63}, {0x1EE65, 0x1EE66}, + {0x1EE6B, 0x1EE6B}, {0x1EE73, 0x1EE73}, {0x1EE78, 0x1EE78}, + {0x1EE7D, 0x1EE7D}, {0x1EE7F, 0x1EE7F}, {0x1EE8A, 0x1EE8A}, + {0x1EE9C, 0x1EEA0}, {0x1EEA4, 0x1EEA4}, {0x1EEAA, 0x1EEAA}, + {0x1EEBC, 0x1EEEF}, {0x1EEF2, 0x1EFFF}, {0x1F02C, 0x1F02F}, + {0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0}, + {0x1F0D0, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F10D, 0x1F10F}, + {0x1F12F, 0x1F12F}, {0x1F16C, 0x1F16F}, {0x1F1AD, 0x1F1E5}, + {0x1F203, 0x1F20F}, {0x1F23C, 0x1F23F}, {0x1F249, 0x1F24F}, + {0x1F252, 0x1F2FF}, {0x1F6D3, 0x1F6DF}, {0x1F6ED, 0x1F6EF}, + {0x1F6F7, 0x1F6FF}, {0x1F774, 0x1F77F}, {0x1F7D5, 0x1F7FF}, + {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, {0x1F85A, 0x1F85F}, + {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F90F}, {0x1F91F, 0x1F91F}, + {0x1F928, 0x1F92F}, {0x1F931, 0x1F932}, {0x1F93F, 0x1F93F}, + {0x1F94C, 0x1F94F}, {0x1F95F, 0x1F97F}, {0x1F992, 0x1F9BF}, + {0x1F9C1, 0x1FFFF}, {0x2A6D7, 0x2A6FF}, {0x2B735, 0x2B73F}, + {0x2B81E, 0x2B81F}, {0x2CEA2, 0x2F7FF}, {0x2FA1E, 0xE0000}, + {0xE0002, 0xE001F}, {0xE0080, 0xE00FF}, {0xE01F0, 0xEFFFF}, + {0xFFFFE, 0xFFFFF}, +} + +var neutral = table{ + {0x0000, 0x001F}, {0x007F, 0x007F}, {0x0080, 0x009F}, + {0x00A0, 0x00A0}, {0x00A9, 0x00A9}, {0x00AB, 0x00AB}, + {0x00B5, 0x00B5}, {0x00BB, 0x00BB}, {0x00C0, 0x00C5}, + {0x00C7, 0x00CF}, {0x00D1, 0x00D6}, {0x00D9, 0x00DD}, + {0x00E2, 0x00E5}, {0x00E7, 0x00E7}, {0x00EB, 0x00EB}, + {0x00EE, 0x00EF}, {0x00F1, 0x00F1}, {0x00F4, 0x00F6}, + {0x00FB, 0x00FB}, {0x00FD, 0x00FD}, {0x00FF, 0x00FF}, + {0x0100, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112}, + {0x0114, 0x011A}, {0x011C, 0x0125}, {0x0128, 0x012A}, + {0x012C, 0x0130}, {0x0134, 0x0137}, {0x0139, 0x013E}, + {0x0143, 0x0143}, {0x0145, 0x0147}, {0x014C, 0x014C}, + {0x014E, 0x0151}, {0x0154, 0x0165}, {0x0168, 0x016A}, + {0x016C, 0x017F}, {0x0180, 0x01BA}, {0x01BB, 0x01BB}, + {0x01BC, 0x01BF}, {0x01C0, 0x01C3}, {0x01C4, 0x01CD}, + {0x01CF, 0x01CF}, {0x01D1, 0x01D1}, {0x01D3, 0x01D3}, + {0x01D5, 0x01D5}, {0x01D7, 0x01D7}, {0x01D9, 0x01D9}, + {0x01DB, 0x01DB}, {0x01DD, 0x024F}, {0x0250, 0x0250}, + {0x0252, 0x0260}, {0x0262, 0x0293}, {0x0294, 0x0294}, + {0x0295, 0x02AF}, {0x02B0, 0x02C1}, {0x02C2, 0x02C3}, + {0x02C5, 0x02C5}, {0x02C6, 0x02C6}, {0x02C8, 0x02C8}, + {0x02CC, 0x02CC}, {0x02CE, 0x02CF}, {0x02D1, 0x02D1}, + {0x02D2, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE}, + {0x02E0, 0x02E4}, {0x02E5, 0x02EB}, {0x02EC, 0x02EC}, + {0x02ED, 0x02ED}, {0x02EE, 0x02EE}, {0x02EF, 0x02FF}, + {0x0370, 0x0373}, {0x0374, 0x0374}, {0x0375, 0x0375}, + {0x0376, 0x0377}, {0x037A, 0x037A}, {0x037B, 0x037D}, + {0x037E, 0x037E}, {0x037F, 0x037F}, {0x0384, 0x0385}, + {0x0386, 0x0386}, {0x0387, 0x0387}, {0x0388, 0x038A}, + {0x038C, 0x038C}, {0x038E, 0x0390}, {0x03AA, 0x03B0}, + {0x03C2, 0x03C2}, {0x03CA, 0x03F5}, {0x03F6, 0x03F6}, + {0x03F7, 0x03FF}, {0x0400, 0x0400}, {0x0402, 0x040F}, + {0x0450, 0x0450}, {0x0452, 0x0481}, {0x0482, 0x0482}, + {0x0483, 0x0487}, {0x0488, 0x0489}, {0x048A, 0x04FF}, + {0x0500, 0x052F}, {0x0531, 0x0556}, {0x0559, 0x0559}, + {0x055A, 0x055F}, {0x0561, 0x0587}, {0x0589, 0x0589}, + {0x058A, 0x058A}, {0x058D, 0x058E}, {0x058F, 0x058F}, + {0x0591, 0x05BD}, {0x05BE, 0x05BE}, {0x05BF, 0x05BF}, + {0x05C0, 0x05C0}, {0x05C1, 0x05C2}, {0x05C3, 0x05C3}, + {0x05C4, 0x05C5}, {0x05C6, 0x05C6}, {0x05C7, 0x05C7}, + {0x05D0, 0x05EA}, {0x05F0, 0x05F2}, {0x05F3, 0x05F4}, + {0x0600, 0x0605}, {0x0606, 0x0608}, {0x0609, 0x060A}, + {0x060B, 0x060B}, {0x060C, 0x060D}, {0x060E, 0x060F}, + {0x0610, 0x061A}, {0x061B, 0x061B}, {0x061C, 0x061C}, + {0x061E, 0x061F}, {0x0620, 0x063F}, {0x0640, 0x0640}, + {0x0641, 0x064A}, {0x064B, 0x065F}, {0x0660, 0x0669}, + {0x066A, 0x066D}, {0x066E, 0x066F}, {0x0670, 0x0670}, + {0x0671, 0x06D3}, {0x06D4, 0x06D4}, {0x06D5, 0x06D5}, + {0x06D6, 0x06DC}, {0x06DD, 0x06DD}, {0x06DE, 0x06DE}, + {0x06DF, 0x06E4}, {0x06E5, 0x06E6}, {0x06E7, 0x06E8}, + {0x06E9, 0x06E9}, {0x06EA, 0x06ED}, {0x06EE, 0x06EF}, + {0x06F0, 0x06F9}, {0x06FA, 0x06FC}, {0x06FD, 0x06FE}, + {0x06FF, 0x06FF}, {0x0700, 0x070D}, {0x070F, 0x070F}, + {0x0710, 0x0710}, {0x0711, 0x0711}, {0x0712, 0x072F}, + {0x0730, 0x074A}, {0x074D, 0x074F}, {0x0750, 0x077F}, + {0x0780, 0x07A5}, {0x07A6, 0x07B0}, {0x07B1, 0x07B1}, + {0x07C0, 0x07C9}, {0x07CA, 0x07EA}, {0x07EB, 0x07F3}, + {0x07F4, 0x07F5}, {0x07F6, 0x07F6}, {0x07F7, 0x07F9}, + {0x07FA, 0x07FA}, {0x0800, 0x0815}, {0x0816, 0x0819}, + {0x081A, 0x081A}, {0x081B, 0x0823}, {0x0824, 0x0824}, + {0x0825, 0x0827}, {0x0828, 0x0828}, {0x0829, 0x082D}, + {0x0830, 0x083E}, {0x0840, 0x0858}, {0x0859, 0x085B}, + {0x085E, 0x085E}, {0x08A0, 0x08B4}, {0x08B6, 0x08BD}, + {0x08D4, 0x08E1}, {0x08E2, 0x08E2}, {0x08E3, 0x08FF}, + {0x0900, 0x0902}, {0x0903, 0x0903}, {0x0904, 0x0939}, + {0x093A, 0x093A}, {0x093B, 0x093B}, {0x093C, 0x093C}, + {0x093D, 0x093D}, {0x093E, 0x0940}, {0x0941, 0x0948}, + {0x0949, 0x094C}, {0x094D, 0x094D}, {0x094E, 0x094F}, + {0x0950, 0x0950}, {0x0951, 0x0957}, {0x0958, 0x0961}, + {0x0962, 0x0963}, {0x0964, 0x0965}, {0x0966, 0x096F}, + {0x0970, 0x0970}, {0x0971, 0x0971}, {0x0972, 0x097F}, + {0x0980, 0x0980}, {0x0981, 0x0981}, {0x0982, 0x0983}, + {0x0985, 0x098C}, {0x098F, 0x0990}, {0x0993, 0x09A8}, + {0x09AA, 0x09B0}, {0x09B2, 0x09B2}, {0x09B6, 0x09B9}, + {0x09BC, 0x09BC}, {0x09BD, 0x09BD}, {0x09BE, 0x09C0}, + {0x09C1, 0x09C4}, {0x09C7, 0x09C8}, {0x09CB, 0x09CC}, + {0x09CD, 0x09CD}, {0x09CE, 0x09CE}, {0x09D7, 0x09D7}, + {0x09DC, 0x09DD}, {0x09DF, 0x09E1}, {0x09E2, 0x09E3}, + {0x09E6, 0x09EF}, {0x09F0, 0x09F1}, {0x09F2, 0x09F3}, + {0x09F4, 0x09F9}, {0x09FA, 0x09FA}, {0x09FB, 0x09FB}, + {0x0A01, 0x0A02}, {0x0A03, 0x0A03}, {0x0A05, 0x0A0A}, + {0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, + {0x0A32, 0x0A33}, {0x0A35, 0x0A36}, {0x0A38, 0x0A39}, + {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A40}, {0x0A41, 0x0A42}, + {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, + {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, {0x0A66, 0x0A6F}, + {0x0A70, 0x0A71}, {0x0A72, 0x0A74}, {0x0A75, 0x0A75}, + {0x0A81, 0x0A82}, {0x0A83, 0x0A83}, {0x0A85, 0x0A8D}, + {0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0}, + {0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0ABC}, + {0x0ABD, 0x0ABD}, {0x0ABE, 0x0AC0}, {0x0AC1, 0x0AC5}, + {0x0AC7, 0x0AC8}, {0x0AC9, 0x0AC9}, {0x0ACB, 0x0ACC}, + {0x0ACD, 0x0ACD}, {0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE1}, + {0x0AE2, 0x0AE3}, {0x0AE6, 0x0AEF}, {0x0AF0, 0x0AF0}, + {0x0AF1, 0x0AF1}, {0x0AF9, 0x0AF9}, {0x0B01, 0x0B01}, + {0x0B02, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10}, + {0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, {0x0B32, 0x0B33}, + {0x0B35, 0x0B39}, {0x0B3C, 0x0B3C}, {0x0B3D, 0x0B3D}, + {0x0B3E, 0x0B3E}, {0x0B3F, 0x0B3F}, {0x0B40, 0x0B40}, + {0x0B41, 0x0B44}, {0x0B47, 0x0B48}, {0x0B4B, 0x0B4C}, + {0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B57, 0x0B57}, + {0x0B5C, 0x0B5D}, {0x0B5F, 0x0B61}, {0x0B62, 0x0B63}, + {0x0B66, 0x0B6F}, {0x0B70, 0x0B70}, {0x0B71, 0x0B71}, + {0x0B72, 0x0B77}, {0x0B82, 0x0B82}, {0x0B83, 0x0B83}, + {0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, {0x0B92, 0x0B95}, + {0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F}, + {0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9}, + {0x0BBE, 0x0BBF}, {0x0BC0, 0x0BC0}, {0x0BC1, 0x0BC2}, + {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCC}, {0x0BCD, 0x0BCD}, + {0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BEF}, + {0x0BF0, 0x0BF2}, {0x0BF3, 0x0BF8}, {0x0BF9, 0x0BF9}, + {0x0BFA, 0x0BFA}, {0x0C00, 0x0C00}, {0x0C01, 0x0C03}, + {0x0C05, 0x0C0C}, {0x0C0E, 0x0C10}, {0x0C12, 0x0C28}, + {0x0C2A, 0x0C39}, {0x0C3D, 0x0C3D}, {0x0C3E, 0x0C40}, + {0x0C41, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, + {0x0C55, 0x0C56}, {0x0C58, 0x0C5A}, {0x0C60, 0x0C61}, + {0x0C62, 0x0C63}, {0x0C66, 0x0C6F}, {0x0C78, 0x0C7E}, + {0x0C7F, 0x0C7F}, {0x0C80, 0x0C80}, {0x0C81, 0x0C81}, + {0x0C82, 0x0C83}, {0x0C85, 0x0C8C}, {0x0C8E, 0x0C90}, + {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, + {0x0CBC, 0x0CBC}, {0x0CBD, 0x0CBD}, {0x0CBE, 0x0CBE}, + {0x0CBF, 0x0CBF}, {0x0CC0, 0x0CC4}, {0x0CC6, 0x0CC6}, + {0x0CC7, 0x0CC8}, {0x0CCA, 0x0CCB}, {0x0CCC, 0x0CCD}, + {0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE1}, + {0x0CE2, 0x0CE3}, {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, + {0x0D01, 0x0D01}, {0x0D02, 0x0D03}, {0x0D05, 0x0D0C}, + {0x0D0E, 0x0D10}, {0x0D12, 0x0D3A}, {0x0D3D, 0x0D3D}, + {0x0D3E, 0x0D40}, {0x0D41, 0x0D44}, {0x0D46, 0x0D48}, + {0x0D4A, 0x0D4C}, {0x0D4D, 0x0D4D}, {0x0D4E, 0x0D4E}, + {0x0D4F, 0x0D4F}, {0x0D54, 0x0D56}, {0x0D57, 0x0D57}, + {0x0D58, 0x0D5E}, {0x0D5F, 0x0D61}, {0x0D62, 0x0D63}, + {0x0D66, 0x0D6F}, {0x0D70, 0x0D78}, {0x0D79, 0x0D79}, + {0x0D7A, 0x0D7F}, {0x0D82, 0x0D83}, {0x0D85, 0x0D96}, + {0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD}, + {0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD1}, + {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, + {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF3}, {0x0DF4, 0x0DF4}, + {0x0E01, 0x0E30}, {0x0E31, 0x0E31}, {0x0E32, 0x0E33}, + {0x0E34, 0x0E3A}, {0x0E3F, 0x0E3F}, {0x0E40, 0x0E45}, + {0x0E46, 0x0E46}, {0x0E47, 0x0E4E}, {0x0E4F, 0x0E4F}, + {0x0E50, 0x0E59}, {0x0E5A, 0x0E5B}, {0x0E81, 0x0E82}, + {0x0E84, 0x0E84}, {0x0E87, 0x0E88}, {0x0E8A, 0x0E8A}, + {0x0E8D, 0x0E8D}, {0x0E94, 0x0E97}, {0x0E99, 0x0E9F}, + {0x0EA1, 0x0EA3}, {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EA7}, + {0x0EAA, 0x0EAB}, {0x0EAD, 0x0EB0}, {0x0EB1, 0x0EB1}, + {0x0EB2, 0x0EB3}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, + {0x0EBD, 0x0EBD}, {0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6}, + {0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF}, + {0x0F00, 0x0F00}, {0x0F01, 0x0F03}, {0x0F04, 0x0F12}, + {0x0F13, 0x0F13}, {0x0F14, 0x0F14}, {0x0F15, 0x0F17}, + {0x0F18, 0x0F19}, {0x0F1A, 0x0F1F}, {0x0F20, 0x0F29}, + {0x0F2A, 0x0F33}, {0x0F34, 0x0F34}, {0x0F35, 0x0F35}, + {0x0F36, 0x0F36}, {0x0F37, 0x0F37}, {0x0F38, 0x0F38}, + {0x0F39, 0x0F39}, {0x0F3A, 0x0F3A}, {0x0F3B, 0x0F3B}, + {0x0F3C, 0x0F3C}, {0x0F3D, 0x0F3D}, {0x0F3E, 0x0F3F}, + {0x0F40, 0x0F47}, {0x0F49, 0x0F6C}, {0x0F71, 0x0F7E}, + {0x0F7F, 0x0F7F}, {0x0F80, 0x0F84}, {0x0F85, 0x0F85}, + {0x0F86, 0x0F87}, {0x0F88, 0x0F8C}, {0x0F8D, 0x0F97}, + {0x0F99, 0x0FBC}, {0x0FBE, 0x0FC5}, {0x0FC6, 0x0FC6}, + {0x0FC7, 0x0FCC}, {0x0FCE, 0x0FCF}, {0x0FD0, 0x0FD4}, + {0x0FD5, 0x0FD8}, {0x0FD9, 0x0FDA}, {0x1000, 0x102A}, + {0x102B, 0x102C}, {0x102D, 0x1030}, {0x1031, 0x1031}, + {0x1032, 0x1037}, {0x1038, 0x1038}, {0x1039, 0x103A}, + {0x103B, 0x103C}, {0x103D, 0x103E}, {0x103F, 0x103F}, + {0x1040, 0x1049}, {0x104A, 0x104F}, {0x1050, 0x1055}, + {0x1056, 0x1057}, {0x1058, 0x1059}, {0x105A, 0x105D}, + {0x105E, 0x1060}, {0x1061, 0x1061}, {0x1062, 0x1064}, + {0x1065, 0x1066}, {0x1067, 0x106D}, {0x106E, 0x1070}, + {0x1071, 0x1074}, {0x1075, 0x1081}, {0x1082, 0x1082}, + {0x1083, 0x1084}, {0x1085, 0x1086}, {0x1087, 0x108C}, + {0x108D, 0x108D}, {0x108E, 0x108E}, {0x108F, 0x108F}, + {0x1090, 0x1099}, {0x109A, 0x109C}, {0x109D, 0x109D}, + {0x109E, 0x109F}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FB, 0x10FB}, + {0x10FC, 0x10FC}, {0x10FD, 0x10FF}, {0x1160, 0x11FF}, + {0x1200, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, + {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, + {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, + {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, + {0x1318, 0x135A}, {0x135D, 0x135F}, {0x1360, 0x1368}, + {0x1369, 0x137C}, {0x1380, 0x138F}, {0x1390, 0x1399}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1400, 0x1400}, + {0x1401, 0x166C}, {0x166D, 0x166E}, {0x166F, 0x167F}, + {0x1680, 0x1680}, {0x1681, 0x169A}, {0x169B, 0x169B}, + {0x169C, 0x169C}, {0x16A0, 0x16EA}, {0x16EB, 0x16ED}, + {0x16EE, 0x16F0}, {0x16F1, 0x16F8}, {0x1700, 0x170C}, + {0x170E, 0x1711}, {0x1712, 0x1714}, {0x1720, 0x1731}, + {0x1732, 0x1734}, {0x1735, 0x1736}, {0x1740, 0x1751}, + {0x1752, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770}, + {0x1772, 0x1773}, {0x1780, 0x17B3}, {0x17B4, 0x17B5}, + {0x17B6, 0x17B6}, {0x17B7, 0x17BD}, {0x17BE, 0x17C5}, + {0x17C6, 0x17C6}, {0x17C7, 0x17C8}, {0x17C9, 0x17D3}, + {0x17D4, 0x17D6}, {0x17D7, 0x17D7}, {0x17D8, 0x17DA}, + {0x17DB, 0x17DB}, {0x17DC, 0x17DC}, {0x17DD, 0x17DD}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1800, 0x1805}, + {0x1806, 0x1806}, {0x1807, 0x180A}, {0x180B, 0x180D}, + {0x180E, 0x180E}, {0x1810, 0x1819}, {0x1820, 0x1842}, + {0x1843, 0x1843}, {0x1844, 0x1877}, {0x1880, 0x1884}, + {0x1885, 0x1886}, {0x1887, 0x18A8}, {0x18A9, 0x18A9}, + {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, + {0x1920, 0x1922}, {0x1923, 0x1926}, {0x1927, 0x1928}, + {0x1929, 0x192B}, {0x1930, 0x1931}, {0x1932, 0x1932}, + {0x1933, 0x1938}, {0x1939, 0x193B}, {0x1940, 0x1940}, + {0x1944, 0x1945}, {0x1946, 0x194F}, {0x1950, 0x196D}, + {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, + {0x19D0, 0x19D9}, {0x19DA, 0x19DA}, {0x19DE, 0x19DF}, + {0x19E0, 0x19FF}, {0x1A00, 0x1A16}, {0x1A17, 0x1A18}, + {0x1A19, 0x1A1A}, {0x1A1B, 0x1A1B}, {0x1A1E, 0x1A1F}, + {0x1A20, 0x1A54}, {0x1A55, 0x1A55}, {0x1A56, 0x1A56}, + {0x1A57, 0x1A57}, {0x1A58, 0x1A5E}, {0x1A60, 0x1A60}, + {0x1A61, 0x1A61}, {0x1A62, 0x1A62}, {0x1A63, 0x1A64}, + {0x1A65, 0x1A6C}, {0x1A6D, 0x1A72}, {0x1A73, 0x1A7C}, + {0x1A7F, 0x1A7F}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, + {0x1AA0, 0x1AA6}, {0x1AA7, 0x1AA7}, {0x1AA8, 0x1AAD}, + {0x1AB0, 0x1ABD}, {0x1ABE, 0x1ABE}, {0x1B00, 0x1B03}, + {0x1B04, 0x1B04}, {0x1B05, 0x1B33}, {0x1B34, 0x1B34}, + {0x1B35, 0x1B35}, {0x1B36, 0x1B3A}, {0x1B3B, 0x1B3B}, + {0x1B3C, 0x1B3C}, {0x1B3D, 0x1B41}, {0x1B42, 0x1B42}, + {0x1B43, 0x1B44}, {0x1B45, 0x1B4B}, {0x1B50, 0x1B59}, + {0x1B5A, 0x1B60}, {0x1B61, 0x1B6A}, {0x1B6B, 0x1B73}, + {0x1B74, 0x1B7C}, {0x1B80, 0x1B81}, {0x1B82, 0x1B82}, + {0x1B83, 0x1BA0}, {0x1BA1, 0x1BA1}, {0x1BA2, 0x1BA5}, + {0x1BA6, 0x1BA7}, {0x1BA8, 0x1BA9}, {0x1BAA, 0x1BAA}, + {0x1BAB, 0x1BAD}, {0x1BAE, 0x1BAF}, {0x1BB0, 0x1BB9}, + {0x1BBA, 0x1BBF}, {0x1BC0, 0x1BE5}, {0x1BE6, 0x1BE6}, + {0x1BE7, 0x1BE7}, {0x1BE8, 0x1BE9}, {0x1BEA, 0x1BEC}, + {0x1BED, 0x1BED}, {0x1BEE, 0x1BEE}, {0x1BEF, 0x1BF1}, + {0x1BF2, 0x1BF3}, {0x1BFC, 0x1BFF}, {0x1C00, 0x1C23}, + {0x1C24, 0x1C2B}, {0x1C2C, 0x1C33}, {0x1C34, 0x1C35}, + {0x1C36, 0x1C37}, {0x1C3B, 0x1C3F}, {0x1C40, 0x1C49}, + {0x1C4D, 0x1C4F}, {0x1C50, 0x1C59}, {0x1C5A, 0x1C77}, + {0x1C78, 0x1C7D}, {0x1C7E, 0x1C7F}, {0x1C80, 0x1C88}, + {0x1CC0, 0x1CC7}, {0x1CD0, 0x1CD2}, {0x1CD3, 0x1CD3}, + {0x1CD4, 0x1CE0}, {0x1CE1, 0x1CE1}, {0x1CE2, 0x1CE8}, + {0x1CE9, 0x1CEC}, {0x1CED, 0x1CED}, {0x1CEE, 0x1CF1}, + {0x1CF2, 0x1CF3}, {0x1CF4, 0x1CF4}, {0x1CF5, 0x1CF6}, + {0x1CF8, 0x1CF9}, {0x1D00, 0x1D2B}, {0x1D2C, 0x1D6A}, + {0x1D6B, 0x1D77}, {0x1D78, 0x1D78}, {0x1D79, 0x1D7F}, + {0x1D80, 0x1D9A}, {0x1D9B, 0x1DBF}, {0x1DC0, 0x1DF5}, + {0x1DFB, 0x1DFF}, {0x1E00, 0x1EFF}, {0x1F00, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, {0x1FBD, 0x1FBD}, {0x1FBE, 0x1FBE}, + {0x1FBF, 0x1FC1}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, + {0x1FCD, 0x1FCF}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, + {0x1FDD, 0x1FDF}, {0x1FE0, 0x1FEC}, {0x1FED, 0x1FEF}, + {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x1FFD, 0x1FFE}, + {0x2000, 0x200A}, {0x200B, 0x200F}, {0x2011, 0x2012}, + {0x2017, 0x2017}, {0x201A, 0x201A}, {0x201B, 0x201B}, + {0x201E, 0x201E}, {0x201F, 0x201F}, {0x2023, 0x2023}, + {0x2028, 0x2028}, {0x2029, 0x2029}, {0x202A, 0x202E}, + {0x202F, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034}, + {0x2036, 0x2038}, {0x2039, 0x2039}, {0x203A, 0x203A}, + {0x203C, 0x203D}, {0x203F, 0x2040}, {0x2041, 0x2043}, + {0x2044, 0x2044}, {0x2045, 0x2045}, {0x2046, 0x2046}, + {0x2047, 0x2051}, {0x2052, 0x2052}, {0x2053, 0x2053}, + {0x2054, 0x2054}, {0x2055, 0x205E}, {0x205F, 0x205F}, + {0x2060, 0x2064}, {0x2066, 0x206F}, {0x2070, 0x2070}, + {0x2071, 0x2071}, {0x2075, 0x2079}, {0x207A, 0x207C}, + {0x207D, 0x207D}, {0x207E, 0x207E}, {0x2080, 0x2080}, + {0x2085, 0x2089}, {0x208A, 0x208C}, {0x208D, 0x208D}, + {0x208E, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8}, + {0x20AA, 0x20AB}, {0x20AD, 0x20BE}, {0x20D0, 0x20DC}, + {0x20DD, 0x20E0}, {0x20E1, 0x20E1}, {0x20E2, 0x20E4}, + {0x20E5, 0x20F0}, {0x2100, 0x2101}, {0x2102, 0x2102}, + {0x2104, 0x2104}, {0x2106, 0x2106}, {0x2107, 0x2107}, + {0x2108, 0x2108}, {0x210A, 0x2112}, {0x2114, 0x2114}, + {0x2115, 0x2115}, {0x2117, 0x2117}, {0x2118, 0x2118}, + {0x2119, 0x211D}, {0x211E, 0x2120}, {0x2123, 0x2123}, + {0x2124, 0x2124}, {0x2125, 0x2125}, {0x2127, 0x2127}, + {0x2128, 0x2128}, {0x2129, 0x2129}, {0x212A, 0x212A}, + {0x212C, 0x212D}, {0x212E, 0x212E}, {0x212F, 0x2134}, + {0x2135, 0x2138}, {0x2139, 0x2139}, {0x213A, 0x213B}, + {0x213C, 0x213F}, {0x2140, 0x2144}, {0x2145, 0x2149}, + {0x214A, 0x214A}, {0x214B, 0x214B}, {0x214C, 0x214D}, + {0x214E, 0x214E}, {0x214F, 0x214F}, {0x2150, 0x2152}, + {0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F}, + {0x217A, 0x2182}, {0x2183, 0x2184}, {0x2185, 0x2188}, + {0x218A, 0x218B}, {0x219A, 0x219B}, {0x219C, 0x219F}, + {0x21A0, 0x21A0}, {0x21A1, 0x21A2}, {0x21A3, 0x21A3}, + {0x21A4, 0x21A5}, {0x21A6, 0x21A6}, {0x21A7, 0x21AD}, + {0x21AE, 0x21AE}, {0x21AF, 0x21B7}, {0x21BA, 0x21CD}, + {0x21CE, 0x21CF}, {0x21D0, 0x21D1}, {0x21D3, 0x21D3}, + {0x21D5, 0x21E6}, {0x21E8, 0x21F3}, {0x21F4, 0x21FF}, + {0x2201, 0x2201}, {0x2204, 0x2206}, {0x2209, 0x220A}, + {0x220C, 0x220E}, {0x2210, 0x2210}, {0x2212, 0x2214}, + {0x2216, 0x2219}, {0x221B, 0x221C}, {0x2221, 0x2222}, + {0x2224, 0x2224}, {0x2226, 0x2226}, {0x222D, 0x222D}, + {0x222F, 0x2233}, {0x2238, 0x223B}, {0x223E, 0x2247}, + {0x2249, 0x224B}, {0x224D, 0x2251}, {0x2253, 0x225F}, + {0x2262, 0x2263}, {0x2268, 0x2269}, {0x226C, 0x226D}, + {0x2270, 0x2281}, {0x2284, 0x2285}, {0x2288, 0x2294}, + {0x2296, 0x2298}, {0x229A, 0x22A4}, {0x22A6, 0x22BE}, + {0x22C0, 0x22FF}, {0x2300, 0x2307}, {0x2308, 0x2308}, + {0x2309, 0x2309}, {0x230A, 0x230A}, {0x230B, 0x230B}, + {0x230C, 0x2311}, {0x2313, 0x2319}, {0x231C, 0x231F}, + {0x2320, 0x2321}, {0x2322, 0x2328}, {0x232B, 0x237B}, + {0x237C, 0x237C}, {0x237D, 0x239A}, {0x239B, 0x23B3}, + {0x23B4, 0x23DB}, {0x23DC, 0x23E1}, {0x23E2, 0x23E8}, + {0x23ED, 0x23EF}, {0x23F1, 0x23F2}, {0x23F4, 0x23FE}, + {0x2400, 0x2426}, {0x2440, 0x244A}, {0x24EA, 0x24EA}, + {0x254C, 0x254F}, {0x2574, 0x257F}, {0x2590, 0x2591}, + {0x2596, 0x259F}, {0x25A2, 0x25A2}, {0x25AA, 0x25B1}, + {0x25B4, 0x25B5}, {0x25B8, 0x25BB}, {0x25BE, 0x25BF}, + {0x25C2, 0x25C5}, {0x25C9, 0x25CA}, {0x25CC, 0x25CD}, + {0x25D2, 0x25E1}, {0x25E6, 0x25EE}, {0x25F0, 0x25F7}, + {0x25F8, 0x25FC}, {0x25FF, 0x25FF}, {0x2600, 0x2604}, + {0x2607, 0x2608}, {0x260A, 0x260D}, {0x2610, 0x2613}, + {0x2616, 0x261B}, {0x261D, 0x261D}, {0x261F, 0x263F}, + {0x2641, 0x2641}, {0x2643, 0x2647}, {0x2654, 0x265F}, + {0x2662, 0x2662}, {0x2666, 0x2666}, {0x266B, 0x266B}, + {0x266E, 0x266E}, {0x2670, 0x267E}, {0x2680, 0x2692}, + {0x2694, 0x269D}, {0x26A0, 0x26A0}, {0x26A2, 0x26A9}, + {0x26AC, 0x26BC}, {0x26C0, 0x26C3}, {0x26E2, 0x26E2}, + {0x26E4, 0x26E7}, {0x2700, 0x2704}, {0x2706, 0x2709}, + {0x270C, 0x2727}, {0x2729, 0x273C}, {0x273E, 0x274B}, + {0x274D, 0x274D}, {0x274F, 0x2752}, {0x2756, 0x2756}, + {0x2758, 0x2767}, {0x2768, 0x2768}, {0x2769, 0x2769}, + {0x276A, 0x276A}, {0x276B, 0x276B}, {0x276C, 0x276C}, + {0x276D, 0x276D}, {0x276E, 0x276E}, {0x276F, 0x276F}, + {0x2770, 0x2770}, {0x2771, 0x2771}, {0x2772, 0x2772}, + {0x2773, 0x2773}, {0x2774, 0x2774}, {0x2775, 0x2775}, + {0x2780, 0x2793}, {0x2794, 0x2794}, {0x2798, 0x27AF}, + {0x27B1, 0x27BE}, {0x27C0, 0x27C4}, {0x27C5, 0x27C5}, + {0x27C6, 0x27C6}, {0x27C7, 0x27E5}, {0x27EE, 0x27EE}, + {0x27EF, 0x27EF}, {0x27F0, 0x27FF}, {0x2800, 0x28FF}, + {0x2900, 0x297F}, {0x2980, 0x2982}, {0x2983, 0x2983}, + {0x2984, 0x2984}, {0x2987, 0x2987}, {0x2988, 0x2988}, + {0x2989, 0x2989}, {0x298A, 0x298A}, {0x298B, 0x298B}, + {0x298C, 0x298C}, {0x298D, 0x298D}, {0x298E, 0x298E}, + {0x298F, 0x298F}, {0x2990, 0x2990}, {0x2991, 0x2991}, + {0x2992, 0x2992}, {0x2993, 0x2993}, {0x2994, 0x2994}, + {0x2995, 0x2995}, {0x2996, 0x2996}, {0x2997, 0x2997}, + {0x2998, 0x2998}, {0x2999, 0x29D7}, {0x29D8, 0x29D8}, + {0x29D9, 0x29D9}, {0x29DA, 0x29DA}, {0x29DB, 0x29DB}, + {0x29DC, 0x29FB}, {0x29FC, 0x29FC}, {0x29FD, 0x29FD}, + {0x29FE, 0x29FF}, {0x2A00, 0x2AFF}, {0x2B00, 0x2B1A}, + {0x2B1D, 0x2B2F}, {0x2B30, 0x2B44}, {0x2B45, 0x2B46}, + {0x2B47, 0x2B4C}, {0x2B4D, 0x2B4F}, {0x2B51, 0x2B54}, + {0x2B5A, 0x2B73}, {0x2B76, 0x2B95}, {0x2B98, 0x2BB9}, + {0x2BBD, 0x2BC8}, {0x2BCA, 0x2BD1}, {0x2BEC, 0x2BEF}, + {0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2C7B}, + {0x2C7C, 0x2C7D}, {0x2C7E, 0x2C7F}, {0x2C80, 0x2CE4}, + {0x2CE5, 0x2CEA}, {0x2CEB, 0x2CEE}, {0x2CEF, 0x2CF1}, + {0x2CF2, 0x2CF3}, {0x2CF9, 0x2CFC}, {0x2CFD, 0x2CFD}, + {0x2CFE, 0x2CFF}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, + {0x2D70, 0x2D70}, {0x2D7F, 0x2D7F}, {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, + {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, + {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2DE0, 0x2DFF}, + {0x2E00, 0x2E01}, {0x2E02, 0x2E02}, {0x2E03, 0x2E03}, + {0x2E04, 0x2E04}, {0x2E05, 0x2E05}, {0x2E06, 0x2E08}, + {0x2E09, 0x2E09}, {0x2E0A, 0x2E0A}, {0x2E0B, 0x2E0B}, + {0x2E0C, 0x2E0C}, {0x2E0D, 0x2E0D}, {0x2E0E, 0x2E16}, + {0x2E17, 0x2E17}, {0x2E18, 0x2E19}, {0x2E1A, 0x2E1A}, + {0x2E1B, 0x2E1B}, {0x2E1C, 0x2E1C}, {0x2E1D, 0x2E1D}, + {0x2E1E, 0x2E1F}, {0x2E20, 0x2E20}, {0x2E21, 0x2E21}, + {0x2E22, 0x2E22}, {0x2E23, 0x2E23}, {0x2E24, 0x2E24}, + {0x2E25, 0x2E25}, {0x2E26, 0x2E26}, {0x2E27, 0x2E27}, + {0x2E28, 0x2E28}, {0x2E29, 0x2E29}, {0x2E2A, 0x2E2E}, + {0x2E2F, 0x2E2F}, {0x2E30, 0x2E39}, {0x2E3A, 0x2E3B}, + {0x2E3C, 0x2E3F}, {0x2E40, 0x2E40}, {0x2E41, 0x2E41}, + {0x2E42, 0x2E42}, {0x2E43, 0x2E44}, {0x303F, 0x303F}, + {0x4DC0, 0x4DFF}, {0xA4D0, 0xA4F7}, {0xA4F8, 0xA4FD}, + {0xA4FE, 0xA4FF}, {0xA500, 0xA60B}, {0xA60C, 0xA60C}, + {0xA60D, 0xA60F}, {0xA610, 0xA61F}, {0xA620, 0xA629}, + {0xA62A, 0xA62B}, {0xA640, 0xA66D}, {0xA66E, 0xA66E}, + {0xA66F, 0xA66F}, {0xA670, 0xA672}, {0xA673, 0xA673}, + {0xA674, 0xA67D}, {0xA67E, 0xA67E}, {0xA67F, 0xA67F}, + {0xA680, 0xA69B}, {0xA69C, 0xA69D}, {0xA69E, 0xA69F}, + {0xA6A0, 0xA6E5}, {0xA6E6, 0xA6EF}, {0xA6F0, 0xA6F1}, + {0xA6F2, 0xA6F7}, {0xA700, 0xA716}, {0xA717, 0xA71F}, + {0xA720, 0xA721}, {0xA722, 0xA76F}, {0xA770, 0xA770}, + {0xA771, 0xA787}, {0xA788, 0xA788}, {0xA789, 0xA78A}, + {0xA78B, 0xA78E}, {0xA78F, 0xA78F}, {0xA790, 0xA7AE}, + {0xA7B0, 0xA7B7}, {0xA7F7, 0xA7F7}, {0xA7F8, 0xA7F9}, + {0xA7FA, 0xA7FA}, {0xA7FB, 0xA7FF}, {0xA800, 0xA801}, + {0xA802, 0xA802}, {0xA803, 0xA805}, {0xA806, 0xA806}, + {0xA807, 0xA80A}, {0xA80B, 0xA80B}, {0xA80C, 0xA822}, + {0xA823, 0xA824}, {0xA825, 0xA826}, {0xA827, 0xA827}, + {0xA828, 0xA82B}, {0xA830, 0xA835}, {0xA836, 0xA837}, + {0xA838, 0xA838}, {0xA839, 0xA839}, {0xA840, 0xA873}, + {0xA874, 0xA877}, {0xA880, 0xA881}, {0xA882, 0xA8B3}, + {0xA8B4, 0xA8C3}, {0xA8C4, 0xA8C5}, {0xA8CE, 0xA8CF}, + {0xA8D0, 0xA8D9}, {0xA8E0, 0xA8F1}, {0xA8F2, 0xA8F7}, + {0xA8F8, 0xA8FA}, {0xA8FB, 0xA8FB}, {0xA8FC, 0xA8FC}, + {0xA8FD, 0xA8FD}, {0xA900, 0xA909}, {0xA90A, 0xA925}, + {0xA926, 0xA92D}, {0xA92E, 0xA92F}, {0xA930, 0xA946}, + {0xA947, 0xA951}, {0xA952, 0xA953}, {0xA95F, 0xA95F}, + {0xA980, 0xA982}, {0xA983, 0xA983}, {0xA984, 0xA9B2}, + {0xA9B3, 0xA9B3}, {0xA9B4, 0xA9B5}, {0xA9B6, 0xA9B9}, + {0xA9BA, 0xA9BB}, {0xA9BC, 0xA9BC}, {0xA9BD, 0xA9C0}, + {0xA9C1, 0xA9CD}, {0xA9CF, 0xA9CF}, {0xA9D0, 0xA9D9}, + {0xA9DE, 0xA9DF}, {0xA9E0, 0xA9E4}, {0xA9E5, 0xA9E5}, + {0xA9E6, 0xA9E6}, {0xA9E7, 0xA9EF}, {0xA9F0, 0xA9F9}, + {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA29, 0xAA2E}, + {0xAA2F, 0xAA30}, {0xAA31, 0xAA32}, {0xAA33, 0xAA34}, + {0xAA35, 0xAA36}, {0xAA40, 0xAA42}, {0xAA43, 0xAA43}, + {0xAA44, 0xAA4B}, {0xAA4C, 0xAA4C}, {0xAA4D, 0xAA4D}, + {0xAA50, 0xAA59}, {0xAA5C, 0xAA5F}, {0xAA60, 0xAA6F}, + {0xAA70, 0xAA70}, {0xAA71, 0xAA76}, {0xAA77, 0xAA79}, + {0xAA7A, 0xAA7A}, {0xAA7B, 0xAA7B}, {0xAA7C, 0xAA7C}, + {0xAA7D, 0xAA7D}, {0xAA7E, 0xAA7F}, {0xAA80, 0xAAAF}, + {0xAAB0, 0xAAB0}, {0xAAB1, 0xAAB1}, {0xAAB2, 0xAAB4}, + {0xAAB5, 0xAAB6}, {0xAAB7, 0xAAB8}, {0xAAB9, 0xAABD}, + {0xAABE, 0xAABF}, {0xAAC0, 0xAAC0}, {0xAAC1, 0xAAC1}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADC}, {0xAADD, 0xAADD}, + {0xAADE, 0xAADF}, {0xAAE0, 0xAAEA}, {0xAAEB, 0xAAEB}, + {0xAAEC, 0xAAED}, {0xAAEE, 0xAAEF}, {0xAAF0, 0xAAF1}, + {0xAAF2, 0xAAF2}, {0xAAF3, 0xAAF4}, {0xAAF5, 0xAAF5}, + {0xAAF6, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, + {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, + {0xAB30, 0xAB5A}, {0xAB5B, 0xAB5B}, {0xAB5C, 0xAB5F}, + {0xAB60, 0xAB65}, {0xAB70, 0xABBF}, {0xABC0, 0xABE2}, + {0xABE3, 0xABE4}, {0xABE5, 0xABE5}, {0xABE6, 0xABE7}, + {0xABE8, 0xABE8}, {0xABE9, 0xABEA}, {0xABEB, 0xABEB}, + {0xABEC, 0xABEC}, {0xABED, 0xABED}, {0xABF0, 0xABF9}, + {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDB7F}, + {0xDB80, 0xDBFF}, {0xDC00, 0xDFFF}, {0xFB00, 0xFB06}, + {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1E, 0xFB1E}, + {0xFB1F, 0xFB28}, {0xFB29, 0xFB29}, {0xFB2A, 0xFB36}, + {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, + {0xFB43, 0xFB44}, {0xFB46, 0xFB4F}, {0xFB50, 0xFBB1}, + {0xFBB2, 0xFBC1}, {0xFBD3, 0xFD3D}, {0xFD3E, 0xFD3E}, + {0xFD3F, 0xFD3F}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, + {0xFDF0, 0xFDFB}, {0xFDFC, 0xFDFC}, {0xFDFD, 0xFDFD}, + {0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFC, 0xFFFC}, + {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, + {0x10080, 0x100FA}, {0x10100, 0x10102}, {0x10107, 0x10133}, + {0x10137, 0x1013F}, {0x10140, 0x10174}, {0x10175, 0x10178}, + {0x10179, 0x10189}, {0x1018A, 0x1018B}, {0x1018C, 0x1018E}, + {0x10190, 0x1019B}, {0x101A0, 0x101A0}, {0x101D0, 0x101FC}, + {0x101FD, 0x101FD}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, + {0x102E0, 0x102E0}, {0x102E1, 0x102FB}, {0x10300, 0x1031F}, + {0x10320, 0x10323}, {0x10330, 0x10340}, {0x10341, 0x10341}, + {0x10342, 0x10349}, {0x1034A, 0x1034A}, {0x10350, 0x10375}, + {0x10376, 0x1037A}, {0x10380, 0x1039D}, {0x1039F, 0x1039F}, + {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x103D0, 0x103D0}, + {0x103D1, 0x103D5}, {0x10400, 0x1044F}, {0x10450, 0x1047F}, + {0x10480, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, + {0x1056F, 0x1056F}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x1083F}, {0x10840, 0x10855}, {0x10857, 0x10857}, + {0x10858, 0x1085F}, {0x10860, 0x10876}, {0x10877, 0x10878}, + {0x10879, 0x1087F}, {0x10880, 0x1089E}, {0x108A7, 0x108AF}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x108FF}, + {0x10900, 0x10915}, {0x10916, 0x1091B}, {0x1091F, 0x1091F}, + {0x10920, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x1099F}, + {0x109A0, 0x109B7}, {0x109BC, 0x109BD}, {0x109BE, 0x109BF}, + {0x109C0, 0x109CF}, {0x109D2, 0x109FF}, {0x10A00, 0x10A00}, + {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, + {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A33}, + {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x10A40, 0x10A47}, + {0x10A50, 0x10A58}, {0x10A60, 0x10A7C}, {0x10A7D, 0x10A7E}, + {0x10A7F, 0x10A7F}, {0x10A80, 0x10A9C}, {0x10A9D, 0x10A9F}, + {0x10AC0, 0x10AC7}, {0x10AC8, 0x10AC8}, {0x10AC9, 0x10AE4}, + {0x10AE5, 0x10AE6}, {0x10AEB, 0x10AEF}, {0x10AF0, 0x10AF6}, + {0x10B00, 0x10B35}, {0x10B39, 0x10B3F}, {0x10B40, 0x10B55}, + {0x10B58, 0x10B5F}, {0x10B60, 0x10B72}, {0x10B78, 0x10B7F}, + {0x10B80, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10CFF}, {0x10E60, 0x10E7E}, {0x11000, 0x11000}, + {0x11001, 0x11001}, {0x11002, 0x11002}, {0x11003, 0x11037}, + {0x11038, 0x11046}, {0x11047, 0x1104D}, {0x11052, 0x11065}, + {0x11066, 0x1106F}, {0x1107F, 0x1107F}, {0x11080, 0x11081}, + {0x11082, 0x11082}, {0x11083, 0x110AF}, {0x110B0, 0x110B2}, + {0x110B3, 0x110B6}, {0x110B7, 0x110B8}, {0x110B9, 0x110BA}, + {0x110BB, 0x110BC}, {0x110BD, 0x110BD}, {0x110BE, 0x110C1}, + {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11100, 0x11102}, + {0x11103, 0x11126}, {0x11127, 0x1112B}, {0x1112C, 0x1112C}, + {0x1112D, 0x11134}, {0x11136, 0x1113F}, {0x11140, 0x11143}, + {0x11150, 0x11172}, {0x11173, 0x11173}, {0x11174, 0x11175}, + {0x11176, 0x11176}, {0x11180, 0x11181}, {0x11182, 0x11182}, + {0x11183, 0x111B2}, {0x111B3, 0x111B5}, {0x111B6, 0x111BE}, + {0x111BF, 0x111C0}, {0x111C1, 0x111C4}, {0x111C5, 0x111C9}, + {0x111CA, 0x111CC}, {0x111CD, 0x111CD}, {0x111D0, 0x111D9}, + {0x111DA, 0x111DA}, {0x111DB, 0x111DB}, {0x111DC, 0x111DC}, + {0x111DD, 0x111DF}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1122C, 0x1122E}, {0x1122F, 0x11231}, + {0x11232, 0x11233}, {0x11234, 0x11234}, {0x11235, 0x11235}, + {0x11236, 0x11237}, {0x11238, 0x1123D}, {0x1123E, 0x1123E}, + {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, + {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112A9, 0x112A9}, + {0x112B0, 0x112DE}, {0x112DF, 0x112DF}, {0x112E0, 0x112E2}, + {0x112E3, 0x112EA}, {0x112F0, 0x112F9}, {0x11300, 0x11301}, + {0x11302, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, + {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, + {0x11335, 0x11339}, {0x1133C, 0x1133C}, {0x1133D, 0x1133D}, + {0x1133E, 0x1133F}, {0x11340, 0x11340}, {0x11341, 0x11344}, + {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11350, 0x11350}, + {0x11357, 0x11357}, {0x1135D, 0x11361}, {0x11362, 0x11363}, + {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11400, 0x11434}, + {0x11435, 0x11437}, {0x11438, 0x1143F}, {0x11440, 0x11441}, + {0x11442, 0x11444}, {0x11445, 0x11445}, {0x11446, 0x11446}, + {0x11447, 0x1144A}, {0x1144B, 0x1144F}, {0x11450, 0x11459}, + {0x1145B, 0x1145B}, {0x1145D, 0x1145D}, {0x11480, 0x114AF}, + {0x114B0, 0x114B2}, {0x114B3, 0x114B8}, {0x114B9, 0x114B9}, + {0x114BA, 0x114BA}, {0x114BB, 0x114BE}, {0x114BF, 0x114C0}, + {0x114C1, 0x114C1}, {0x114C2, 0x114C3}, {0x114C4, 0x114C5}, + {0x114C6, 0x114C6}, {0x114C7, 0x114C7}, {0x114D0, 0x114D9}, + {0x11580, 0x115AE}, {0x115AF, 0x115B1}, {0x115B2, 0x115B5}, + {0x115B8, 0x115BB}, {0x115BC, 0x115BD}, {0x115BE, 0x115BE}, + {0x115BF, 0x115C0}, {0x115C1, 0x115D7}, {0x115D8, 0x115DB}, + {0x115DC, 0x115DD}, {0x11600, 0x1162F}, {0x11630, 0x11632}, + {0x11633, 0x1163A}, {0x1163B, 0x1163C}, {0x1163D, 0x1163D}, + {0x1163E, 0x1163E}, {0x1163F, 0x11640}, {0x11641, 0x11643}, + {0x11644, 0x11644}, {0x11650, 0x11659}, {0x11660, 0x1166C}, + {0x11680, 0x116AA}, {0x116AB, 0x116AB}, {0x116AC, 0x116AC}, + {0x116AD, 0x116AD}, {0x116AE, 0x116AF}, {0x116B0, 0x116B5}, + {0x116B6, 0x116B6}, {0x116B7, 0x116B7}, {0x116C0, 0x116C9}, + {0x11700, 0x11719}, {0x1171D, 0x1171F}, {0x11720, 0x11721}, + {0x11722, 0x11725}, {0x11726, 0x11726}, {0x11727, 0x1172B}, + {0x11730, 0x11739}, {0x1173A, 0x1173B}, {0x1173C, 0x1173E}, + {0x1173F, 0x1173F}, {0x118A0, 0x118DF}, {0x118E0, 0x118E9}, + {0x118EA, 0x118F2}, {0x118FF, 0x118FF}, {0x11AC0, 0x11AF8}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C2F, 0x11C2F}, + {0x11C30, 0x11C36}, {0x11C38, 0x11C3D}, {0x11C3E, 0x11C3E}, + {0x11C3F, 0x11C3F}, {0x11C40, 0x11C40}, {0x11C41, 0x11C45}, + {0x11C50, 0x11C59}, {0x11C5A, 0x11C6C}, {0x11C70, 0x11C71}, + {0x11C72, 0x11C8F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CA9}, + {0x11CAA, 0x11CB0}, {0x11CB1, 0x11CB1}, {0x11CB2, 0x11CB3}, + {0x11CB4, 0x11CB4}, {0x11CB5, 0x11CB6}, {0x12000, 0x12399}, + {0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543}, + {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38}, + {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16A6F}, + {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF4}, {0x16AF5, 0x16AF5}, + {0x16B00, 0x16B2F}, {0x16B30, 0x16B36}, {0x16B37, 0x16B3B}, + {0x16B3C, 0x16B3F}, {0x16B40, 0x16B43}, {0x16B44, 0x16B44}, + {0x16B45, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, + {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16F00, 0x16F44}, + {0x16F50, 0x16F50}, {0x16F51, 0x16F7E}, {0x16F8F, 0x16F92}, + {0x16F93, 0x16F9F}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, + {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BC9C}, + {0x1BC9D, 0x1BC9E}, {0x1BC9F, 0x1BC9F}, {0x1BCA0, 0x1BCA3}, + {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D164}, + {0x1D165, 0x1D166}, {0x1D167, 0x1D169}, {0x1D16A, 0x1D16C}, + {0x1D16D, 0x1D172}, {0x1D173, 0x1D17A}, {0x1D17B, 0x1D182}, + {0x1D183, 0x1D184}, {0x1D185, 0x1D18B}, {0x1D18C, 0x1D1A9}, + {0x1D1AA, 0x1D1AD}, {0x1D1AE, 0x1D1E8}, {0x1D200, 0x1D241}, + {0x1D242, 0x1D244}, {0x1D245, 0x1D245}, {0x1D300, 0x1D356}, + {0x1D360, 0x1D371}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, + {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, + {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, + {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, + {0x1D6C1, 0x1D6C1}, {0x1D6C2, 0x1D6DA}, {0x1D6DB, 0x1D6DB}, + {0x1D6DC, 0x1D6FA}, {0x1D6FB, 0x1D6FB}, {0x1D6FC, 0x1D714}, + {0x1D715, 0x1D715}, {0x1D716, 0x1D734}, {0x1D735, 0x1D735}, + {0x1D736, 0x1D74E}, {0x1D74F, 0x1D74F}, {0x1D750, 0x1D76E}, + {0x1D76F, 0x1D76F}, {0x1D770, 0x1D788}, {0x1D789, 0x1D789}, + {0x1D78A, 0x1D7A8}, {0x1D7A9, 0x1D7A9}, {0x1D7AA, 0x1D7C2}, + {0x1D7C3, 0x1D7C3}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, + {0x1D800, 0x1D9FF}, {0x1DA00, 0x1DA36}, {0x1DA37, 0x1DA3A}, + {0x1DA3B, 0x1DA6C}, {0x1DA6D, 0x1DA74}, {0x1DA75, 0x1DA75}, + {0x1DA76, 0x1DA83}, {0x1DA84, 0x1DA84}, {0x1DA85, 0x1DA86}, + {0x1DA87, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, + {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, + {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E800, 0x1E8C4}, + {0x1E8C7, 0x1E8CF}, {0x1E8D0, 0x1E8D6}, {0x1E900, 0x1E943}, + {0x1E944, 0x1E94A}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, + {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, + {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, + {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, + {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, + {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, + {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, + {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, + {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F003}, {0x1F005, 0x1F02B}, + {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, + {0x1F0C1, 0x1F0CE}, {0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10C}, + {0x1F12E, 0x1F12E}, {0x1F16A, 0x1F16B}, {0x1F1E6, 0x1F1FF}, + {0x1F321, 0x1F32C}, {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, + {0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE}, {0x1F3D4, 0x1F3DF}, + {0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7}, {0x1F43F, 0x1F43F}, + {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE}, {0x1F53E, 0x1F54A}, + {0x1F54F, 0x1F54F}, {0x1F568, 0x1F579}, {0x1F57B, 0x1F594}, + {0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA}, {0x1F650, 0x1F67F}, + {0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF}, {0x1F6E0, 0x1F6EA}, + {0x1F6F0, 0x1F6F3}, {0x1F700, 0x1F773}, {0x1F780, 0x1F7D4}, + {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, + {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0xE0001, 0xE0001}, + {0xE0020, 0xE007F}, +} + +// Condition have flag EastAsianWidth whether the current locale is CJK or not. +type Condition struct { + EastAsianWidth bool +} + +// NewCondition return new instance of Condition which is current locale. +func NewCondition() *Condition { + return &Condition{EastAsianWidth} +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func (c *Condition) RuneWidth(r rune) int { + switch { + case r < 0 || r > 0x10FFFF || + inTables(r, nonprint, combining, notassigned): + return 0 + case (c.EastAsianWidth && IsAmbiguousWidth(r)) || + inTables(r, doublewidth, emoji): + return 2 + default: + return 1 + } +} + +// StringWidth return width as you can see +func (c *Condition) StringWidth(s string) (width int) { + for _, r := range []rune(s) { + width += c.RuneWidth(r) + } + return width +} + +// Truncate return string truncated with w cells +func (c *Condition) Truncate(s string, w int, tail string) string { + if c.StringWidth(s) <= w { + return s + } + r := []rune(s) + tw := c.StringWidth(tail) + w -= tw + width := 0 + i := 0 + for ; i < len(r); i++ { + cw := c.RuneWidth(r[i]) + if width+cw > w { + break + } + width += cw + } + return string(r[0:i]) + tail +} + +// Wrap return string wrapped with w cells +func (c *Condition) Wrap(s string, w int) string { + width := 0 + out := "" + for _, r := range []rune(s) { + cw := RuneWidth(r) + if r == '\n' { + out += string(r) + width = 0 + continue + } else if width+cw > w { + out += "\n" + width = 0 + out += string(r) + width += cw + continue + } + out += string(r) + width += cw + } + return out +} + +// FillLeft return string filled in left by spaces in w cells +func (c *Condition) FillLeft(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return string(b) + s + } + return s +} + +// FillRight return string filled in left by spaces in w cells +func (c *Condition) FillRight(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return s + string(b) + } + return s +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func RuneWidth(r rune) int { + return DefaultCondition.RuneWidth(r) +} + +// IsAmbiguousWidth returns whether is ambiguous width or not. +func IsAmbiguousWidth(r rune) bool { + return inTables(r, private, ambiguous) +} + +// IsNeutralWidth returns whether is neutral width or not. +func IsNeutralWidth(r rune) bool { + return inTable(r, neutral) +} + +// StringWidth return width as you can see +func StringWidth(s string) (width int) { + return DefaultCondition.StringWidth(s) +} + +// Truncate return string truncated with w cells +func Truncate(s string, w int, tail string) string { + return DefaultCondition.Truncate(s, w, tail) +} + +// Wrap return string wrapped with w cells +func Wrap(s string, w int) string { + return DefaultCondition.Wrap(s, w) +} + +// FillLeft return string filled in left by spaces in w cells +func FillLeft(s string, w int) string { + return DefaultCondition.FillLeft(s, w) +} + +// FillRight return string filled in left by spaces in w cells +func FillRight(s string, w int) string { + return DefaultCondition.FillRight(s, w) +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_js.go b/vendor/github.com/mattn/go-runewidth/runewidth_js.go new file mode 100644 index 00000000000..0ce32c5e7b7 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_js.go @@ -0,0 +1,8 @@ +// +build js + +package runewidth + +func IsEastAsian() bool { + // TODO: Implement this for the web. Detect east asian in a compatible way, and return true. + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_posix.go b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go new file mode 100644 index 00000000000..c579e9a3144 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go @@ -0,0 +1,77 @@ +// +build !windows,!js + +package runewidth + +import ( + "os" + "regexp" + "strings" +) + +var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`) + +var mblenTable = map[string]int{ + "utf-8": 6, + "utf8": 6, + "jis": 8, + "eucjp": 3, + "euckr": 2, + "euccn": 2, + "sjis": 2, + "cp932": 2, + "cp51932": 2, + "cp936": 2, + "cp949": 2, + "cp950": 2, + "big5": 2, + "gbk": 2, + "gb2312": 2, +} + +func isEastAsian(locale string) bool { + charset := strings.ToLower(locale) + r := reLoc.FindStringSubmatch(locale) + if len(r) == 2 { + charset = strings.ToLower(r[1]) + } + + if strings.HasSuffix(charset, "@cjk_narrow") { + return false + } + + for pos, b := range []byte(charset) { + if b == '@' { + charset = charset[:pos] + break + } + } + max := 1 + if m, ok := mblenTable[charset]; ok { + max = m + } + if max > 1 && (charset[0] != 'u' || + strings.HasPrefix(locale, "ja") || + strings.HasPrefix(locale, "ko") || + strings.HasPrefix(locale, "zh")) { + return true + } + return false +} + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + locale := os.Getenv("LC_CTYPE") + if locale == "" { + locale = os.Getenv("LANG") + } + + // ignore C locale + if locale == "POSIX" || locale == "C" { + return false + } + if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') { + return false + } + + return isEastAsian(locale) +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_windows.go b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go new file mode 100644 index 00000000000..0258876b99d --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go @@ -0,0 +1,25 @@ +package runewidth + +import ( + "syscall" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32") + procGetConsoleOutputCP = kernel32.NewProc("GetConsoleOutputCP") +) + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + r1, _, _ := procGetConsoleOutputCP.Call() + if r1 == 0 { + return false + } + + switch int(r1) { + case 932, 51932, 936, 949, 950: + return true + } + + return false +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 4581d05af6c..540c4e9af9c 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -816,6 +816,12 @@ "path": "github.com/hashicorp/yamux", "revision": "df949784da9ed028ee76df44652e42d37a09d7e4" }, + { + "checksumSHA1": "28nznojcl6Ejtm6I1tKozgy0isg=", + "path": "github.com/jlaffaye/ftp", + "revision": "025815df64030c144b86e368fedf80ee195fe464", + "revisionTime": "2016-02-29T21:52:38Z" + }, { "checksumSHA1": "3/Bhy+ua/DCv2ElMD5GzOYSGN6g=", "comment": "0.2.2-2-gc01cf91", @@ -917,6 +923,12 @@ "path": "github.com/mattn/go-isatty", "revision": "56b76bdf51f7708750eac80fa38b952bb9f32639" }, + { + "checksumSHA1": "MNkKJyk2TazKMJYbal5wFHybpyA=", + "path": "github.com/mattn/go-runewidth", + "revision": "14207d285c6c197daabb5c9793d63e7af9ab2d50", + "revisionTime": "2017-02-01T02:35:40Z" + }, { "checksumSHA1": "UP+pXl+ic9y6qrpZA5MqDIAuGfw=", "path": "github.com/mitchellh/cli", From 783c976b2e608ca395eaf5e718b46b8bb43df882 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 1 Mar 2017 14:52:55 -0600 Subject: [PATCH 11/18] Fixed a config_test that should've failed but didn't because ftp:// uris work now. HeH! --- builder/virtualbox/ovf/config_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/builder/virtualbox/ovf/config_test.go b/builder/virtualbox/ovf/config_test.go index 5e67b242db6..e9c536e54f5 100644 --- a/builder/virtualbox/ovf/config_test.go +++ b/builder/virtualbox/ovf/config_test.go @@ -76,17 +76,6 @@ func TestNewConfig_sourcePath(t *testing.T) { t.Fatalf("Nonexistant file should throw a validation error!") } - // Bad - c = testConfig(t) - c["source_path"] = "ftp://i/dont/exist" - _, warns, err = NewConfig(c) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should error") - } - // Good tf := getTempFile(t) defer os.Remove(tf.Name()) From 2995b5b12dea88058fc9ecaaf36504342a3d8977 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 23 Mar 2017 14:51:45 -0500 Subject: [PATCH 12/18] go fmt builder/vmware/iso/step_create_vmx.go to calm down Travis CI. --- builder/vmware/iso/step_create_vmx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 62546d21208..3f3644e7012 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -44,7 +44,7 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) // Convert the iso_path into a path relative to the .vmx file if possible - if relativeIsoPath,err := filepath.Rel(config.VMXTemplatePath, filepath.FromSlash(isoPath)); err == nil { + if relativeIsoPath, err := filepath.Rel(config.VMXTemplatePath, filepath.FromSlash(isoPath)); err == nil { isoPath = relativeIsoPath } From 54c9cea1f65b87daf2a2ea191038222db1067434 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 23 Mar 2017 15:12:25 -0500 Subject: [PATCH 13/18] Fix common/download_test.go to avoid formatting the volume name to a hidden windows share when not on windows. --- common/download_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/download_test.go b/common/download_test.go index 81cc3244f9c..114ca8ec33e 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -452,7 +452,10 @@ func TestFileUriTransforms(t *testing.T) { cwd = filepath.ToSlash(cwd) volume = filepath.VolumeName(cwd) share = volume - if share[len(share)-1] == ':' { + + // if a volume was found (on windows), replace the ':' from + // C: to C$ to convert it into a hidden windows share. + if len(share) > 1 && share[len(share)-1] == ':' { share = share[:len(share)-1] + "$" } cwd = cwd[len(volume):] From 77bdddf18b78cd5dcd8103ed72a735dc9239870d Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 23 Mar 2017 15:29:38 -0500 Subject: [PATCH 14/18] Modified common/download_test.go to not test the smb:// uri on platforms other than windows. Added an immediate platform error to SMBDownloader.Download as opposed to letting .toPath return it (which would have left the structure partially initialized). --- common/download.go | 6 ++++++ common/download_test.go | 24 ++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/common/download.go b/common/download.go index 110cb15365e..5d3621d1cda 100644 --- a/common/download.go +++ b/common/download.go @@ -637,6 +637,12 @@ func (d *SMBDownloader) toPath(base string, uri url.URL) (string, error) { } func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { + + /* first we warn the world if we're not running windows */ + if runtime.GOOS != "windows" { + return fmt.Errorf("Support for SMB based uri's are not supported on %s", runtime.GOOS) + } + d.active = false /* convert the uri using the net/url module to a UNC path */ diff --git a/common/download_test.go b/common/download_test.go index 114ca8ec33e..a62fceb963e 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -483,15 +483,19 @@ func TestFileUriTransforms(t *testing.T) { t.Logf("TestFileUriTransforms : Result Path '%s'", res) } - // ...and finally the oddball windows native path - // smb://host/sharename/file -> \\host\sharename\file - testcase := host + "/" + share + "/" + cwd[1:] + "/%s" - uri := "smb://" + fmt.Sprintf(testcase, testpath) - t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) - res, err := SimulateFileUriDownload(t, uri) - if err != nil { - t.Errorf("Unable to transform uri '%s' into a path", uri) - return + // smb protocol depends on platform support which currently + // only exists on windows. + if runtime.GOOS == "windows" { + // ...and finally the oddball windows native path + // smb://host/sharename/file -> \\host\sharename\file + testcase := host + "/" + share + "/" + cwd[1:] + "/%s" + uri := "smb://" + fmt.Sprintf(testcase, testpath) + t.Logf("TestFileUriTransforms : Trying Uri '%s'", uri) + res, err := SimulateFileUriDownload(t, uri) + if err != nil { + t.Errorf("Unable to transform uri '%s' into a path", uri) + return + } + t.Logf("TestFileUriTransforms : Result Path '%s'", res) } - t.Logf("TestFileUriTransforms : Result Path '%s'", res) } From 7994d86e7cc9575351f26ffbc09857fb2b8a6bb7 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 23 Mar 2017 15:36:40 -0500 Subject: [PATCH 15/18] grr. removed an assignment that was dead in common/download.go. --- common/download.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/download.go b/common/download.go index 5d3621d1cda..188a8ca0740 100644 --- a/common/download.go +++ b/common/download.go @@ -161,7 +161,6 @@ func (d *DownloadClient) Get() (string, error) { local, ok := d.downloader.(LocalDownloader) if !ok && !d.config.CopyFile { return "", fmt.Errorf("Not allowed to use uri scheme %s in no copy file mode : %T", u.Scheme, d.downloader) - d.config.CopyFile = true } // If we're copying the file, then just use the actual downloader From 0b02a49442d81c5130eda66cabbaef1479e62b14 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 6 Dec 2017 16:29:00 -0600 Subject: [PATCH 16/18] Changed a critical error to a non-critical one when dealing with the strange .CopyFile flag in common/download.go. --- common/download.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/download.go b/common/download.go index 188a8ca0740..46cc1213e3e 100644 --- a/common/download.go +++ b/common/download.go @@ -155,12 +155,12 @@ func (d *DownloadClient) Get() (string, error) { remote, ok := d.downloader.(RemoteDownloader) if !ok { - return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader : %T", u.Scheme, d.downloader) + return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader. : %T", u.Scheme, d.downloader) } local, ok := d.downloader.(LocalDownloader) if !ok && !d.config.CopyFile { - return "", fmt.Errorf("Not allowed to use uri scheme %s in no copy file mode : %T", u.Scheme, d.downloader) + d.config.CopyFile = true } // If we're copying the file, then just use the actual downloader From 745d62f6e3f0e8c5915081dd926a7df3d1870439 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 6 Dec 2017 17:57:03 -0600 Subject: [PATCH 17/18] Consolidated the progress-bar's format into common/step_download.go. Removed DownloadClient's PercentProgress callback since cheggaaa's progress-bar already does that. --- common/download.go | 124 ++++++++++++++++++++++------------------ common/step_download.go | 33 +++++++---- 2 files changed, 91 insertions(+), 66 deletions(-) diff --git a/common/download.go b/common/download.go index 46cc1213e3e..be53dfed4fe 100644 --- a/common/download.go +++ b/common/download.go @@ -83,19 +83,19 @@ func HashForType(t string) hash.Hash { // NewDownloadClient returns a new DownloadClient for the given // configuration. -func NewDownloadClient(c *DownloadConfig) *DownloadClient { +func NewDownloadClient(c *DownloadConfig, bar *pb.ProgressBar) *DownloadClient { const mtu = 1500 /* ethernet */ - 20 /* ipv4 */ - 20 /* tcp */ if c.DownloaderMap == nil { + // Create downloader map c.DownloaderMap = map[string]Downloader{ - "file": &FileDownloader{bufferSize: nil}, - "ftp": &FTPDownloader{userInfo: url.UserPassword("anonymous", "anonymous@"), mtu: mtu}, - "http": &HTTPDownloader{userAgent: c.UserAgent}, - "https": &HTTPDownloader{userAgent: c.UserAgent}, - "smb": &SMBDownloader{bufferSize: nil}, + "file": &FileDownloader{progress: bar, bufferSize: nil}, + "ftp": &FTPDownloader{progress: bar, userInfo: url.UserPassword("anonymous", "anonymous@"), mtu: mtu}, + "http": &HTTPDownloader{progress: bar, userAgent: c.UserAgent}, + "https": &HTTPDownloader{progress: bar, userAgent: c.UserAgent}, + "smb": &SMBDownloader{progress: bar, bufferSize: nil}, } } - return &DownloadClient{config: c} } @@ -209,14 +209,6 @@ func (d *DownloadClient) Get() (string, error) { return finalPath, err } -// PercentProgress returns the download progress as a percentage. -func (d *DownloadClient) PercentProgress() int { - if d.downloader == nil { - return -1 - } - return int((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100) -} - // VerifyChecksum tests that the path matches the checksum for the // download. func (d *DownloadClient) VerifyChecksum(path string) (bool, error) { @@ -239,9 +231,11 @@ func (d *DownloadClient) VerifyChecksum(path string) (bool, error) { // HTTPDownloader is an implementation of Downloader that downloads // files over HTTP. type HTTPDownloader struct { - progress uint64 + current uint64 total uint64 userAgent string + + progress *pb.ProgressBar } func (d *HTTPDownloader) Cancel() { @@ -261,7 +255,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { } // Reset our progress - d.progress = 0 + d.current = 0 // Make the request. We first make a HEAD request so we can check // if the server supports range queries. If the server/URL doesn't @@ -290,7 +284,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { if _, err = dst.Seek(0, os.SEEK_END); err == nil { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) - d.progress = uint64(fi.Size()) + d.current = uint64(fi.Size()) } } } @@ -304,9 +298,11 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return err } - d.total = d.progress + uint64(resp.ContentLength) - progressBar := pb.New64(int64(d.Total())).Start() - progressBar.Set64(int64(d.Progress())) + d.total = d.current + uint64(resp.ContentLength) + + d.progress.Total = int64(d.total) + progressBar := d.progress.Start() + progressBar.Set64(int64(d.current)) var buffer [4096]byte for { @@ -315,8 +311,8 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return err } - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) if _, werr := dst.Write(buffer[:n]); werr != nil { return werr @@ -326,12 +322,12 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { break } } - + progressBar.Finish() return nil } func (d *HTTPDownloader) Progress() uint64 { - return d.progress + return d.current } func (d *HTTPDownloader) Total() uint64 { @@ -344,13 +340,15 @@ type FTPDownloader struct { userInfo *url.Userinfo mtu uint - active bool - progress uint64 - total uint64 + active bool + current uint64 + total uint64 + + progress *pb.ProgressBar } func (d *FTPDownloader) Progress() uint64 { - return d.progress + return d.current } func (d *FTPDownloader) Total() uint64 { @@ -438,13 +436,15 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { } log.Printf("Found file : %s : %v bytes\n", entry.Name, entry.Size) - d.progress = 0 + d.current = 0 d.total = entry.Size - progressBar := pb.New64(int64(d.Total())).Start() + + d.progress.Total = int64(d.total) + progressBar := d.progress.Start() // download specified file d.active = true - reader, err := cli.RetrFrom(uri.Path, d.progress) + reader, err := cli.RetrFrom(uri.Path, d.current) if err != nil { return nil } @@ -458,8 +458,8 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { break } - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) } d.active = false e <- err @@ -467,10 +467,12 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { // spin until it's done err = <-errch + + progressBar.Finish() reader.Close() - if err == nil && d.progress != d.total { - err = fmt.Errorf("FTP total transfer size was %d when %d was expected", d.progress, d.total) + if err == nil && d.current != d.total { + err = fmt.Errorf("FTP total transfer size was %d when %d was expected", d.current, d.total) } // log out @@ -483,13 +485,15 @@ func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { type FileDownloader struct { bufferSize *uint - active bool - progress uint64 - total uint64 + active bool + current uint64 + total uint64 + + progress *pb.ProgressBar } func (d *FileDownloader) Progress() uint64 { - return d.progress + return d.current } func (d *FileDownloader) Total() uint64 { @@ -549,7 +553,7 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { } /* download the file using the operating system's facilities */ - d.progress = 0 + d.current = 0 d.active = true f, err := os.Open(realpath) @@ -564,7 +568,9 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { return err } d.total = uint64(fi.Size()) - progressBar := pb.New64(int64(d.Total())).Start() + + d.progress.Total = int64(d.total) + progressBar := d.progress.Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { @@ -572,8 +578,8 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { n, err = io.Copy(dst, f) d.active = false - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) // use a goro in case someone else wants to enable cancel/resume } else { @@ -585,8 +591,8 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { break } - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) } d.active = false e <- err @@ -595,6 +601,7 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { // ...and we spin until it's done err = <-errch } + progressBar.Finish() f.Close() return err } @@ -604,13 +611,15 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { type SMBDownloader struct { bufferSize *uint - active bool - progress uint64 - total uint64 + active bool + current uint64 + total uint64 + + progress *pb.ProgressBar } func (d *SMBDownloader) Progress() uint64 { - return d.progress + return d.current } func (d *SMBDownloader) Total() uint64 { @@ -663,7 +672,7 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { } /* Open up the "\\"-prefixed path using the Windows filesystem */ - d.progress = 0 + d.current = 0 d.active = true f, err := os.Open(realpath) @@ -678,7 +687,9 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { return err } d.total = uint64(fi.Size()) - progressBar := pb.New64(int64(d.Total())).Start() + + d.progress.Total = int64(d.total) + progressBar := d.progress.Start() // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { @@ -686,8 +697,8 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { n, err = io.Copy(dst, f) d.active = false - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) // use a goro in case someone else wants to enable cancel/resume } else { @@ -699,8 +710,8 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { break } - d.progress += uint64(n) - progressBar.Set64(int64(d.Progress())) + d.current += uint64(n) + progressBar.Set64(int64(d.current)) } d.active = false e <- err @@ -709,6 +720,7 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { // ...and as usual we spin until it's done err = <-errch } + progressBar.Finish() f.Close() return err } diff --git a/common/step_download.go b/common/step_download.go index 765a07d23a2..2ce48cfff2c 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -9,6 +9,8 @@ import ( "github.com/hashicorp/packer/packer" "github.com/mitchellh/multistep" + + "gopkg.in/cheggaaa/pb.v1" ) // StepDownload downloads a remote file using the download client within @@ -139,7 +141,23 @@ func (s *StepDownload) Cleanup(multistep.StateBag) {} func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag) (string, error, bool) { var path string ui := state.Get("ui").(packer.Ui) - download := NewDownloadClient(config) + + // design the appearance of the progress bar + bar := pb.New64(0) + bar.ShowPercent = true + bar.ShowCounters = true + bar.ShowSpeed = false + bar.ShowBar = true + bar.ShowTimeLeft = false + bar.ShowFinalTime = false + bar.SetUnits(pb.U_BYTES) + bar.Format("[=>-]") + bar.SetRefreshRate(1 * time.Second) + bar.SetWidth(25) + bar.Callback = ui.Message + + // create download client with config and progress bar + download := NewDownloadClient(config, bar) downloadCompleteCh := make(chan error, 1) go func() { @@ -148,24 +166,19 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag downloadCompleteCh <- err }() - progressTicker := time.NewTicker(5 * time.Second) - defer progressTicker.Stop() - for { select { case err := <-downloadCompleteCh: + bar.Finish() + if err != nil { return "", err, true } - return path, nil, true - case <-progressTicker.C: - progress := download.PercentProgress() - if progress >= 0 { - ui.Message(fmt.Sprintf("Download progress: %d%%", progress)) - } + case <-time.After(1 * time.Second): if _, ok := state.GetOk(multistep.StateCancelled); ok { + bar.Finish() ui.Say("Interrupt received. Cancelling download...") return "", nil, false } From 058b39904f0fcffc426e863db73c0d5d1609ac18 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 7 Dec 2017 11:57:40 -0600 Subject: [PATCH 18/18] Added second argument for custom-formatted progress-bar to NewDownloadClient in common/download_test.go. This second parameter was added as a result of commit f0bd9018f3e318caafb1fe7d46e04c470e07c092 which lets you customize the progress-bar format. --- common/download.go | 7 ++++++- common/download_test.go | 20 ++++++++++---------- common/step_download.go | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/common/download.go b/common/download.go index be53dfed4fe..4826d1dd1ab 100644 --- a/common/download.go +++ b/common/download.go @@ -86,8 +86,13 @@ func HashForType(t string) hash.Hash { func NewDownloadClient(c *DownloadConfig, bar *pb.ProgressBar) *DownloadClient { const mtu = 1500 /* ethernet */ - 20 /* ipv4 */ - 20 /* tcp */ + // If no custom progress-bar was specified, then create a default one. + if bar == nil { + bar = pb.New64(0) + } + + // Create downloader map if it hasn't been specified already. if c.DownloaderMap == nil { - // Create downloader map c.DownloaderMap = map[string]Downloader{ "file": &FileDownloader{progress: bar, bufferSize: nil}, "ftp": &FTPDownloader{progress: bar, userInfo: url.UserPassword("anonymous", "anonymous@"), mtu: mtu}, diff --git a/common/download_test.go b/common/download_test.go index a62fceb963e..50bdfa971d8 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -36,7 +36,7 @@ func TestDownloadClientVerifyChecksum(t *testing.T) { Checksum: checksum, } - d := NewDownloadClient(config) + d := NewDownloadClient(config, nil) result, err := d.VerifyChecksum(tf.Name()) if err != nil { t.Fatalf("Verify err: %s", err) @@ -59,7 +59,7 @@ func TestDownloadClient_basic(t *testing.T) { Url: ts.URL + "/basic.txt", TargetPath: tf.Name(), CopyFile: true, - }) + }, nil) path, err := client.Get() if err != nil { @@ -95,7 +95,7 @@ func TestDownloadClient_checksumBad(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }) + }, nil) if _, err := client.Get(); err == nil { t.Fatal("should error") } @@ -120,7 +120,7 @@ func TestDownloadClient_checksumGood(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }) + }, nil) path, err := client.Get() if err != nil { t.Fatalf("err: %s", err) @@ -151,7 +151,7 @@ func TestDownloadClient_checksumNoDownload(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }) + }, nil) path, err := client.Get() if err != nil { t.Fatalf("err: %s", err) @@ -190,7 +190,7 @@ func TestDownloadClient_resume(t *testing.T) { Url: ts.URL, TargetPath: tf.Name(), CopyFile: true, - }) + }, nil) path, err := client.Get() if err != nil { t.Fatalf("err: %s", err) @@ -250,7 +250,7 @@ func TestDownloadClient_usesDefaultUserAgent(t *testing.T) { CopyFile: true, } - client := NewDownloadClient(config) + client := NewDownloadClient(config, nil) _, err = client.Get() if err != nil { t.Fatal(err) @@ -282,7 +282,7 @@ func TestDownloadClient_setsUserAgent(t *testing.T) { CopyFile: true, } - client := NewDownloadClient(config) + client := NewDownloadClient(config, nil) _, err = client.Get() if err != nil { t.Fatal(err) @@ -381,7 +381,7 @@ func TestDownloadFileUrl(t *testing.T) { CopyFile: false, } - client := NewDownloadClient(config) + client := NewDownloadClient(config, nil) // Verify that we fail to match the checksum _, err = client.Get() @@ -412,7 +412,7 @@ func SimulateFileUriDownload(t *testing.T, uri string) (string, error) { } // go go go - client := NewDownloadClient(config) + client := NewDownloadClient(config, nil) path, err := client.Get() // ignore any non-important checksum errors if it's not a unc path diff --git a/common/step_download.go b/common/step_download.go index 2ce48cfff2c..a4f24929060 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -96,7 +96,7 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { } downloadConfigs[i] = config - if match, _ := NewDownloadClient(config).VerifyChecksum(config.TargetPath); match { + if match, _ := NewDownloadClient(config, nil).VerifyChecksum(config.TargetPath); match { ui.Message(fmt.Sprintf("Found already downloaded, initial checksum matched, no download needed: %s", url)) finalPath = config.TargetPath break