diff --git a/common/download.go b/common/download.go index 6d3d2a685b6..abee0eea53d 100644 --- a/common/download.go +++ b/common/download.go @@ -21,8 +21,10 @@ import ( // imports related to each Downloader implementation import ( "io" + "net" "net/http" "path/filepath" + "github.com/jlaffaye/ftp" ) // DownloadConfig is the configuration given to instantiate a new @@ -91,6 +93,7 @@ func NewDownloadClient(c *DownloadConfig) *DownloadClient { "http": &HTTPDownloader{userAgent: c.UserAgent}, "https": &HTTPDownloader{userAgent: c.UserAgent}, "smb": &SMBDownloader{bufferSize: nil}, + "ftp": &FTPDownloader{userInfo: url.UserPassword("anonymous", "anonymous@"), mtu: mtu}, } } return &DownloadClient{config: c} @@ -440,7 +443,7 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { } else { errch := make(chan error) go func(d *FileDownloader, r io.Reader, w io.Writer, e chan error) { - for d.active { + for d.active && d.current < d.total { n, err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break @@ -551,7 +554,7 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { } else { errch := make(chan error) go func(d *SMBDownloader, r io.Reader, w io.Writer, e chan error) { - for d.active { + for d.active && d.current < d.total { n, err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break @@ -569,3 +572,160 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { f.Close() return err } + +// FTPDownloader is an implementation of Downloader that downloads +// files over FTP. +type FTPDownloader struct { + userInfo *url.Userinfo + mtu uint + + active bool + current uint64 + total uint64 +} + +func (d *FTPDownloader) Progress() uint64 { + return d.current +} + +func (d *FTPDownloader) Total() uint64 { + return d.total +} + +func (d *FTPDownloader) Cancel() { + d.active = false +} + +func (d *FTPDownloader) Resume() { + // TODO: Implement +} + +func (d *FTPDownloader) Download(dst *os.File, src *url.URL) error { + var userinfo *url.Userinfo + + userinfo = d.userInfo + d.active = false + + // check the uri is correct + if src == nil || strings.ToLower(src.Scheme) != "ftp" { + return fmt.Errorf("Unexpected uri scheme: %s", src.Scheme) + } + uri := src + + // add the default ftp port + if uri.Port() == "" { + port, err := net.LookupPort("ip4", "ftp") + if err != nil { + port = 21 + } + uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port) + } + + // connect to ftp server + var cli *ftp.ServerConn + + log.Printf("Starting download over FTP: %s(:%s) -> %s\n", uri.Hostname(), uri.Port(), uri.Path) + cli, err := ftp.Dial(fmt.Sprintf("%s:%s", uri.Hostname(), uri.Port())) + if err != nil { + return err + } + defer cli.Quit() + + // handle authentication + if uri.User != nil { + userinfo = uri.User + } + + pass, ok := userinfo.Password() + if !ok { + pass = "ftp@" + } + + log.Printf("Authenticating to FTP server: %s : %s\n", userinfo.Username(), pass) + err = cli.Login(userinfo.Username(), pass) + if err != nil { + return err + } + + // locate specified path + p := path.Dir(uri.Path) + + log.Printf("Changing to FTP directory : %s\n", p) + err = cli.ChangeDir(p) + if err != nil { + return err + } + + curpath, err := cli.CurrentDir() + if err != nil { + return err + } + log.Printf("Current FTP directory : %s\n", curpath) + + // collect stats about the specified file + var name string + var entry *ftp.Entry + + _, name = path.Split(uri.Path) + entry = nil + + log.Printf("Enumerating files in current directory : %s\n", curpath) + entries, err := cli.List(".") + if err != nil { + return fmt.Errorf("Unable to list files in directory \"%s\". (%s)", curpath, err) + } + if len(entries) == 0 { + return fmt.Errorf("Unable to find any files in directory \"%s\".", curpath) + } + + for _, e := range entries { + log.Printf("Checking file name: %v\n", e) + if e.Type == ftp.EntryTypeFile && e.Name == name { + log.Printf("Found matching name: %v\n", e) + entry = e + break + } + } + + if entry == nil { + return fmt.Errorf("Unable to find file \"%s\".", uri.Path) + } + log.Printf("Found file: %s (%d bytes).\n", entry.Name, entry.Size) + + d.current = 0 + d.total = entry.Size + + // download specified file + d.active = true + reader, err := cli.RetrFrom(uri.Path, d.current) + if err != nil { + return err + } + + // do it in a goro so that if someone wants to cancel it, they can + errch := make(chan error) + go func(d *FTPDownloader, r io.Reader, w io.Writer, e chan error) { + for d.active && d.current < d.total { + n, err := io.CopyN(w, r, int64(d.mtu)) + if err != nil { + break + } + d.current += uint64(n) + } + d.active = false + e <- err + }(d, reader, dst, errch) + + // spin until it's done + err = <-errch + + reader.Close() + + if err == nil && d.current != d.total { + err = fmt.Errorf("FTP total transfer size was %d when %d was expected.", d.current, d.total) + } + + // log out + cli.Logout() + return err +} diff --git a/vendor/github.com/jlaffaye/ftp/LICENSE b/vendor/github.com/jlaffaye/ftp/LICENSE new file mode 100644 index 00000000000..9ab085c5146 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2011-2013, Julien Laffaye + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/jlaffaye/ftp/README.md b/vendor/github.com/jlaffaye/ftp/README.md new file mode 100644 index 00000000000..b711be7ad36 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/README.md @@ -0,0 +1,17 @@ +# goftp # + +[![Build Status](https://travis-ci.org/jlaffaye/ftp.svg?branch=master)](https://travis-ci.org/jlaffaye/ftp) +[![Coverage Status](https://coveralls.io/repos/jlaffaye/ftp/badge.svg?branch=master&service=github)](https://coveralls.io/github/jlaffaye/ftp?branch=master) +[![Go ReportCard](http://goreportcard.com/badge/jlaffaye/ftp)](http://goreportcard.com/report/jlaffaye/ftp) + +A FTP client package for Go + +## Install ## + +``` +go get -u github.com/jlaffaye/ftp +``` + +## Documentation ## + +http://godoc.org/github.com/jlaffaye/ftp diff --git a/vendor/github.com/jlaffaye/ftp/ftp.go b/vendor/github.com/jlaffaye/ftp/ftp.go new file mode 100644 index 00000000000..bf7f9ecd415 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/ftp.go @@ -0,0 +1,587 @@ +// Package ftp implements a FTP client as described in RFC 959. +// +// A textproto.Error is returned for errors at the protocol level. +package ftp + +import ( + "bufio" + "errors" + "io" + "net" + "net/textproto" + "strconv" + "strings" + "time" +) + +// EntryType describes the different types of an Entry. +type EntryType int + +// The differents types of an Entry +const ( + EntryTypeFile EntryType = iota + EntryTypeFolder + EntryTypeLink +) + +// ServerConn represents the connection to a remote FTP server. +// It should be protected from concurrent accesses. +type ServerConn struct { + // Do not use EPSV mode + DisableEPSV bool + + conn *textproto.Conn + host string + timeout time.Duration + features map[string]string + mlstSupported bool +} + +// Entry describes a file and is returned by List(). +type Entry struct { + Name string + Type EntryType + Size uint64 + Time time.Time +} + +// Response represents a data-connection +type Response struct { + conn net.Conn + c *ServerConn + closed bool +} + +// Connect is an alias to Dial, for backward compatibility +func Connect(addr string) (*ServerConn, error) { + return Dial(addr) +} + +// Dial is like DialTimeout with no timeout +func Dial(addr string) (*ServerConn, error) { + return DialTimeout(addr, 0) +} + +// DialTimeout initializes the connection to the specified ftp server address. +// +// It is generally followed by a call to Login() as most FTP commands require +// an authenticated user. +func DialTimeout(addr string, timeout time.Duration) (*ServerConn, error) { + tconn, err := net.DialTimeout("tcp", addr, timeout) + if err != nil { + return nil, err + } + + // Use the resolved IP address in case addr contains a domain name + // If we use the domain name, we might not resolve to the same IP. + remoteAddr := tconn.RemoteAddr().(*net.TCPAddr) + + conn := textproto.NewConn(tconn) + + c := &ServerConn{ + conn: conn, + host: remoteAddr.IP.String(), + timeout: timeout, + features: make(map[string]string), + } + + _, _, err = c.conn.ReadResponse(StatusReady) + if err != nil { + c.Quit() + return nil, err + } + + err = c.feat() + if err != nil { + c.Quit() + return nil, err + } + + if _, mlstSupported := c.features["MLST"]; mlstSupported { + c.mlstSupported = true + } + + return c, nil +} + +// Login authenticates the client with specified user and password. +// +// "anonymous"/"anonymous" is a common user/password scheme for FTP servers +// that allows anonymous read-only accounts. +func (c *ServerConn) Login(user, password string) error { + code, message, err := c.cmd(-1, "USER %s", user) + if err != nil { + return err + } + + switch code { + case StatusLoggedIn: + case StatusUserOK: + _, _, err = c.cmd(StatusLoggedIn, "PASS %s", password) + if err != nil { + return err + } + default: + return errors.New(message) + } + + // Switch to binary mode + if _, _, err = c.cmd(StatusCommandOK, "TYPE I"); err != nil { + return err + } + + // Switch to UTF-8 + err = c.setUTF8() + + return err +} + +// feat issues a FEAT FTP command to list the additional commands supported by +// the remote FTP server. +// FEAT is described in RFC 2389 +func (c *ServerConn) feat() error { + code, message, err := c.cmd(-1, "FEAT") + if err != nil { + return err + } + + if code != StatusSystem { + // The server does not support the FEAT command. This is not an + // error: we consider that there is no additional feature. + return nil + } + + lines := strings.Split(message, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, " ") { + continue + } + + line = strings.TrimSpace(line) + featureElements := strings.SplitN(line, " ", 2) + + command := featureElements[0] + + var commandDesc string + if len(featureElements) == 2 { + commandDesc = featureElements[1] + } + + c.features[command] = commandDesc + } + + return nil +} + +// setUTF8 issues an "OPTS UTF8 ON" command. +func (c *ServerConn) setUTF8() error { + if _, ok := c.features["UTF8"]; !ok { + return nil + } + + code, message, err := c.cmd(-1, "OPTS UTF8 ON") + if err != nil { + return err + } + + // The ftpd "filezilla-server" has FEAT support for UTF8, but always returns + // "202 UTF8 mode is always enabled. No need to send this command." when + // trying to use it. That's OK + if code == StatusCommandNotImplemented { + return nil + } + + if code != StatusCommandOK { + return errors.New(message) + } + + return nil +} + +// epsv issues an "EPSV" command to get a port number for a data connection. +func (c *ServerConn) epsv() (port int, err error) { + _, line, err := c.cmd(StatusExtendedPassiveMode, "EPSV") + if err != nil { + return + } + + start := strings.Index(line, "|||") + end := strings.LastIndex(line, "|") + if start == -1 || end == -1 { + err = errors.New("Invalid EPSV response format") + return + } + port, err = strconv.Atoi(line[start+3 : end]) + return +} + +// pasv issues a "PASV" command to get a port number for a data connection. +func (c *ServerConn) pasv() (port int, err error) { + _, line, err := c.cmd(StatusPassiveMode, "PASV") + if err != nil { + return + } + + // PASV response format : 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). + start := strings.Index(line, "(") + end := strings.LastIndex(line, ")") + if start == -1 || end == -1 { + return 0, errors.New("Invalid PASV response format") + } + + // We have to split the response string + pasvData := strings.Split(line[start+1:end], ",") + + if len(pasvData) < 6 { + return 0, errors.New("Invalid PASV response format") + } + + // Let's compute the port number + portPart1, err1 := strconv.Atoi(pasvData[4]) + if err1 != nil { + err = err1 + return + } + + portPart2, err2 := strconv.Atoi(pasvData[5]) + if err2 != nil { + err = err2 + return + } + + // Recompose port + port = portPart1*256 + portPart2 + return +} + +// getDataConnPort returns a port for a new data connection +// it uses the best available method to do so +func (c *ServerConn) getDataConnPort() (int, error) { + if !c.DisableEPSV { + if port, err := c.epsv(); err == nil { + return port, nil + } + + // if there is an error, disable EPSV for the next attempts + c.DisableEPSV = true + } + + return c.pasv() +} + +// openDataConn creates a new FTP data connection. +func (c *ServerConn) openDataConn() (net.Conn, error) { + port, err := c.getDataConnPort() + if err != nil { + return nil, err + } + + return net.DialTimeout("tcp", net.JoinHostPort(c.host, strconv.Itoa(port)), c.timeout) +} + +// cmd is a helper function to execute a command and check for the expected FTP +// return code +func (c *ServerConn) cmd(expected int, format string, args ...interface{}) (int, string, error) { + _, err := c.conn.Cmd(format, args...) + if err != nil { + return 0, "", err + } + + return c.conn.ReadResponse(expected) +} + +// cmdDataConnFrom executes a command which require a FTP data connection. +// Issues a REST FTP command to specify the number of bytes to skip for the transfer. +func (c *ServerConn) cmdDataConnFrom(offset uint64, format string, args ...interface{}) (net.Conn, error) { + conn, err := c.openDataConn() + if err != nil { + return nil, err + } + + if offset != 0 { + _, _, err := c.cmd(StatusRequestFilePending, "REST %d", offset) + if err != nil { + conn.Close() + return nil, err + } + } + + _, err = c.conn.Cmd(format, args...) + if err != nil { + conn.Close() + return nil, err + } + + code, msg, err := c.conn.ReadResponse(-1) + if err != nil { + conn.Close() + return nil, err + } + if code != StatusAlreadyOpen && code != StatusAboutToSend { + conn.Close() + return nil, &textproto.Error{Code: code, Msg: msg} + } + + return conn, nil +} + +// NameList issues an NLST FTP command. +func (c *ServerConn) NameList(path string) (entries []string, err error) { + conn, err := c.cmdDataConnFrom(0, "NLST %s", path) + if err != nil { + return + } + + r := &Response{conn: conn, c: c} + defer r.Close() + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + entries = append(entries, scanner.Text()) + } + if err = scanner.Err(); err != nil { + return entries, err + } + return +} + +// List issues a LIST FTP command. +func (c *ServerConn) List(path string) (entries []*Entry, err error) { + var cmd string + var parser parseFunc + + if c.mlstSupported { + cmd = "MLSD" + parser = parseRFC3659ListLine + } else { + cmd = "LIST" + parser = parseListLine + } + + conn, err := c.cmdDataConnFrom(0, "%s %s", cmd, path) + if err != nil { + return + } + + r := &Response{conn: conn, c: c} + defer r.Close() + + scanner := bufio.NewScanner(r) + now := time.Now() + for scanner.Scan() { + entry, err := parser(scanner.Text(), now) + if err == nil { + entries = append(entries, entry) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return +} + +// ChangeDir issues a CWD FTP command, which changes the current directory to +// the specified path. +func (c *ServerConn) ChangeDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CWD %s", path) + return err +} + +// ChangeDirToParent issues a CDUP FTP command, which changes the current +// directory to the parent directory. This is similar to a call to ChangeDir +// with a path set to "..". +func (c *ServerConn) ChangeDirToParent() error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CDUP") + return err +} + +// CurrentDir issues a PWD FTP command, which Returns the path of the current +// directory. +func (c *ServerConn) CurrentDir() (string, error) { + _, msg, err := c.cmd(StatusPathCreated, "PWD") + if err != nil { + return "", err + } + + start := strings.Index(msg, "\"") + end := strings.LastIndex(msg, "\"") + + if start == -1 || end == -1 { + return "", errors.New("Unsuported PWD response format") + } + + return msg[start+1 : end], nil +} + +// FileSize issues a SIZE FTP command, which Returns the size of the file +func (c *ServerConn) FileSize(path string) (int64, error) { + _, msg, err := c.cmd(StatusFile, "SIZE %s", path) + if err != nil { + return 0, err + } + + return strconv.ParseInt(msg, 10, 64) +} + +// Retr issues a RETR FTP command to fetch the specified file from the remote +// FTP server. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) Retr(path string) (*Response, error) { + return c.RetrFrom(path, 0) +} + +// RetrFrom issues a RETR FTP command to fetch the specified file from the remote +// FTP server, the server will not send the offset first bytes of the file. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) RetrFrom(path string, offset uint64) (*Response, error) { + conn, err := c.cmdDataConnFrom(offset, "RETR %s", path) + if err != nil { + return nil, err + } + + return &Response{conn: conn, c: c}, nil +} + +// Stor issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) Stor(path string, r io.Reader) error { + return c.StorFrom(path, r, 0) +} + +// StorFrom issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader, writing +// on the server will start at the given file offset. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) StorFrom(path string, r io.Reader, offset uint64) error { + conn, err := c.cmdDataConnFrom(offset, "STOR %s", path) + if err != nil { + return err + } + + _, err = io.Copy(conn, r) + conn.Close() + if err != nil { + return err + } + + _, _, err = c.conn.ReadResponse(StatusClosingDataConnection) + return err +} + +// Rename renames a file on the remote FTP server. +func (c *ServerConn) Rename(from, to string) error { + _, _, err := c.cmd(StatusRequestFilePending, "RNFR %s", from) + if err != nil { + return err + } + + _, _, err = c.cmd(StatusRequestedFileActionOK, "RNTO %s", to) + return err +} + +// Delete issues a DELE FTP command to delete the specified file from the +// remote FTP server. +func (c *ServerConn) Delete(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "DELE %s", path) + return err +} + +// RemoveDirRecur deletes a non-empty folder recursively using +// RemoveDir and Delete +func (c *ServerConn) RemoveDirRecur(path string) error { + err := c.ChangeDir(path) + if err != nil { + return err + } + currentDir, err := c.CurrentDir() + if err != nil { + return err + } + entries, err := c.List(currentDir) + for _, entry := range entries { + if entry.Name != ".." && entry.Name != "." { + if entry.Type == EntryTypeFolder { + err = c.RemoveDirRecur(currentDir + "/" + entry.Name) + if err != nil { + return err + } + } else { + err = c.Delete(entry.Name) + if err != nil { + return err + } + } + } + } + err = c.ChangeDirToParent() + if err != nil { + return err + } + err = c.RemoveDir(currentDir) + return err +} + +// MakeDir issues a MKD FTP command to create the specified directory on the +// remote FTP server. +func (c *ServerConn) MakeDir(path string) error { + _, _, err := c.cmd(StatusPathCreated, "MKD %s", path) + return err +} + +// RemoveDir issues a RMD FTP command to remove the specified directory from +// the remote FTP server. +func (c *ServerConn) RemoveDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "RMD %s", path) + return err +} + +// NoOp issues a NOOP FTP command. +// NOOP has no effects and is usually used to prevent the remote FTP server to +// close the otherwise idle connection. +func (c *ServerConn) NoOp() error { + _, _, err := c.cmd(StatusCommandOK, "NOOP") + return err +} + +// Logout issues a REIN FTP command to logout the current user. +func (c *ServerConn) Logout() error { + _, _, err := c.cmd(StatusReady, "REIN") + return err +} + +// Quit issues a QUIT FTP command to properly close the connection from the +// remote FTP server. +func (c *ServerConn) Quit() error { + c.conn.Cmd("QUIT") + return c.conn.Close() +} + +// Read implements the io.Reader interface on a FTP data connection. +func (r *Response) Read(buf []byte) (int, error) { + return r.conn.Read(buf) +} + +// Close implements the io.Closer interface on a FTP data connection. +// After the first call, Close will do nothing and return nil. +func (r *Response) Close() error { + if r.closed { + return nil + } + err := r.conn.Close() + _, _, err2 := r.c.conn.ReadResponse(StatusClosingDataConnection) + if err2 != nil { + err = err2 + } + r.closed = true + return err +} + +// SetDeadline sets the deadlines associated with the connection. +func (r *Response) SetDeadline(t time.Time) error { + return r.conn.SetDeadline(t) +} diff --git a/vendor/github.com/jlaffaye/ftp/parse.go b/vendor/github.com/jlaffaye/ftp/parse.go new file mode 100644 index 00000000000..db63378255f --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/parse.go @@ -0,0 +1,255 @@ +package ftp + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +var errUnsupportedListLine = errors.New("Unsupported LIST line") + +type parseFunc func(string, time.Time) (*Entry, error) + +var listLineParsers = []parseFunc{ + parseRFC3659ListLine, + parseLsListLine, + parseDirListLine, + parseHostedFTPLine, +} + +var dirTimeFormats = []string{ + "01-02-06 03:04PM", + "2006-01-02 15:04", +} + +// parseRFC3659ListLine parses the style of directory line defined in RFC 3659. +func parseRFC3659ListLine(line string, now time.Time) (*Entry, error) { + iSemicolon := strings.Index(line, ";") + iWhitespace := strings.Index(line, " ") + + if iSemicolon < 0 || iSemicolon > iWhitespace { + return nil, errUnsupportedListLine + } + + e := &Entry{ + Name: line[iWhitespace+1:], + } + + for _, field := range strings.Split(line[:iWhitespace-1], ";") { + i := strings.Index(field, "=") + if i < 1 { + return nil, errUnsupportedListLine + } + + key := strings.ToLower(field[:i]) + value := field[i+1:] + + switch key { + case "modify": + var err error + e.Time, err = time.Parse("20060102150405", value) + if err != nil { + return nil, err + } + case "type": + switch value { + case "dir", "cdir", "pdir": + e.Type = EntryTypeFolder + case "file": + e.Type = EntryTypeFile + } + case "size": + e.setSize(value) + } + } + return e, nil +} + +// parseLsListLine parses a directory line in a format based on the output of +// the UNIX ls command. +func parseLsListLine(line string, now time.Time) (*Entry, error) { + + // Has the first field a length of 10 bytes? + if strings.IndexByte(line, ' ') != 10 { + return nil, errUnsupportedListLine + } + + scanner := newScanner(line) + fields := scanner.NextFields(6) + + if len(fields) < 6 { + return nil, errUnsupportedListLine + } + + if fields[1] == "folder" && fields[2] == "0" { + e := &Entry{ + Type: EntryTypeFolder, + Name: scanner.Remaining(), + } + if err := e.setTime(fields[3:6], now); err != nil { + return nil, err + } + + return e, nil + } + + if fields[1] == "0" { + fields = append(fields, scanner.Next()) + e := &Entry{ + Type: EntryTypeFile, + Name: scanner.Remaining(), + } + + if err := e.setSize(fields[2]); err != nil { + return nil, errUnsupportedListLine + } + if err := e.setTime(fields[4:7], now); err != nil { + return nil, err + } + + return e, nil + } + + // Read two more fields + fields = append(fields, scanner.NextFields(2)...) + if len(fields) < 8 { + return nil, errUnsupportedListLine + } + + e := &Entry{ + Name: scanner.Remaining(), + } + switch fields[0][0] { + case '-': + e.Type = EntryTypeFile + if err := e.setSize(fields[4]); err != nil { + return nil, err + } + case 'd': + e.Type = EntryTypeFolder + case 'l': + e.Type = EntryTypeLink + default: + return nil, errors.New("Unknown entry type") + } + + if err := e.setTime(fields[5:8], now); err != nil { + return nil, err + } + + return e, nil +} + +// parseDirListLine parses a directory line in a format based on the output of +// the MS-DOS DIR command. +func parseDirListLine(line string, now time.Time) (*Entry, error) { + e := &Entry{} + var err error + + // Try various time formats that DIR might use, and stop when one works. + for _, format := range dirTimeFormats { + if len(line) > len(format) { + e.Time, err = time.Parse(format, line[:len(format)]) + if err == nil { + line = line[len(format):] + break + } + } + } + if err != nil { + // None of the time formats worked. + return nil, errUnsupportedListLine + } + + line = strings.TrimLeft(line, " ") + if strings.HasPrefix(line, "") { + e.Type = EntryTypeFolder + line = strings.TrimPrefix(line, "") + } else { + space := strings.Index(line, " ") + if space == -1 { + return nil, errUnsupportedListLine + } + e.Size, err = strconv.ParseUint(line[:space], 10, 64) + if err != nil { + return nil, errUnsupportedListLine + } + e.Type = EntryTypeFile + line = line[space:] + } + + e.Name = strings.TrimLeft(line, " ") + return e, nil +} + +// parseHostedFTPLine parses a directory line in the non-standard format used +// by hostedftp.com +// -r-------- 0 user group 65222236 Feb 24 00:39 UABlacklistingWeek8.csv +// (The link count is inexplicably 0) +func parseHostedFTPLine(line string, now time.Time) (*Entry, error) { + // Has the first field a length of 10 bytes? + if strings.IndexByte(line, ' ') != 10 { + return nil, errUnsupportedListLine + } + + scanner := newScanner(line) + fields := scanner.NextFields(2) + + if len(fields) < 2 || fields[1] != "0" { + return nil, errUnsupportedListLine + } + + // Set link count to 1 and attempt to parse as Unix. + return parseLsListLine(fields[0]+" 1 "+scanner.Remaining(), now) +} + +// parseListLine parses the various non-standard format returned by the LIST +// FTP command. +func parseListLine(line string, now time.Time) (*Entry, error) { + for _, f := range listLineParsers { + e, err := f(line, now) + if err != errUnsupportedListLine { + return e, err + } + } + return nil, errUnsupportedListLine +} + +func (e *Entry) setSize(str string) (err error) { + e.Size, err = strconv.ParseUint(str, 0, 64) + return +} + +func (e *Entry) setTime(fields []string, now time.Time) (err error) { + if strings.Contains(fields[2], ":") { // contains time + thisYear, _, _ := now.Date() + timeStr := fmt.Sprintf("%s %s %d %s GMT", fields[1], fields[0], thisYear, fields[2]) + e.Time, err = time.Parse("_2 Jan 2006 15:04 MST", timeStr) + + /* + On unix, `info ls` shows: + + 10.1.6 Formatting file timestamps + --------------------------------- + + A timestamp is considered to be “recent” if it is less than six + months old, and is not dated in the future. If a timestamp dated today + is not listed in recent form, the timestamp is in the future, which + means you probably have clock skew problems which may break programs + like ‘make’ that rely on file timestamps. + */ + if !e.Time.Before(now.AddDate(0, 6, 0)) { + e.Time = e.Time.AddDate(-1, 0, 0) + } + + } else { // only the date + if len(fields[2]) != 4 { + return errors.New("Invalid year format in time string") + } + timeStr := fmt.Sprintf("%s %s %s 00:00 GMT", fields[1], fields[0], fields[2]) + e.Time, err = time.Parse("_2 Jan 2006 15:04 MST", timeStr) + } + return +} diff --git a/vendor/github.com/jlaffaye/ftp/scanner.go b/vendor/github.com/jlaffaye/ftp/scanner.go new file mode 100644 index 00000000000..0dcc8ae6474 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/scanner.go @@ -0,0 +1,58 @@ +package ftp + +// A scanner for fields delimited by one or more whitespace characters +type scanner struct { + bytes []byte + position int +} + +// newScanner creates a new scanner +func newScanner(str string) *scanner { + return &scanner{ + bytes: []byte(str), + } +} + +// NextFields returns the next `count` fields +func (s *scanner) NextFields(count int) []string { + fields := make([]string, 0, count) + for i := 0; i < count; i++ { + if field := s.Next(); field != "" { + fields = append(fields, field) + } else { + break + } + } + return fields +} + +// Next returns the next field +func (s *scanner) Next() string { + sLen := len(s.bytes) + + // skip trailing whitespace + for s.position < sLen { + if s.bytes[s.position] != ' ' { + break + } + s.position++ + } + + start := s.position + + // skip non-whitespace + for s.position < sLen { + if s.bytes[s.position] == ' ' { + s.position++ + return string(s.bytes[start : s.position-1]) + } + s.position++ + } + + return string(s.bytes[start:s.position]) +} + +// Remaining returns the remaining string +func (s *scanner) Remaining() string { + return string(s.bytes[s.position:len(s.bytes)]) +} diff --git a/vendor/github.com/jlaffaye/ftp/status.go b/vendor/github.com/jlaffaye/ftp/status.go new file mode 100644 index 00000000000..e90ca621122 --- /dev/null +++ b/vendor/github.com/jlaffaye/ftp/status.go @@ -0,0 +1,106 @@ +package ftp + +// FTP status codes, defined in RFC 959 +const ( + StatusInitiating = 100 + StatusRestartMarker = 110 + StatusReadyMinute = 120 + StatusAlreadyOpen = 125 + StatusAboutToSend = 150 + + StatusCommandOK = 200 + StatusCommandNotImplemented = 202 + StatusSystem = 211 + StatusDirectory = 212 + StatusFile = 213 + StatusHelp = 214 + StatusName = 215 + StatusReady = 220 + StatusClosing = 221 + StatusDataConnectionOpen = 225 + StatusClosingDataConnection = 226 + StatusPassiveMode = 227 + StatusLongPassiveMode = 228 + StatusExtendedPassiveMode = 229 + StatusLoggedIn = 230 + StatusLoggedOut = 231 + StatusLogoutAck = 232 + StatusRequestedFileActionOK = 250 + StatusPathCreated = 257 + + StatusUserOK = 331 + StatusLoginNeedAccount = 332 + StatusRequestFilePending = 350 + + StatusNotAvailable = 421 + StatusCanNotOpenDataConnection = 425 + StatusTransfertAborted = 426 + StatusInvalidCredentials = 430 + StatusHostUnavailable = 434 + StatusFileActionIgnored = 450 + StatusActionAborted = 451 + Status452 = 452 + + StatusBadCommand = 500 + StatusBadArguments = 501 + StatusNotImplemented = 502 + StatusBadSequence = 503 + StatusNotImplementedParameter = 504 + StatusNotLoggedIn = 530 + StatusStorNeedAccount = 532 + StatusFileUnavailable = 550 + StatusPageTypeUnknown = 551 + StatusExceededStorage = 552 + StatusBadFileName = 553 +) + +var statusText = map[int]string{ + // 200 + StatusCommandOK: "Command okay.", + StatusCommandNotImplemented: "Command not implemented, superfluous at this site.", + StatusSystem: "System status, or system help reply.", + StatusDirectory: "Directory status.", + StatusFile: "File status.", + StatusHelp: "Help message.", + StatusName: "", + StatusReady: "Service ready for new user.", + StatusClosing: "Service closing control connection.", + StatusDataConnectionOpen: "Data connection open; no transfer in progress.", + StatusClosingDataConnection: "Closing data connection. Requested file action successful.", + StatusPassiveMode: "Entering Passive Mode.", + StatusLongPassiveMode: "Entering Long Passive Mode.", + StatusExtendedPassiveMode: "Entering Extended Passive Mode.", + StatusLoggedIn: "User logged in, proceed.", + StatusLoggedOut: "User logged out; service terminated.", + StatusLogoutAck: "Logout command noted, will complete when transfer done.", + StatusRequestedFileActionOK: "Requested file action okay, completed.", + StatusPathCreated: "Path created.", + + // 300 + StatusUserOK: "User name okay, need password.", + StatusLoginNeedAccount: "Need account for login.", + StatusRequestFilePending: "Requested file action pending further information.", + + // 400 + StatusNotAvailable: "Service not available, closing control connection.", + StatusCanNotOpenDataConnection: "Can't open data connection.", + StatusTransfertAborted: "Connection closed; transfer aborted.", + StatusInvalidCredentials: "Invalid username or password.", + StatusHostUnavailable: "Requested host unavailable.", + StatusFileActionIgnored: "Requested file action not taken.", + StatusActionAborted: "Requested action aborted. Local error in processing.", + Status452: "Insufficient storage space in system.", + + // 500 + StatusBadCommand: "Command unrecognized.", + StatusBadArguments: "Syntax error in parameters or arguments.", + StatusNotImplemented: "Command not implemented.", + StatusBadSequence: "Bad sequence of commands.", + StatusNotImplementedParameter: "Command not implemented for that parameter.", + StatusNotLoggedIn: "Not logged in.", + StatusStorNeedAccount: "Need account for storing files.", + StatusFileUnavailable: "File unavailable.", + StatusPageTypeUnknown: "Page type unknown.", + StatusExceededStorage: "Exceeded storage allocation.", + StatusBadFileName: "File name not allowed.", +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 421c8710ad2..a9ba48597e0 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -840,6 +840,12 @@ "path": "github.com/hashicorp/yamux", "revision": "df949784da9ed028ee76df44652e42d37a09d7e4" }, + { + "checksumSHA1": "WKkSFBHssaE3Q3wIQ/xNkTuGqfs=", + "path": "github.com/jlaffaye/ftp", + "revision": "83891dbe0099af272b7f8d094427215a09b5fd0f", + "revisionTime": "2018-01-05T07:56:08Z" + }, { "checksumSHA1": "3/Bhy+ua/DCv2ElMD5GzOYSGN6g=", "comment": "0.2.2-2-gc01cf91",