From c64af70fba9c8cb9bb0534919f43973987e6cc3d Mon Sep 17 00:00:00 2001 From: Catalin Tanasescu Date: Fri, 11 Aug 2017 19:23:27 +0300 Subject: [PATCH] add glob method support --- client_test.go | 225 +++++++++++++++++++++++++++++++- ftp.go | 22 +++- match.go | 347 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 589 insertions(+), 5 deletions(-) create mode 100644 match.go diff --git a/client_test.go b/client_test.go index 865be12..1e0740e 100644 --- a/client_test.go +++ b/client_test.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "net/textproto" "os" + "path/filepath" + "sort" "strings" "testing" "time" @@ -29,21 +31,21 @@ func getConnection() (*ServerConn, error) { } func TestConnPASV(t *testing.T) { - testConn(t, true, false) + testConn(t, true) } func TestConnEPSV(t *testing.T) { - testConn(t, false, false) + testConn(t, false) } func TestConnTLS(t *testing.T) { if !isTLSServer() { t.Skip("skipping test in non TLS server env.") } - testConn(t, false, true) + testConn(t, false) } -func testConn(t *testing.T, disableEPSV bool, implicitTLS bool) { +func testConn(t *testing.T, disableEPSV bool) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -482,3 +484,218 @@ func TestMissingFolderDeleteDirRecur(t *testing.T) { c.Quit() } + +var globTests = []struct { + pattern, result string +}{ + {"glob/match.go", "glob/match.go"}, + {"glob/mat?h.go", "glob/match.go"}, + {"glob/ma*ch.go", "glob/match.go"}, + {"**/match.go", "glob/match.go"}, + {"**/*", "glob/match.go"}, +} + +type MatchTest struct { + pattern, s string + match bool + err error +} + +var matchTests = []MatchTest{ + {"abc", "abc", true, nil}, + {"*", "abc", true, nil}, + {"*c", "abc", true, nil}, + {"a*", "a", true, nil}, + {"a*", "abc", true, nil}, + {"a*", "ab/c", false, nil}, + {"a*/b", "abc/b", true, nil}, + {"a*/b", "a/c/b", false, nil}, + {"a*b*c*d*e*/f", "axbxcxdxe/f", true, nil}, + {"a*b*c*d*e*/f", "axbxcxdxexxx/f", true, nil}, + {"a*b*c*d*e*/f", "axbxcxdxe/xxx/f", false, nil}, + {"a*b*c*d*e*/f", "axbxcxdxexxx/fff", false, nil}, + {"a*b?c*x", "abxbbxdbxebxczzx", true, nil}, + {"a*b?c*x", "abxbbxdbxebxczzy", false, nil}, + {"ab[c]", "abc", true, nil}, + {"ab[b-d]", "abc", true, nil}, + {"ab[e-g]", "abc", false, nil}, + {"ab[^c]", "abc", false, nil}, + {"ab[^b-d]", "abc", false, nil}, + {"ab[^e-g]", "abc", true, nil}, + {"a\\*b", "a*b", true, nil}, + {"a\\*b", "ab", false, nil}, + {"a?b", "a☺b", true, nil}, + {"a[^a]b", "a☺b", true, nil}, + {"a???b", "a☺b", false, nil}, + {"a[^a][^a][^a]b", "a☺b", false, nil}, + {"[a-ζ]*", "α", true, nil}, + {"*[a-ζ]", "A", false, nil}, + {"a?b", "a/b", false, nil}, + {"a*b", "a/b", false, nil}, + {"[\\]a]", "]", true, nil}, + {"[\\-]", "-", true, nil}, + {"[x\\-]", "x", true, nil}, + {"[x\\-]", "-", true, nil}, + {"[x\\-]", "z", false, nil}, + {"[\\-x]", "x", true, nil}, + {"[\\-x]", "-", true, nil}, + {"[\\-x]", "a", false, nil}, + {"[]a]", "]", false, ErrBadPattern}, + {"[-]", "-", false, ErrBadPattern}, + {"[x-]", "x", false, ErrBadPattern}, + {"[x-]", "-", false, ErrBadPattern}, + {"[x-]", "z", false, ErrBadPattern}, + {"[-x]", "x", false, ErrBadPattern}, + {"[-x]", "-", false, ErrBadPattern}, + {"[-x]", "a", false, ErrBadPattern}, + {"\\", "a", false, ErrBadPattern}, + {"[a-b-c]", "a", false, ErrBadPattern}, + {"[", "a", false, ErrBadPattern}, + {"[^", "a", false, ErrBadPattern}, + {"[^bc", "a", false, ErrBadPattern}, + {"a[", "a", false, nil}, + {"a[", "ab", false, ErrBadPattern}, + {"*x", "xxx", true, nil}, +} + +func errp(e error) string { + if e == nil { + return "" + } + return e.Error() +} + +// contains returns true if vector contains the string s. +func contains(vector []string, s string) bool { + for _, elem := range vector { + if elem == s { + return true + } + } + return false +} + +type globTest struct { + pattern string + matches []string +} + +func (test *globTest) buildWant(root string) []string { + var want []string + for _, m := range test.matches { + want = append(want, root+filepath.FromSlash(m)) + } + sort.Strings(want) + return want +} + +func TestMatch(t *testing.T) { + for _, tt := range matchTests { + pattern := tt.pattern + s := tt.s + ok, err := Match(pattern, s) + if ok != tt.match || err != tt.err { + t.Errorf("Match(%#q, %#q) = %v, %q want %v, %q", pattern, s, ok, errp(err), tt.match, errp(tt.err)) + } + } +} + +func TestGlob(t *testing.T) { + + c, err := getConnection() + if err != nil { + t.Fatal(err) + } + if !c.mlstSupported { + t.Skip("skipping test, server not supporting MLST.") + } + err = c.Login("anonymous", "anonymous") + if err != nil { + t.Fatal(err) + } + + err = c.ChangeDir("incoming") + if err != nil { + t.Error(err) + } + + err = c.MakeDir("glob") + if err != nil { + t.Error(err) + } + + data := bytes.NewBufferString("") + err = c.Stor("glob/match.go", data) + if err != nil { + t.Error(err) + } + + for _, tt := range globTests { + pattern := tt.pattern + result := tt.result + matches, err := c.Glob(pattern) + if err != nil { + t.Errorf("Glob error for %q: %s", pattern, err) + continue + } + if !contains(matches, result) { + t.Errorf("Glob(%#q) = %#v want %v", pattern, matches, result) + } + } + for _, pattern := range []string{"no_match", "../*/no_match"} { + matches, err := c.Glob(pattern) + if err != nil { + t.Errorf("Glob error for %q: %s", pattern, err) + continue + } + if len(matches) != 0 { + t.Errorf("Glob(%#q) = %#v want []", pattern, matches) + } + } + + err = c.Logout() + if err != nil { + if protoErr := err.(*textproto.Error); protoErr != nil { + if protoErr.Code != StatusNotImplemented { + t.Error(err) + } + } else { + t.Error(err) + } + } + + c.Quit() +} + +func TestGlobError(t *testing.T) { + c, err := getConnection() + if err != nil { + t.Fatal(err) + } + if !c.mlstSupported { + t.Skip("skipping test, server not supporting MLST.") + } + + err = c.Login("anonymous", "anonymous") + if err != nil { + t.Fatal(err) + } + + _, err = c.Glob("[7]") + if err != nil { + t.Error("expected error for bad pattern; got none") + } + + err = c.Logout() + if err != nil { + if protoErr := err.(*textproto.Error); protoErr != nil { + if protoErr.Code != StatusNotImplemented { + t.Error(err) + } + } else { + t.Error(err) + } + } + + c.Quit() +} diff --git a/ftp.go b/ftp.go index a1417af..ab2ac63 100644 --- a/ftp.go +++ b/ftp.go @@ -46,6 +46,10 @@ type Entry struct { Time time.Time } +func (m Entry) IsDir() bool { + return m.Type == EntryTypeFolder +} + // Response represents a data-connection type Response struct { conn net.Conn @@ -340,7 +344,6 @@ func (c *ServerConn) cmdDataConnFrom(offset uint64, format string, args ...inter conn.Close() return nil, &textproto.Error{Code: code, Msg: msg} } - return conn, nil } @@ -364,6 +367,23 @@ func (c *ServerConn) NameList(path string) (entries []string, err error) { return } +// Stat issues an MLST FTP command. +func (c *ServerConn) Stat(path string) (entry *Entry, err error) { + _, msg, err := c.cmd(StatusRequestedFileActionOK, "MLST %s", path) + if err != nil { + return + } + + lines := strings.Split(msg, "\n") + for _, line := range lines { + entry, err = parseRFC3659ListLine(strings.TrimSpace(line)) + if err == nil { + return entry, err + } + } + return +} + // List issues a LIST FTP command. func (c *ServerConn) List(path string) (entries []*Entry, err error) { var cmd string diff --git a/match.go b/match.go new file mode 100644 index 0000000..e12b034 --- /dev/null +++ b/match.go @@ -0,0 +1,347 @@ +package ftp + +import ( + "errors" + "strings" + "unicode/utf8" +) + +// ErrBadPattern indicates a globbing pattern was malformed. +var ErrBadPattern = errors.New("syntax error in pattern") + +// Unix separator +const separator = "/" + +// Match reports whether name matches the shell file name pattern. +// The pattern syntax is: +// +// pattern: +// { term } +// term: +// '*' matches any sequence of non-Separator characters +// '?' matches any single non-Separator character +// '[' [ '^' ] { character-range } ']' +// character class (must be non-empty) +// c matches character c (c != '*', '?', '\\', '[') +// '\\' c matches character c +// +// character-range: +// c matches character c (c != '\\', '-', ']') +// '\\' c matches character c +// lo '-' hi matches character c for lo <= c <= hi +// +// Match requires pattern to match all of name, not just a substring. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// +func Match(pattern, name string) (matched bool, err error) { +Pattern: + for len(pattern) > 0 { + var star bool + var chunk string + star, chunk, pattern = scanChunk(pattern) + if star && chunk == "" { + // Trailing * matches rest of string unless it has a /. + return !strings.Contains(name, separator), nil + } + // Look for match at current position. + t, ok, err := matchChunk(chunk, name) + // if we're the last chunk, make sure we've exhausted the name + // otherwise we'll give a false result even if we could still match + // using the star + if ok && (len(t) == 0 || len(pattern) > 0) { + name = t + continue + } + if err != nil { + return false, err + } + if star { + // Look for match skipping i+1 bytes. + // Cannot skip /. + for i := 0; i < len(name) && !isPathSeparator(name[i]); i++ { + t, ok, err := matchChunk(chunk, name[i+1:]) + if ok { + // if we're the last chunk, make sure we exhausted the name + if len(pattern) == 0 && len(t) > 0 { + continue + } + name = t + continue Pattern + } + if err != nil { + return false, err + } + } + } + return false, nil + } + return len(name) == 0, nil +} + +// detect if byte(char) is path separator +func isPathSeparator(c byte) bool { + return string(c) == "/" +} + +// scanChunk gets the next segment of pattern, which is a non-star string +// possibly preceded by a star. +func scanChunk(pattern string) (star bool, chunk, rest string) { + for len(pattern) > 0 && pattern[0] == '*' { + pattern = pattern[1:] + star = true + } + inrange := false + var i int +Scan: + for i = 0; i < len(pattern); i++ { + switch pattern[i] { + case '\\': + + // error check handled in matchChunk: bad pattern. + if i+1 < len(pattern) { + i++ + } + case '[': + inrange = true + case ']': + inrange = false + case '*': + if !inrange { + break Scan + } + } + } + return star, pattern[0:i], pattern[i:] +} + +// matchChunk checks whether chunk matches the beginning of s. +// If so, it returns the remainder of s (after the match). +// Chunk is all single-character operators: literals, char classes, and ?. +func matchChunk(chunk, s string) (rest string, ok bool, err error) { + for len(chunk) > 0 { + if len(s) == 0 { + return + } + switch chunk[0] { + case '[': + // character class + r, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + // We can't end right after '[', we're expecting at least + // a closing bracket and possibly a caret. + if len(chunk) == 0 { + err = ErrBadPattern + return + } + // possibly negated + negated := chunk[0] == '^' + if negated { + chunk = chunk[1:] + } + // parse all ranges + match := false + nrange := 0 + for { + if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 { + chunk = chunk[1:] + break + } + var lo, hi rune + if lo, chunk, err = getEsc(chunk); err != nil { + return + } + hi = lo + if chunk[0] == '-' { + if hi, chunk, err = getEsc(chunk[1:]); err != nil { + return + } + } + if lo <= r && r <= hi { + match = true + } + nrange++ + } + if match == negated { + return + } + + case '?': + if isPathSeparator(s[0]) { + return + } + _, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + + case '\\': + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + fallthrough + + default: + if chunk[0] != s[0] { + return + } + s = s[1:] + chunk = chunk[1:] + } + } + return s, true, nil +} + +// getEsc gets a possibly-escaped character from chunk, for a character class. +func getEsc(chunk string) (r rune, nchunk string, err error) { + if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' { + err = ErrBadPattern + return + } + if chunk[0] == '\\' { + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + } + r, n := utf8.DecodeRuneInString(chunk) + if r == utf8.RuneError && n == 1 { + err = ErrBadPattern + } + nchunk = chunk[n:] + if len(nchunk) == 0 { + err = ErrBadPattern + } + return +} + +// Split splits path immediately following the final Separator, +// separating it into a directory and file name component. +// If there is no Separator in path, Split returns an empty dir +// and file set to path. +// The returned values have the property that path = dir+file. +func Split(path string) (dir, file string) { + i := len(path) - 1 + for i >= 0 && !isPathSeparator(path[i]) { + i-- + } + return path[:i+1], path[i+1:] +} + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +func (c *ServerConn) Glob(pattern string) (matches []string, err error) { + if !hasMeta(pattern) { + _, err := c.Stat(pattern) + if err != nil { + return nil, nil + } + dir, _ := Split(pattern) + dir = cleanGlobPath(dir) + return []string{pattern}, nil + } + + dir, file := Split(pattern) + dir = cleanGlobPath(dir) + + if !hasMeta(dir) { + return c.glob(dir, file, nil) + } + + // Prevent infinite recursion. See issue 15879. + if dir == pattern { + return nil, ErrBadPattern + } + + var m []string + m, err = c.Glob(dir) + if err != nil { + return + } + for _, d := range m { + matches, err = c.glob(d, file, matches) + if err != nil { + return + } + } + return +} + +// cleanGlobPath prepares path for glob matching. +func cleanGlobPath(path string) string { + switch path { + case "": + return "." + case string(separator): + // do nothing to the path + return path + default: + return path[0 : len(path)-1] // chop off trailing separator + } +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func (c *ServerConn) glob(dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := c.Stat(dir) + if err != nil { + return + } + if !fi.IsDir() { + return + } + names, err := c.List(dir) + if err != nil { + return + } + + for _, n := range names { + if n.Name == "." || n.Name == ".." { + continue + } + matched, err := Match(pattern, n.Name) + if err != nil { + return m, err + } + if matched { + m = append(m, Join(dir, n.Name)) + } + } + return +} + +// Join joins any number of path elements into a single path, adding +// a Separator if necessary. +// all empty strings are ignored. +func Join(elem ...string) string { + return join(elem) +} +func join(elem []string) string { + // If there's a bug here, fix the logic in ./path_plan9.go too. + for i, e := range elem { + if e != "" && e != "." { + return strings.Join(elem[i:], string(separator)) + } + } + return "" +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.ContainsAny(path, "*?[") +}