Merge pull request #233 from rclone/pr-modtime

Add SetTime command, related options and state getters

client_test.go

@@ -319,3 +319,82 @@ func TestListCurrentDir(t *testing.T) {
 
 	mock.Wait()
 }
+
+func TestTimeUnsupported(t *testing.T) {
+	mock, c := openConnExt(t, "127.0.0.1", "no-time")
+
+	assert.False(t, c.mdtmSupported, "MDTM must NOT be supported")
+	assert.False(t, c.mfmtSupported, "MFMT must NOT be supported")
+
+	assert.False(t, c.IsGetTimeSupported(), "GetTime must NOT be supported")
+	assert.False(t, c.IsSetTimeSupported(), "SetTime must NOT be supported")
+
+	_, err := c.GetTime("file1")
+	assert.NotNil(t, err)
+
+	err = c.SetTime("file1", time.Now())
+	assert.NotNil(t, err)
+
+	assert.NoError(t, c.Quit())
+	mock.Wait()
+}
+
+func TestTimeStandard(t *testing.T) {
+	mock, c := openConnExt(t, "127.0.0.1", "std-time")
+
+	assert.True(t, c.mdtmSupported, "MDTM must be supported")
+	assert.True(t, c.mfmtSupported, "MFMT must be supported")
+
+	assert.True(t, c.IsGetTimeSupported(), "GetTime must be supported")
+	assert.True(t, c.IsSetTimeSupported(), "SetTime must be supported")
+
+	tm, err := c.GetTime("file1")
+	assert.NoError(t, err)
+	assert.False(t, tm.IsZero(), "GetTime must return valid time")
+
+	err = c.SetTime("file1", time.Now())
+	assert.NoError(t, err)
+
+	assert.NoError(t, c.Quit())
+	mock.Wait()
+}
+
+func TestTimeVsftpdPartial(t *testing.T) {
+	mock, c := openConnExt(t, "127.0.0.1", "vsftpd")
+
+	assert.True(t, c.mdtmSupported, "MDTM must be supported")
+	assert.False(t, c.mfmtSupported, "MFMT must NOT be supported")
+
+	assert.True(t, c.IsGetTimeSupported(), "GetTime must be supported")
+	assert.False(t, c.IsSetTimeSupported(), "SetTime must NOT be supported")
+
+	tm, err := c.GetTime("file1")
+	assert.NoError(t, err)
+	assert.False(t, tm.IsZero(), "GetTime must return valid time")
+
+	err = c.SetTime("file1", time.Now())
+	assert.NotNil(t, err)
+
+	assert.NoError(t, c.Quit())
+	mock.Wait()
+}
+
+func TestTimeVsftpdFull(t *testing.T) {
+	mock, c := openConnExt(t, "127.0.0.1", "vsftpd", DialWithWritingMDTM(true))
+
+	assert.True(t, c.mdtmSupported, "MDTM must be supported")
+	assert.False(t, c.mfmtSupported, "MFMT must NOT be supported")
+
+	assert.True(t, c.IsGetTimeSupported(), "GetTime must be supported")
+	assert.True(t, c.IsSetTimeSupported(), "SetTime must be supported")
+
+	tm, err := c.GetTime("file1")
+	assert.NoError(t, err)
+	assert.False(t, tm.IsZero(), "GetTime must return valid time")
+
+	err = c.SetTime("file1", time.Now())
+	assert.NoError(t, err)
+
+	assert.NoError(t, c.Quit())
+	mock.Wait()
+}

conn_test.go

@@ -11,10 +11,12 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
 type ftpMock struct {
 	address  string
+	modtime  string // no-time, std-time, vsftpd
 	listener *net.TCPListener
 	proto    *textproto.Conn
 	commands []string // list of received commands
@@ -28,8 +30,15 @@ type ftpMock struct {
 // newFtpMock returns a mock implementation of a FTP server
 // For simplication, a mock instance only accepts a signle connection and terminates afer
 func newFtpMock(t *testing.T, address string) (*ftpMock, error) {
+	return newFtpMockExt(t, address, "no-time")
+}
+
+func newFtpMockExt(t *testing.T, address, modtime string) (*ftpMock, error) {
 	var err error
-	mock := &ftpMock{address: address}
+	mock := &ftpMock{
+		address: address,
+		modtime: modtime,
+	}
 
 	l, err := net.Listen("tcp", address+":0")
 	if err != nil {
@@ -77,7 +86,15 @@ func (mock *ftpMock) listen(t *testing.T) {
 		// At least one command must have a multiline response
 		switch cmdParts[0] {
 		case "FEAT":
-			mock.proto.Writer.PrintfLine("211-Features:\r\n FEAT\r\n PASV\r\n EPSV\r\n UTF8\r\n SIZE\r\n211 End")
+			features := "211-Features:\r\n FEAT\r\n PASV\r\n EPSV\r\n UTF8\r\n SIZE\r\n"
+			switch mock.modtime {
+			case "std-time":
+				features += " MDTM\r\n MFMT\r\n"
+			case "vsftpd":
+				features += " MDTM\r\n"
+			}
+			features += "211 End"
+			_ = mock.proto.Writer.PrintfLine(features)
 		case "USER":
 			if cmdParts[1] == "anonymous" {
 				mock.proto.Writer.PrintfLine("331 Please send your password")
@@ -196,6 +213,36 @@ func (mock *ftpMock) listen(t *testing.T) {
 			}
 			mock.rest = rest
 			mock.proto.Writer.PrintfLine("350 Restarting at %s. Send STORE or RETRIEVE to initiate transfer", cmdParts[1])
+		case "MDTM":
+			var answer string
+			switch {
+			case mock.modtime == "no-time":
+				answer = "500 Unknown command MDTM"
+			case len(cmdParts) == 3 && mock.modtime == "vsftpd":
+				answer = "213 UTIME OK"
+				_, err := time.ParseInLocation(timeFormat, cmdParts[1], time.UTC)
+				if err != nil {
+					answer = "501 Can't get a time stamp"
+				}
+			case len(cmdParts) == 2:
+				answer = "213 20201213202400"
+			default:
+				answer = "500 wrong number of arguments"
+			}
+			_ = mock.proto.Writer.PrintfLine(answer)
+		case "MFMT":
+			var answer string
+			switch {
+			case mock.modtime == "std-time" && len(cmdParts) == 3:
+				answer = "213 UTIME OK"
+				_, err := time.ParseInLocation(timeFormat, cmdParts[1], time.UTC)
+				if err != nil {
+					answer = "501 Can't get a time stamp"
+				}
+			default:
+				answer = "500 Unknown command MFMT"
+			}
+			_ = mock.proto.Writer.PrintfLine(answer)
 		case "NOOP":
 			mock.proto.Writer.PrintfLine("200 NOOP ok.")
 		case "OPTS":
@@ -307,7 +354,11 @@ func (mock *ftpMock) Close() {
 
 // Helper to return a client connected to a mock server
 func openConn(t *testing.T, addr string, options ...DialOption) (*ftpMock, *ServerConn) {
-	mock, err := newFtpMock(t, addr)
+	return openConnExt(t, addr, "no-time", options...)
+}
+
+func openConnExt(t *testing.T, addr, modtime string, options ...DialOption) (*ftpMock, *ServerConn) {
+	mock, err := newFtpMockExt(t, addr, modtime)
 	if err != nil {
 		t.Fatal(err)
 	}

ftp.go

@@ -27,6 +27,9 @@ const (
 	EntryTypeLink
 )
 
+// Time format used by the MDTM and MFMT commands
+const timeFormat = "20060102150405"
+
 // ServerConn represents the connection to a remote FTP server.
 // A single connection only supports one in-flight data connection.
 // It is not safe to be called concurrently.
@@ -40,6 +43,9 @@ type ServerConn struct {
 	features      map[string]string
 	skipEPSV      bool
 	mlstSupported bool
+	mfmtSupported bool
+	mdtmSupported bool
+	mdtmCanWrite  bool
 	usePRET       bool
 }
 
@@ -58,6 +64,7 @@ type dialOptions struct {
 	disableEPSV bool
 	disableUTF8 bool
 	disableMLSD bool
+	writingMDTM bool
 	location    *time.Location
 	debugOutput io.Writer
 	dialFunc    func(network, address string) (net.Conn, error)
@@ -199,6 +206,18 @@ func DialWithDisabledMLSD(disabled bool) DialOption {
 	}}
 }
 
+// DialWithWritingMDTM returns a DialOption making ServerConn use MDTM to set file time
+//
+// This option addresses a quirk in the VsFtpd server which doesn't support
+// the MFMT command for setting file time like other servers but by default
+// uses the MDTM command with non-standard arguments for that.
+// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
+func DialWithWritingMDTM(enabled bool) DialOption {
+	return DialOption{func(do *dialOptions) {
+		do.writingMDTM = enabled
+	}}
+}
+
 // DialWithLocation returns a DialOption that configures the ServerConn with specified time.Location
 // The location is used to parse the dates sent by the server which are in server's timezone
 func DialWithLocation(location *time.Location) DialOption {
@@ -313,9 +332,11 @@ func (c *ServerConn) Login(user, password string) error {
 	if _, mlstSupported := c.features["MLST"]; mlstSupported && !c.options.disableMLSD {
 		c.mlstSupported = true
 	}
-	if _, usePRET := c.features["PRET"]; usePRET {
+	_, c.usePRET = c.features["PRET"]
-		c.usePRET = true
+
-	}
+	_, c.mfmtSupported = c.features["MFMT"]
+	_, c.mdtmSupported = c.features["MDTM"]
+	c.mdtmCanWrite = c.mdtmSupported && c.options.writingMDTM
 
 	// Switch to binary mode
 	if _, _, err = c.cmd(StatusCommandOK, "TYPE I"); err != nil {
@@ -633,6 +654,12 @@ func (c *ServerConn) List(path string) (entries []*Entry, err error) {
 	return entries, err
 }
 
+// IsTimePreciseInList returns true if client and server support the MLSD
+// command so List can return time with 1-second precision for all files.
+func (c *ServerConn) IsTimePreciseInList() bool {
+	return c.mlstSupported
+}
+
 // ChangeDir issues a CWD FTP command, which changes the current directory to
 // the specified path.
 func (c *ServerConn) ChangeDir(path string) error {
@@ -676,6 +703,49 @@ func (c *ServerConn) FileSize(path string) (int64, error) {
 	return strconv.ParseInt(msg, 10, 64)
 }
 
+// GetTime issues the MDTM FTP command to obtain the file modification time.
+// It returns a UTC time.
+func (c *ServerConn) GetTime(path string) (time.Time, error) {
+	var t time.Time
+	if !c.mdtmSupported {
+		return t, errors.New("GetTime is not supported")
+	}
+	_, msg, err := c.cmd(StatusFile, "MDTM %s", path)
+	if err != nil {
+		return t, err
+	}
+	return time.ParseInLocation(timeFormat, msg, time.UTC)
+}
+
+// IsGetTimeSupported allows library callers to check in advance that they
+// can use GetTime to get file time.
+func (c *ServerConn) IsGetTimeSupported() bool {
+	return c.mdtmSupported
+}
+
+// SetTime issues the MFMT FTP command to set the file modification time.
+// Also it can use a non-standard form of the MDTM command supported by
+// the VsFtpd server instead of MFMT for the same purpose.
+// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
+func (c *ServerConn) SetTime(path string, t time.Time) (err error) {
+	utime := t.In(time.UTC).Format(timeFormat)
+	switch {
+	case c.mfmtSupported:
+		_, _, err = c.cmd(StatusFile, "MFMT %s %s", utime, path)
+	case c.mdtmCanWrite:
+		_, _, err = c.cmd(StatusFile, "MDTM %s %s", utime, path)
+	default:
+		err = errors.New("SetTime is not supported")
+	}
+	return
+}
+
+// IsSetTimeSupported allows library callers to check in advance that they
+// can use SetTime to set file time.
+func (c *ServerConn) IsSetTimeSupported() bool {
+	return c.mfmtSupported || c.mdtmCanWrite
+}
+
 // Retr issues a RETR FTP command to fetch the specified file from the remote
 // FTP server.
 //