Commit 3aa23002 authored by Jérome Perrin's avatar Jérome Perrin

Allow sorting files in FTPConnector

Extend`FTPConnector.listFiles` to sort files based on some server-side computed attributes of files.

This uses [`listdir_attr`](http://docs.paramiko.org/en/2.4/api/sftp.html#paramiko.sftp_client.SFTPClient.listdir_attr) method of paramiko. `sort_on` parameter is directly attributes of [`paramiko.sftp_attr.SFTPAttributes`]( http://docs.paramiko.org/en/2.4/api/sftp.html#paramiko.sftp_attr.SFTPAttributes).

At the same time, adds a minimal tests, that will only run if `testSFTPConnection_SFTP_URL` environment variable is set to an URI of a running sftp server supporting password authentication - which means that in our current tests infrastructure it will not run.

/cc @vpelletier @aurel @rafael 

/reviewed-on !680
parents 3bdde386 3b666bab
...@@ -79,11 +79,11 @@ class FTPConnector(XMLObject): ...@@ -79,11 +79,11 @@ class FTPConnector(XMLObject):
finally: finally:
conn.logout() conn.logout()
def listFiles(self, path="."): def listFiles(self, path=".", sort_on=None):
""" List file of a directory """ """ List file of a directory """
conn = self.getConnection() conn = self.getConnection()
try: try:
return conn.getDirectoryContent(path) return conn.getDirectoryContent(path, sort_on=sort_on)
finally: finally:
conn.logout() conn.logout()
......
##############################################################################
#
# Copyright (c) 2018- Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
##############################################################################
import os
import unittest
import urlparse
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
class TestSFTPConnection(ERP5TypeTestCase):
if os.environ.get("testSFTPConnection_SFTP_URL"):
def afterSetUp(self):
url = os.environ["testSFTPConnection_SFTP_URL"]
parsed_url = urlparse.urlparse(url)
self.connection = self.portal.portal_web_services.newContent(
portal_type='FTP Connector',
reference=self.id(),
user_id=parsed_url.username,
password=parsed_url.password,
url_string=url,
url_protocol='sftp',
use_temporary_file_on_write=False)
def beforeTearDown(self):
for f in self.connection.listFiles("."):
self.connection.removeFile(f)
self.assertEqual([], self.connection.listFiles("."))
def test_create_read_delete_file(self):
self.connection.putFile("filename", "file content")
self.assertEqual(
"file content",
self.connection.getFile("filename")
)
self.connection.removeFile("filename")
# after file is removed, an IOError is raised when trying to read it.
self.assertRaises(
IOError,
self.connection.getFile, "filename"
)
def test_put_rename(self):
self.connection.putFile("filename", "file content")
self.connection.renameFile("filename", "new name")
self.assertEqual(
"file content",
self.connection.getFile("new name")
)
def test_list_dir(self):
self.connection.putFile("first_file", "first file content ( a bit bigger )")
self.connection.putFile("second_file", "second file content")
# by default, ordering is not specified
self.assertItemsEqual(
["first_file", "second_file"],
self.connection.listFiles(".")
)
# but we can sort by modification date
self.assertEqual(
["first_file", "second_file"],
self.connection.listFiles(".", sort_on="st_mtime")
)
# or by file size
self.assertEqual(
["second_file" , "first_file"],
self.connection.listFiles(".", sort_on="st_size")
)
else:
def test_no_SFTP_URL_in_environ(self):
raise unittest.SkipTest(
"""This test needs the environment variable testSFTPConnection_SFTP_URL set to the URL of a SFTP connection.
The URL must contain login and password, such as sftp://user:pass@[::1]:8022
The directory from this URL must be empty and writeable.
"""
)
\ No newline at end of file
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Test Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>testFTPConnection</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>test.erp5.testFTPConnection</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Test Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
test.erp5.testFTPConnection
\ No newline at end of file
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
############################################################################## ##############################################################################
import os, socket import os, socket
import operator
from urlparse import urlparse from urlparse import urlparse
from socket import gaierror, error, socket, getaddrinfo, AF_UNSPEC, SOCK_STREAM from socket import gaierror, error, socket, getaddrinfo, AF_UNSPEC, SOCK_STREAM
from xmlrpclib import Binary from xmlrpclib import Binary
...@@ -136,9 +137,17 @@ class SFTPConnection: ...@@ -136,9 +137,17 @@ class SFTPConnection:
# normalise CRLF/CR/LF like FTP's ASCII mode transfer. # normalise CRLF/CR/LF like FTP's ASCII mode transfer.
return os.linesep.join(self._getFile(filepath).splitlines()) return os.linesep.join(self._getFile(filepath).splitlines())
def getDirectoryContent(self, path): def getDirectoryContent(self, path, sort_on=None):
"""retrieve all entries in a givan path as a list""" """retrieve all entries in a givan path as a list.
`sort_on` parameter allows to retrieve the directory content in a sorted
order, it understands all parameters from
paramiko.sftp_attr.SFTPAttributes, the most useful being `st_mtime` to sort
by modification date.
"""
try: try:
if sort_on:
return [x.filename for x in sorted(self.conn.listdir_attr(path), key=operator.attrgetter(sort_on))]
return self.conn.listdir(path) return self.conn.listdir(path)
except (EOFError, error), msg: except (EOFError, error), msg:
raise SFTPError(str(msg) + ' while trying to list %s on %s' % (path, self.url)) raise SFTPError(str(msg) + ' while trying to list %s on %s' % (path, self.url))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment