Repository: TACC/agavepy

Issue:
Write a cleaner implementation of files.list
The current implementation of `Agave.files.list` is untested and relies on the swaggerpy library that is packaged with agavepy. Write a cleaner implementation of a method to list files from a remote agave execution system.

Discussion:

Patch:
diff --git a/Makefile b/Makefile
index 43a069c..b811166 100644
--- a/Makefile
+++ b/Makefile
@@ -29,6 +29,7 @@ clean:
 	rm -rf agavepy/tenants/*.pyc agavepy/tenants/__pycache__/
 	rm -rf agavepy/clients/*.pyc agavepy/clients/__pycache__/
 	rm -rf agavepy/tokens/*.pyc agavepy/tokens/__pycache__/
+	rm -rf agavepy/files/*.pyc agavepy/files/__pycache__/
 	rm -rf agavepy/utils/*.pyc agavepy/utils/__pycache__/
 	rm -rf agavepy/tests/__pycache__/
 	rm -rf agavepy/*.pyc
diff --git a/agavepy/agave.py b/agavepy/agave.py
index d0caa80..92d9289 100644
--- a/agavepy/agave.py
+++ b/agavepy/agave.py
@@ -23,7 +23,7 @@ from agavepy.tenants import tenant_list
 from agavepy.clients import clients_create, clients_list
 from agavepy.tokens import token_create, refresh_token
 from agavepy.utils import load_config, save_config
-from agavepy.files import files_download, files_upload
+from agavepy.files import files_download, files_list, files_upload
 
 
 import sys
@@ -736,6 +736,17 @@ class Agave(object):
         # Download file.
         files_download(self.api_server, self.token, source, destination)
 
+
+    def files_list(self, system_path, long_format=False):
+        """ List files on remote system
+        """
+        # Check if tokens need to be refreshed.
+        self.refresh_tokens()
+
+        # List files.
+        files_list(self.api_server, self.token, system_path, long_format=long_format)
+
+
     def files_upload(self, source, destination):
         """ Upload file to remote system
         """
diff --git a/agavepy/files/__init__.py b/agavepy/files/__init__.py
index 335e2b7..3e55cdd 100644
--- a/agavepy/files/__init__.py
+++ b/agavepy/files/__init__.py
@@ -1,2 +1,3 @@
 from .download import files_download
+from .list import files_list
 from .upload import files_upload
diff --git a/agavepy/files/list.py b/agavepy/files/list.py
new file mode 100644
index 0000000..157db67
--- /dev/null
+++ b/agavepy/files/list.py
@@ -0,0 +1,149 @@
+"""
+    files.py
+"""
+from __future__ import print_function, division
+import py
+import os
+import requests
+import shutil
+import time
+from operator import itemgetter
+from .exceptions import AgaveFilesError                                         
+from ..utils import handle_bad_response_status_code
+
+
+
+file_permissions = {
+    "READ"          : "-r--",
+    "WRITE"         : "--w-",
+    "EXECUTE"       : "---x",
+    "READ_WRITE"    : "-rw-",
+    "READ_EXECUTE"  : "-r-x",
+    "WRITE_EXECUTE" : "--wx",
+    "ALL"           : "-rwx",
+    "NONE"          : "----"
+}
+
+
+def parse_time(ftime):
+    """ Convert timestamp to local time
+
+    Expect format: 2018-07-10T12:28:01.000-05:00
+
+    """
+    # 2018-07-10T12:28:01.000-05:00 (rm last ':')
+    if ftime[-3] == ":": ftime = ftime[:-3] + ftime[-2:]
+
+    try:
+        ftime = time.strptime(ftime,'%Y-%m-%dT%H:%M:%S.%f%z')
+    except ValueError:
+        tz = ftime[-5:] # get the timezone part (i.e., -0500)
+        ftime = time.strptime(ftime[:-5],'%Y-%m-%dT%H:%M:%S.%f')
+        if tz.startswith("-"):
+            sign = -1
+            tz = tz[1:] # get the hous offset
+        elif tz.startswith("+"):
+            tz = tz[1:] # get the hous offset
+        seconds = ( int(tz[0:2])*60 + int(tz[2:4]) ) * 60
+        seconds *= sign
+        # Convert ftime to seconds since epoch,
+        ftime = time.mktime(ftime) + seconds
+        # Convert seconds since epoch to localtime.
+        ftime = time.localtime(ftime)
+
+    outtime = time.strftime("%b %d %H:%M", ftime).split()
+
+    return ftime, outtime
+
+
+
+def files_list(tenant_url, access_token, system_path, long_format=False):
+    """ List files on a remote Agave system
+    """
+    # Set endpoint.
+    endpoint = "{0}/{1}/{2}".format(tenant_url, "files/v2/listings/system", system_path)
+
+    # Make request.                                                             
+    try:
+        headers  = {"Authorization":"Bearer {0}".format(access_token)}
+        params   = {"pretty": "true"}
+        resp = requests.get(endpoint, headers=headers, params=params)
+    except Exception as err:
+        raise AgaveFilesError(err)
+    
+    # Handle bad status code.
+    handle_bad_response_status_code(resp)
+
+
+    # Sort results alphabetically.
+    resp.json()["result"] = sorted(resp.json()["result"], key=itemgetter("name"), reverse=True)
+
+    # Get the length of the longest filename.
+    longest_name = 8
+    for f in resp.json()["result"]:
+        if len(f["name"]) > longest_name:
+            longest_name = len(f["name"])
+    # Get the size fo the biggest file.
+    largest_file = 0
+    for f in resp.json()["result"]:
+        if len(str(f["length"])) > largest_file:
+            largest_file = len(str(f["length"]))
+
+    # Get size of terminal.
+    try: # python3 prefered
+        terminal_size = shutil.get_terminal_size()
+        terminal_size_columns = terminal_size.columns 
+        columns = terminal_size.columns // longest_name
+    except AttributeError:
+        # _rows, columns = os.popen('stty size', 'r').read().split()
+        columns = py.io.TerminalWriter().fullwidth
+        terminal_size_columns = int(columns)
+        columns = int(columns) // longest_name
+
+
+    if not long_format:
+        files = ""
+        line_length = 0
+        for f in resp.json()["result"]:
+            name   = f["name"]
+            # If directory, then append '/' to its name.
+            if f["type"] == "dir":
+                name += "/"
+
+            # Give enough space to print name of file/directory.
+            name += " " * (longest_name - len(name) + 3)
+
+            # If there is not enough space in the current line to print this
+            # file then move to the next line.
+            if line_length + len(name) > terminal_size_columns:
+                files += "\n"
+                line_length = 0
+
+            line_length += len(name)
+            files += name
+        
+        # Print all files.
+        print(files)
+
+    else: # Print file long format.
+        for f in resp.json()["result"]:
+            # File permissions and name.
+            name   = f["name"]
+
+            # Set file permissions.
+            perm = file_permissions[f["permissions"]]
+            # Prefix perfmissions for directories with 'd'.
+            if f["type"] == "dir":
+                perm = "d{}".format(perm[1:])
+                name += "/"
+            name += " " * (longest_name - len(name) + 3)
+
+            # File size.
+            fsize = f["length"]
+
+            # Date created.
+            ftime, outtime = parse_time( f["lastModified"] )
+
+            print("{0:<4} {1:>{size_width}} {2:<3} {3:>2} {4:<5} {5:}".format(
+                    perm, fsize, outtime[0], ftime.tm_mday, outtime[2], name, 
+                    size_width=largest_file))
diff --git a/docs/docsite/files/files.rst b/docs/docsite/files/files.rst
index d80b5f0..5dd9849 100644
--- a/docs/docsite/files/files.rst
+++ b/docs/docsite/files/files.rst
@@ -7,6 +7,48 @@ Working with files
 If you have already initiated a session, `authentication`, and have a storage
 system setup then you start working with files.
 
+
+Listing files and directories
+#############################
+
+Before we even get started, we should know what it is we are going to be
+working with.
+To see any files or directories in your remote storage system, let's say this
+is called ``tacc-globalfs-username``.
+
+.. code-block:: pycon
+
+    >>> agave.files_list("tacc-globalfs-username/")
+    ./                 .slurm/            apps/              benchmarks/
+    bigfile            cptests/           users/             myotherfile
+    old-sd2e-apps/     rpmbuild/          sd2e-apps/         sd2e-data/
+    singularity-test/  somefile           SP1-copy.fq        SP1.fq 
+    tests/             
+
+For more details, such as permissions and size:
+
+.. code-block:: pycon
+
+    >>> agave.files_list("tacc-globalfs-username/", long_format=True)
+    drwx       4096 Oct 25 14:47 ./
+    drwx       4096 May 25 16:13 .slurm/
+    drwx       4096 Jun  5 10:14 apps/ 
+    drwx       4096 Jun  9 18:39 benchmarks/
+    -rw- 1073741824 Oct 25 14:48 bigfile
+    drwx       4096 Jul 13 09:17 cptests/
+    drwx       4096 Aug  1 14:26 users/ 
+    -rw-         24 Oct 25 14:42 myotherfile 
+    drwx       4096 Jul 10 11:27 old-sd2e-apps/   
+    drwx       4096 Jun 29 10:40 rpmbuild/    
+    drwx       4096 Jul 13 15:41 sd2e-apps/
+    drwx       4096 Jul  7 20:43 sd2e-data/   
+    drwx       4096 Jul  9 16:50 singularity-test/  
+    -rw-         24 Oct 25 14:20 somefile
+    -rw-      22471 Aug 22 14:40 SP1-copy.fq   
+    -rw-      22471 Jul  7 20:42 SP1.fq 
+    drwx       4096 Aug 24 13:44 tests/   
+    
+
 Download a file
 ###############
 
@@ -24,10 +66,10 @@ where you want to store the file (i.e., ``/opt/data/SP1.fq``) as such:
 Upload a file
 #############
 
-Similarly, if you want to upload a file, ``my_important_data.ext``, and you
+Similarly, if you want to upload a file, ``important_data.ext``, and you
 want to save it on your stoorage system named ``tacc-globalfs-username`` under
-the name ``super_cool_data.bin``.
+the name ``cool_data.bin``.
 
 .. code-block:: pycon
 
-    >>> agave.files_upload("./my_important_data.ext", "tacc-globalfs-username/super_cool_data.bin")
+    >>> agave.files_upload("./important_data.ext", "tacc-globalfs-username/cool_data.bin")


Test patch:
diff --git a/tests/files-listing_test.py b/tests/files-listing_test.py
new file mode 100644
index 0000000..5e1dbf7
--- /dev/null
+++ b/tests/files-listing_test.py
@@ -0,0 +1,63 @@
+"""
+    files-listings_tests.py
+"""
+import pytest
+import json
+import os
+import sys
+from testsuite_utils import MockServer
+from response_templates import response_template_to_json
+try: # python 2                                                                 
+    from BaseHTTPServer import BaseHTTPRequestHandler                           
+except ImportError: # python 3                                                  
+    from http.server import BaseHTTPRequestHandler
+from agavepy.agave import Agave
+
+
+# Sample successful responses from the agave api.
+sample_files_list_response = response_template_to_json("files-list.json")
+
+
+
+class MockServerListingsEndpoints(BaseHTTPRequestHandler):
+    """ Mock the Agave API
+    """
+    def do_GET(self):
+        """ Mock oauth client listing.
+        """
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(json.dumps(sample_files_list_response).encode())
+
+
+
+class TestMockServer(MockServer):
+    """ Test file listing-related agave api endpoints 
+    """
+
+    @classmethod
+    def setup_class(cls):
+        """ Set up an agave mock server
+
+        Listen and serve mock api as a daemon.
+        """
+        MockServer.serve.__func__(cls, MockServerListingsEndpoints)
+
+
+    def test_files_list(self, capfd):
+        """ Test files listings
+        """
+        local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
+        agave = Agave(api_server=local_uri)
+        agave.token = "mock-access-token"
+
+        # List files.
+        agave.files_list("tacc-globalfs-username/tests", long_format=False)
+
+        # Stdout should contain the putput from the command.
+        # Stderr will contain logs from the mock http server.
+        out, err = capfd.readouterr()
+        assert "copy.txt" in out
+        assert "sp1-copy.fq" in out
+        assert "upload.txt" in out
+        assert "\"GET //files/v2/listings/system/tacc-globalfs-username/tests?pretty=true HTTP/1.1\" 200" in err 
diff --git a/tests/files_test.py b/tests/files_test.py
index b5e61cb..e94a4da 100644
--- a/tests/files_test.py
+++ b/tests/files_test.py
@@ -17,10 +17,6 @@ try: # python 2
     from BaseHTTPServer import BaseHTTPRequestHandler
 except ImportError: # python 3
     from http.server import BaseHTTPRequestHandler
-try:
-    from unittest.mock import patch
-except ImportError:
-    from mock import patch
 
 from agavepy.agave import Agave
 
diff --git a/tests/sample_responses/files-list.json b/tests/sample_responses/files-list.json
new file mode 100644
index 0000000..7a248c1
--- /dev/null
+++ b/tests/sample_responses/files-list.json
@@ -0,0 +1,89 @@
+{
+  "status": "success",
+  "message": null,
+  "version": "2.2.22-r7deb380",
+  "result": [
+    {
+      "name": ".",
+      "path": "/tests",
+      "lastModified": "2018-08-24T13:44:12.000-05:00",
+      "length": 4096,
+      "permissions": "ALL",
+      "format": "folder",
+      "mimeType": "text/directory",
+      "type": "dir",
+      "system": "tacc-globalfs-username",
+      "_links": {
+        "self": {
+          "href": "https://api.sd2e.org/files/v2/media/system/tacc-globalfs-username//tests"
+        },
+        "system": {
+          "href": "https://api.sd2e.org/systems/v2/tacc-globalfs-username"
+        },
+        "metadata": {
+          "href": "https://api.sd2e.org/meta/v2/data?q=%7B%22associationIds%22%3A%222700041210-4a1300-02%2%D"
+        },
+        "history": {
+          "href": "https://api.sd2e.org/files/v2/history/system/tacc-globalfs-username//tests"
+        }
+      }
+    },
+    {
+      "name": "copy.txt",
+      "path": "/tests/copy.txt",
+      "lastModified": "2018-07-26T15:09:56.000-05:00",
+      "length": 10,
+      "permissions": "READ_WRITE",
+      "format": "raw",
+      "system": "tacc-globalfs-username",
+      "mimeType": "text/plain",
+      "type": "file",
+      "_links": {
+        "self": {
+          "href": "https://api.sd2e.org/files/v2/media/system/tacc-globalfs-username//tests/copy.txt"
+        },
+        "system": {
+          "href": "https://api.sd2e.org/systems/v2/tacc-globalfs-username"
+        }
+      }
+    },
+    {
+      "name": "sp1-copy.fq",
+      "path": "/tests/sp1-copy.fq",
+      "lastModified": "2018-08-22T15:12:32.000-05:00",
+      "length": 22471,
+      "permissions": "READ_WRITE",
+      "format": "raw",
+      "system": "tacc-globalfs-username",
+      "mimeType": "application/octet-stream",
+      "type": "file",
+      "_links": {
+        "self": {
+          "href": "https://api.sd2e.org/files/v2/media/system/tacc-globalfs-username//tests/sp1-copy.fq"
+        },
+        "system": {
+          "href": "https://api.sd2e.org/systems/v2/tacc-globalfs-username"
+        }
+      }
+    },
+    {
+      "name": "upload.txt",
+      "path": "/tests/upload.txt",
+      "lastModified": "2018-07-27T15:02:09.000-05:00",
+      "length": 15,
+      "permissions": "READ_WRITE",
+      "format": "raw",
+      "system": "tacc-globalfs-username",
+      "mimeType": "text/plain",
+      "type": "file",
+      "_links": {
+        "self": {
+          "href": "https://api.sd2e.org/files/v2/media/system/tacc-globalfs-username//tests/upload.txt"
+        },
+        "system": {
+          "href": "https://api.sd2e.org/systems/v2/tacc-globalfs-username"
+        }
+      }
+    }
+  ]
+}