Repository: TACC/agavepy

Issue:
Implement files_import
**Is your feature request related to a problem? Please describe.**
Part of the changes proposed for release v3.0.0 of `TACC-Cloud/agave-cli`

**Describe the solution you'd like**
Implement a method to import a remote URI to a remote storage system.

Discussion:

Patch:
diff --git a/agavepy/agave.py b/agavepy/agave.py
index 26d0b79..b967bd8 100644
--- a/agavepy/agave.py
+++ b/agavepy/agave.py
@@ -25,8 +25,8 @@ from agavepy.clients import (clients_create, clients_delete, clients_list,
 from agavepy.tokens import token_create, refresh_token
 from agavepy.utils import load_config, save_config
 from agavepy.files import (files_copy, files_delete, files_download, 
-    files_history, files_list, files_mkdir, files_move, files_pems_delete, 
-    files_pems_list, files_pems_update, files_upload)
+    files_history, files_import, files_list, files_mkdir, files_move, 
+    files_pems_delete, files_pems_list, files_pems_update, files_upload)
 
 
 import sys
@@ -826,6 +826,19 @@ class Agave(object):
         files_history(self.api_server, self.token, path)
 
 
+    def files_import(self, source, destination):
+        """ Imports a remote URI to a remote storage system
+        
+        If 'source' is an agave source then prefix the uri with 'agave://'. For
+        example, source = 'agave://data-sd2e-community/test.txt'.
+        """
+        # Check if tokens need to be refreshed.
+        self.refresh_tokens()
+
+        # Import file.
+        files_import(self.api_server, self.token, source, destination)
+
+
     def files_list(self, system_path, long_format=False):
         """ List files on remote system
         """
diff --git a/agavepy/files/__init__.py b/agavepy/files/__init__.py
index 5aedd4e..5783bf1 100644
--- a/agavepy/files/__init__.py
+++ b/agavepy/files/__init__.py
@@ -2,6 +2,7 @@ from .copy import files_copy
 from .delete import files_delete
 from .download import files_download
 from .history import files_history
+from .import_files import files_import
 from .list import files_list
 from .mkdir import files_mkdir
 from .move import files_move
diff --git a/agavepy/files/import_files.py b/agavepy/files/import_files.py
new file mode 100644
index 0000000..cfdbe8d
--- /dev/null
+++ b/agavepy/files/import_files.py
@@ -0,0 +1,34 @@
+"""
+    import.py
+"""
+import requests
+from .exceptions import AgaveFilesError
+from ..utils import handle_bad_response_status_code
+
+
+def files_import(tenant_url, access_token, source, destination):
+    """ Imports a remote URI to a remote storage system
+
+    If 'source' is an agave source then prefix the uri with 'agave://'. For 
+    example, source = 'agave://data-sd2e-community/test.txt'.
+    """
+    # Set request url.
+    endpoint = "{0}/{1}/{2}".format(tenant_url, "files/v2/media/system", destination)
+
+    # Make request.
+    try:
+        # TODO: document the other fields.
+        data = {
+            "urlToIngest": source,
+            "notifications": "",
+            "fileType": "",
+            "fileName": "",
+        }
+        headers  = {"Authorization":"Bearer {0}".format(access_token)}
+        params   = {"pretty": "true"}
+        resp = requests.post(endpoint, headers=headers, data=data, params=params)
+    except Exception as err:
+        raise AgaveFilesError(err)
+
+    # Handle bad status code.
+    handle_bad_response_status_code(resp)
diff --git a/docs/docsite/files/files.rst b/docs/docsite/files/files.rst
index 82c46c2..6af157d 100644
--- a/docs/docsite/files/files.rst
+++ b/docs/docsite/files/files.rst
@@ -109,6 +109,23 @@ off:
     >>> agave.files_delete("data-tacc-user/somefile-or-directory")
 
 
+Import files from ther systems
+##############################
+
+It may be useful to import data from other storage systems, e.g. from the
+community data space to your private data space.
+The ``files_import`` method can be used for that purpose.
+
+.. code-block:: pycon
+
+    agave.files_import("agave://data-community/test.txt", "system-id/")
+
+Please also note that even though you are able to import files from other Agave
+storage systems, you may not always need to import those files.
+Also, note that the source, the first argument, must be an agave compliant uri
+by prefixing the system if and path convination with the string ``agave://``.
+
+
 Making a directory
 ##################
 


Test patch:
diff --git a/tests/files_test.py b/tests/files_test.py
index d23a830..c6be1d1 100644
--- a/tests/files_test.py
+++ b/tests/files_test.py
@@ -77,7 +77,7 @@ class MockServerFilesEndpoints(BaseHTTPRequestHandler):
 
 
     def do_POST(self):
-        """ Upload file
+        """ Upload or import a file
 
         Save uploaded file to current working directory and send a response.
         """
@@ -97,29 +97,47 @@ class MockServerFilesEndpoints(BaseHTTPRequestHandler):
                 "CONTENT_TYPE": self.headers["Content-Type"]
             })
 
-        # Save uploaded file.
-        fname = form["fileToUpload"].filename
-        with open(fname, "wb") as flocal:
-            shutil.copyfileobj(form["fileToUpload"].file, flocal)
-
-        # Send response.
-        request_source = self.headers.get("HOST")
-        file_path      = "/".join(elements[5:] + [fname])
-        system_id      = elements[4]
-        request_url    = "".join(["https://tenant", self.path, "/", fname])
-        system_url     = "".join(["https://tenant", "/systems/v2/", system_id])
-        history_url    = "".join(["https://tenant", "/files/v2/history/system/", system_id, "/", file_path])
-        sample_files_upload_response["result"]["name"] = fname
-        sample_files_upload_response["result"]["source"] = request_source
-        sample_files_upload_response["result"]["path"] = file_path
-        sample_files_upload_response["result"]["systemId"] = system_id
-        sample_files_upload_response["result"]["_links"]["self"]["href"] = request_url
-        sample_files_upload_response["result"]["_links"]["system"]["href"] = system_url
-        sample_files_upload_response["result"]["_links"]["history"]["href"] = history_url
-        try: # python 2
-            self.wfile.write(json.dumps(sample_files_upload_response))
-        except TypeError:
-            self.wfile.write(json.dumps(sample_files_upload_response).encode())
+        # Check if request is for uploading or importing a file.
+        action = ""
+        if form.getvalue("fileToUpload", "") != "":
+            action = "upload"
+        elif form.getvalue("urlToIngest", "") != "":
+            action = "import"
+        else:
+            self.send_response(400)
+            self.end_headers()
+            return
+
+        if action == "upload":
+            # Save uploaded file.
+            fname = form["fileToUpload"].filename
+            with open(fname, "wb") as flocal:
+                shutil.copyfileobj(form["fileToUpload"].file, flocal)
+                
+            # Send response.
+            request_source = self.headers.get("HOST")
+            file_path      = "/".join(elements[5:] + [fname])
+            system_id      = elements[4]
+            request_url    = "".join(["https://tenant", self.path, "/", fname])
+            system_url     = "".join(["https://tenant", "/systems/v2/", system_id])
+            history_url    = "".join(["https://tenant", "/files/v2/history/system/", system_id, "/", file_path])
+            sample_files_upload_response["result"]["name"] = fname
+            sample_files_upload_response["result"]["source"] = request_source
+            sample_files_upload_response["result"]["path"] = file_path
+            sample_files_upload_response["result"]["systemId"] = system_id
+            sample_files_upload_response["result"]["_links"]["self"]["href"] = request_url
+            sample_files_upload_response["result"]["_links"]["system"]["href"] = system_url
+            sample_files_upload_response["result"]["_links"]["history"]["href"] = history_url
+            
+            try: # python 2
+                self.wfile.write(json.dumps(sample_files_upload_response))
+            except TypeError:
+                self.wfile.write(json.dumps(sample_files_upload_response).encode())
+        
+        elif action == "import":
+            self.send_response(200)
+            self.end_headers()
+            return
 
 
     def do_PUT(self):
@@ -241,6 +259,8 @@ class TestMockServer(MockServer):
             local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
             agave = Agave(api_server=local_uri)
             agave.token = "mock-access-token"
+            agave.created_at = str(int(time.time()))
+            agave.expires_in = str(14400)
 
             agave.files_download("tacc-globalfs-user/"+tmp_file, local_file)
         finally:
@@ -264,6 +284,8 @@ class TestMockServer(MockServer):
             local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
             agave = Agave(api_server=local_uri)
             agave.token = "mock-access-token"
+            agave.created_at = str(int(time.time()))
+            agave.expires_in = str(14400)
 
             agave.files_upload(tmp_file, "tacc-globalfs-user/")
         finally:
@@ -286,6 +308,8 @@ class TestMockServer(MockServer):
             local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
             agave = Agave(api_server=local_uri)
             agave.token = "mock-access-token"
+            agave.created_at = str(int(time.time()))
+            agave.expires_in = str(14400)
 
             agave.files_delete("tacc-globalfs-user/"+tmp_file)
         finally:
@@ -305,6 +329,8 @@ class TestMockServer(MockServer):
         local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
         agave = Agave(api_server=local_uri)
         agave.token = "mock-access-token"
+        agave.created_at = str(int(time.time()))
+        agave.expires_in = str(14400)
         
         agave.files_copy("tacc-globalfs/file", "tacc-globalfs/file-copy")
 
@@ -318,6 +344,8 @@ class TestMockServer(MockServer):
         local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
         agave = Agave(api_server=local_uri)
         agave.token = "mock-access-token"
+        agave.created_at = str(int(time.time()))
+        agave.expires_in = str(14400)
 
         agave.files_copy("tacc-globalfs/file", "tacc-globalfs/another-file")
 
@@ -331,5 +359,22 @@ class TestMockServer(MockServer):
         local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
         agave = Agave(api_server=local_uri)
         agave.token = "mock-access-token"
+        agave.created_at = str(int(time.time()))
+        agave.expires_in = str(14400)
 
         agave.files_mkdir("tacc-globalfs/new/path")
+
+
+    def test_files_import(self, capfd):
+        """ Test files import
+        """
+        local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
+        agave = Agave(api_server=local_uri)
+        agave.token = "mock-access-token"
+        agave.created_at = str(int(time.time()))
+        agave.expires_in = str(14400)
+
+        # Import file.
+        agave.files_import("agave://data-sd2e-community/test.txt", "tacc-globalfs-username/")
+        out, err = capfd.readouterr()
+        assert "200" in err