Repository: infobloxopen/infoblox-client

Issue:
Missing required support for file upload and download on the fileop object
- OS - Mac OS X 11.2.1 (big sur)
- Python 3.9.2
- infoblox-client 0.5.0 (master)

There is currently no support for uploading files or downloading files when using the fileop object. For example, when calling uploadinit function on the Connector as part of a CSV import, there is no mechanism for performing a POST w/ files=<files_object> to the fully-qualified URL provided in the uploadinit response. The flow of performing a CSV import is as follows:

1. call function on the connector _uploadinit_ - the repose contains TOKEN and URL (a fully-qualified URL upload path)
2. perform a POST to the fully-qualified URL provided in step 1 w/ a files=<file object> as the payload
3. call function on the connector _csv_import_ using the TOKEN provided in step 1

There are two (2) items of functionality that are not handled in the infoblox-client:
1. there is no method on the Connector class that allows you to pass a fully-qualified URL to a POST or GET
2. there is no support for handling multipart-form/data support for handling files when uploading - there is only data=<payload> support, not files=<file_payload>

One option to fix this is to add two (2) generic methods on the Connector class a simple POST and GET method call with the required params in the method signature.

Another option is to examine the Connector class and see if it can be incorporated into the call_func() method. Not sure this is the right spot since call_func() is likely to call WAPI functions on supported objects.

Discussion:

Patch:
diff --git a/.gitignore b/.gitignore
index aeb75fc..b500602 100644
--- a/.gitignore
+++ b/.gitignore
@@ -57,3 +57,4 @@ docs/_build
 Session.vim
 .netrwhist
 *~
+pyvenv.cfg
diff --git a/infoblox_client/connector.py b/infoblox_client/connector.py
index 2690e0a..4e3fb52 100644
--- a/infoblox_client/connector.py
+++ b/infoblox_client/connector.py
@@ -222,9 +222,9 @@ class Connector(object):
         except ValueError:
             raise ib_ex.InfobloxConnectionError(reason=request.content)
 
-    def _log_request(self, type, url, opts):
+    def _log_request(self, verb, url, opts):
         message = ("Sending %s request to %s with parameters %s",
-                   type, url, opts)
+                   verb, url, opts)
         if self.log_api_calls_as_info:
             LOG.info(*message)
         else:
@@ -344,6 +344,47 @@ class Connector(object):
             return None
         return self._parse_reply(r)
 
+    @reraise_neutron_exception
+    def download_file(self, url):
+        if self.session.cookies:
+            self.session.auth = None
+        headers = {'content-type': 'application/force-download'}
+        r = self.session.get(url, headers=headers)
+        if r.status_code != requests.codes.ok:
+            response = utils.safe_json_load(r.content)
+            raise ib_ex.InfobloxFileDownloadFailed(
+                response=response,
+                url=url
+            )
+
+        return r
+
+    @reraise_neutron_exception
+    def upload_file(self, url, files):
+        """Upload file to fully-qualified upload url
+
+        Args:
+            url (str): upload url provided by infoblox fileop uploadinit
+            files (dict): file contents payload
+        Returns:
+            The requests response
+        Raises:
+            InfobloxException
+        """
+        if self.session.cookies:
+            self.session.auth = None
+        r = self.session.post(url, files=files)
+        if r.status_code != requests.codes.ok:
+            response = utils.safe_json_load(r.content)
+            raise ib_ex.InfobloxFileUploadFailed(
+                response=response,
+                url=url,
+                content=response,
+                code=r.status_code,
+            )
+
+        return r
+
     @reraise_neutron_exception
     def create_object(self, obj_type, payload, return_fields=None):
         """Create an Infoblox object of type 'obj_type'
@@ -430,6 +471,7 @@ class Connector(object):
         Args:
             ref      (str): Infoblox object reference
             payload (dict): Payload with data to send
+            return_fields (list): List of fields to return
         Returns:
             The object reference of the updated object
         Raises:
diff --git a/infoblox_client/exceptions.py b/infoblox_client/exceptions.py
index a5e219c..61610c6 100644
--- a/infoblox_client/exceptions.py
+++ b/infoblox_client/exceptions.py
@@ -42,6 +42,14 @@ class InfobloxSearchError(InfobloxException):
               "%(content)s [code %(code)s]"
 
 
+class InfobloxFileDownloadFailed(InfobloxException):
+    message = "Unable to download file from '%(url)s'"
+
+
+class InfobloxFileUploadFailed(InfobloxException):
+    message = "Unable to upload file to '%(url)s '%(content)s' '%(code)s'"
+
+
 class InfobloxCannotCreateObject(InfobloxException):
     message = "Cannot create '%(obj_type)s' object(s): " \
               "%(content)s [code %(code)s]"
diff --git a/infoblox_client/objects.py b/infoblox_client/objects.py
index 19cd765..02cd787 100644
--- a/infoblox_client/objects.py
+++ b/infoblox_client/objects.py
@@ -4286,6 +4286,12 @@ class Fileop(InfobloxObject):
     def uploadserviceaccount(self, *args, **kwargs):
         return self._call_func("uploadserviceaccount", *args, **kwargs)
 
+    def upload_file(self, *args, **kwargs):
+        return self.connector.upload_file(*args, **kwargs)
+
+    def download_file(self, *args, **kwargs):
+        return self.connector.download_file(*args, **kwargs)
+
 
 class Filterfingerprint(InfobloxObject):
     """ Filterfingerprint: DHCP Fingerprint Filter object.


Test patch:
diff --git a/tests/test_connector.py b/tests/test_connector.py
index e80a673..75cdaf4 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -14,6 +14,7 @@
 #    under the License.
 import unittest
 
+import os
 import mock
 import requests
 from mock import patch
@@ -36,6 +37,23 @@ class TestInfobloxConnector(unittest.TestCase):
         self.default_opts = self._prepare_options()
         self.connector = connector.Connector(self.default_opts)
 
+    @staticmethod
+    def _create_infoblox_csv():
+        # build data file for file upload test
+        file_data = [
+            'header-network,address,netmask,comment'
+            'network,10.10.10.0,255.255.255.0,test1',
+            'network,10.10.11.0,255.255.255.0,test2'
+        ]
+        with open('tests/ibx_networks.csv', 'w') as fh:
+            fh.write('\n'.join(file_data))
+            fh.close()
+
+    @staticmethod
+    def _delete_infoblox_csv():
+        if os.path.exists('tests/ibx_networks.csv'):
+            os.unlink('tests/ibx_networks.csv')
+
     @staticmethod
     def _prepare_options():
         opts = mock.Mock()
@@ -454,6 +472,63 @@ class TestInfobloxConnector(unittest.TestCase):
                 verify=self.default_opts.ssl_verify,
             )
 
+    def test_call_upload_file(self):
+        upload_file_path = '/http_direct_file_io/req_id-UPLOAD-0302163936014609/ibx_networks.csv'
+        upload_url = 'https://infoblox.example.org' + upload_file_path
+        self._create_infoblox_csv()
+        with open('tests/ibx_networks.csv', 'r') as fh:
+            data = fh.read()
+            fh.close()
+        payload = dict(file=data)
+        with patch.object(requests.Session, 'post',
+                          return_value=mock.Mock()) as patched_post:
+            self.connector.session.cookies = ['cookies']
+            patched_post.return_value.status_code = 200
+            patched_post.return_value.content = '{}'
+            self.connector.upload_file(upload_url, payload)
+            self.assertEqual(None, self.connector.session.auth)
+            self._delete_infoblox_csv()
+
+    def test_call_upload_file_with_error_403(self):
+        upload_file_path = '/http_direct_file_io/req_id-UPLOAD-0302163936014609/ibx_networks.csv'
+        upload_url = 'https://infoblox.example.org' + upload_file_path
+        self._create_infoblox_csv()
+        with open('tests/ibx_networks.csv', 'r') as fh:
+            data = fh.read()
+            fh.close()
+        payload = dict(file=data)
+        with patch.object(requests.Session, 'post',
+                          return_value=mock.Mock()) as patched_post:
+            patched_post.return_value.status_code = 403
+            patched_post.return_value.content = '{}'
+            self.assertRaises(exceptions.InfobloxFileUploadFailed,
+                              self.connector.upload_file,
+                              upload_url,
+                              payload)
+            self._delete_infoblox_csv()
+
+    def test_call_download_file(self):
+        download_file_path = '/http_direct_file_io/req_id-DOWNLOAD-0302163936014609/ibx_networks.csv'
+        download_url = 'https://infoblox.example.org' + download_file_path
+        with patch.object(requests.Session, 'get',
+                          return_value=mock.Mock()) as patched_get:
+            self.connector.session.cookies = ['cookies']
+            patched_get.return_value.status_code = 200
+            patched_get.return_value.content = '{}'
+            self.connector.download_file(download_url)
+            self.assertEqual(None, self.connector.session.auth)
+
+    def test_call_download_file_with_error_403(self):
+        download_file_path = '/http_direct_file_io/req_id-DOWNLOAD-0302163936014609/ibx_networks.csv'
+        download_url = 'https://infoblox.example.org' + download_file_path
+        with patch.object(requests.Session, 'get',
+                          return_value=mock.Mock()) as patched_get:
+            patched_get.return_value.status_code = 403
+            patched_get.return_value.content = '{}'
+            self.assertRaises(exceptions.InfobloxFileDownloadFailed,
+                              self.connector.download_file,
+                              download_url)
+
     def test_call_func_with_http_error(self):
         objtype = 'network'
         payload = {'ip': '0.0.0.0'}
diff --git a/tests/test_objects.py b/tests/test_objects.py
index 697622b..36e7b8e 100644
--- a/tests/test_objects.py
+++ b/tests/test_objects.py
@@ -14,6 +14,7 @@
 #    under the License.
 
 import unittest
+import os
 import copy
 import mock
 
@@ -59,6 +60,21 @@ DEFAULT_TXT_RECORD = {
 
 
 class TestObjects(unittest.TestCase):
+    @staticmethod
+    def _create_infoblox_csv():
+        file_data = [
+            'header-network,address,netmask,comment'
+            'network,10.10.10.0,255.255.255.0,test1',
+            'network,10.10.11.0,255.255.255.0,test2'
+        ]
+        with open('tests/ibx_networks.csv', 'w') as fh:
+            fh.write('\n'.join(file_data))
+            fh.close()
+
+    @staticmethod
+    def _delete_infoblox_csv():
+        if os.path.exists('tests/ibx_networks.csv'):
+            os.unlink('tests/ibx_networks.csv')
 
     def _mock_connector(self, get_object=None, create_object=None,
                         delete_object=None):
@@ -547,3 +563,21 @@ class TestObjects(unittest.TestCase):
              'text': 'hello_text',
              'view': 'my_dns_view',
             }, ['extattrs', 'name', 'text', 'view'])
+
+    def test_call_upload_file(self):
+        upload_file_path = '/http_direct_file_io/req_id-UPLOAD-0302163936014609/ibx_networks.csv'
+        upload_url = 'https://infoblox.example.org' + upload_file_path
+        self._create_infoblox_csv()
+        with open('tests/ibx_networks.csv', 'r') as fh:
+            data = fh.read()
+            fh.close()
+        payload = dict(file=data)
+        connector = self._mock_connector()
+        fo = objects.Fileop(connector)
+        result = fo.upload_file(upload_url, payload)
+        self.assertIsInstance(fo, objects.Fileop)
+        self.assertTrue(result)
+        # clean up and remove csv file
+        self._delete_infoblox_csv()
+
+