Repository: conda/conda

Issue:
add boto3 to conda's s3 adapter



Discussion:

Resolving patch:
diff --git a/conda/gateways/adapters/s3.py b/conda/gateways/adapters/s3.py
--- a/conda/gateways/adapters/s3.py
+++ b/conda/gateways/adapters/s3.py
@@ -1,13 +1,16 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import json
 from logging import getLogger
+from tempfile import SpooledTemporaryFile, mkstemp
+
 from requests import Response
 from requests.adapters import BaseAdapter
 from requests.structures import CaseInsensitiveDict
-from tempfile import mkstemp
 
 from ..disk.delete import rm_rf
+from ...common.compat import ensure_binary
 from ...common.url import url_to_s3_info
 
 log = getLogger(__name__)
@@ -21,21 +24,71 @@ def __init__(self):
         self._temp_file = None
 
     def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None):
-
         resp = Response()
         resp.status_code = 200
         resp.url = request.url
 
         try:
-            import boto
+            import boto3
+            return self._send_boto3(boto3, resp, request)
         except ImportError:
-            stderrlog.info('\nError: boto is required for S3 channels. '
-                           'Please install it with `conda install boto`\n'
-                           'Make sure to run `source deactivate` if you '
-                           'are in a conda environment.\n')
+            try:
+                import boto
+                return self._send_boto(boto, resp, request)
+            except ImportError:
+                stderrlog.info('\nError: boto3 is required for S3 channels. '
+                               'Please install with `conda install boto3`\n'
+                               'Make sure to run `source deactivate` if you '
+                               'are in a conda environment.\n')
+                resp.status_code = 404
+                return resp
+
+    def close(self):
+        if self._temp_file:
+            rm_rf(self._temp_file)
+
+    def _send_boto3(self, boto3, resp, request):
+        from botocore.exceptions import BotoCoreError, ClientError
+        bucket_name, key_string = url_to_s3_info(request.url)
+
+        # Get the key without validation that it exists and that we have
+        # permissions to list its contents.
+        key = boto3.resource('s3').Object(bucket_name, key_string[1:])
+
+        try:
+            response = key.get()
+        except (BotoCoreError, ClientError) as exc:
+            # This exception will occur if the bucket does not exist or if the
+            # user does not have permission to list its contents.
             resp.status_code = 404
+            message = {
+                "error": "error downloading file from s3",
+                "path": request.url,
+                "exception": repr(exc),
+            }
+            fh = SpooledTemporaryFile()
+            fh.write(ensure_binary(json.dumps(message)))
+            fh.seek(0)
+            resp.raw = fh
+            resp.close = resp.raw.close
             return resp
 
+        key_headers = response['ResponseMetadata']['HTTPHeaders']
+        resp.headers = CaseInsensitiveDict({
+            "Content-Type": key_headers.get('content-type', "text/plain"),
+            "Content-Length": key_headers['content-length'],
+            "Last-Modified": key_headers['last-modified'],
+        })
+
+        f = SpooledTemporaryFile()
+        key.download_fileobj(f)
+        f.seek(0)
+        resp.raw = f
+        resp.close = resp.raw.close
+
+        return resp
+
+    def _send_boto(self, boto, resp, request):
         conn = boto.connect_s3()
 
         bucket_name, key_string = url_to_s3_info(request.url)
@@ -69,9 +122,3 @@ def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxi
             resp.close = resp.raw.close
         else:
             resp.status_code = 404
-
-        return resp
-
-    def close(self):
-        if self._temp_file:
-            rm_rf(self._temp_file)


Test patch: