Repository: iterative/dvc

Issue:
Unable to use IAM role to access S3
I'm trying to use the latest version of DVC on a linux EC2 production machine with an IAM role, but it appears that s3.py is hard-coded to use API keys (line 61 and 62). I can work around it, but IAM roles should be supported as they are an AWS security best practice.




Discussion:
Hi @mcheshier1 !

That's a great point! We will be sure to take a look at it shortly.

Thanks,
Ruslan

Resolving patch:
diff --git a/dvc/config.py b/dvc/config.py
--- a/dvc/config.py
+++ b/dvc/config.py
@@ -117,9 +117,9 @@ class Config(object):
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
     }
 
     # backward compatibility
@@ -150,9 +150,9 @@ class Config(object):
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
         Optional(SECTION_AWS_REGION): str,
-        Optional(SECTION_AWS_PROFILE, default='default'): str,
-        Optional(SECTION_AWS_CREDENTIALPATH, default=''): str,
-        Optional(SECTION_AWS_ENDPOINT_URL, default=None): str,
+        Optional(SECTION_AWS_PROFILE): str,
+        Optional(SECTION_AWS_CREDENTIALPATH): str,
+        Optional(SECTION_AWS_ENDPOINT_URL): str,
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
         Optional(SECTION_CACHE_PROTECTED,
diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py
--- a/dvc/remote/s3.py
+++ b/dvc/remote/s3.py
@@ -40,29 +40,29 @@ class RemoteS3(RemoteBase):
     PARAM_ETAG = 'etag'
 
     def __init__(self, project, config):
-        import configobj
-
         self.project = project
-        storagepath = 's3://' + config.get(Config.SECTION_AWS_STORAGEPATH,
-                                           '').lstrip('/')
+
+        storagepath = 's3://{}'.format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        )
+
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
-        self.region = config.get(Config.SECTION_AWS_REGION, None)
-        self.profile = os.getenv('AWS_PROFILE',
-                                 config.get(Config.SECTION_AWS_PROFILE,
-                                            'default'))
-        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL, None)
-
-        credentialpath = config.get(Config.SECTION_AWS_CREDENTIALPATH, None)
-        if credentialpath:
-            creds_conf = configobj.ConfigObj(credentialpath)
-            creds = creds_conf.get(self.profile, {})
-        else:
-            creds = self._get_aws_config_from_env()
-
-        self.creds = creds
-        self.region = creds.get('region', self.region)
-        self.aws_access_key_id = creds.get('aws_access_key_id', None)
-        self.aws_secret_access_key = creds.get('aws_secret_access_key', None)
+
+        self.region = (
+            os.environ.get('AWS_DEFAULT_REGION') or
+            config.get(Config.SECTION_AWS_REGION)
+        )
+
+        self.profile = (
+            os.environ.get('AWS_PROFILE') or
+            config.get(Config.SECTION_AWS_PROFILE)
+        )
+
+        self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
+
+        shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
+        if shared_creds:
+            os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
 
     @staticmethod
     def compat_config(config):
@@ -81,22 +81,10 @@ def prefix(self):
 
     @property
     def s3(self):
-        if not self.creds:
-            session = boto3.session.Session(profile_name=self.profile)
-        else:
-            session = boto3.session.Session(
-                             aws_access_key_id=self.aws_access_key_id,
-                             aws_secret_access_key=self.aws_secret_access_key,
-                             region_name=self.region)
-        return session.client('s3', endpoint_url=self.endpoint_url)
+        session = boto3.session.Session(profile_name=self.profile,
+                                        region_name=self.region)
 
-    def _get_aws_config_from_env(self):
-        creds = {
-            'region': os.getenv('AWS_DEFAULT_REGION'),
-            'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
-            'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
-        }
-        return {k: v for k, v in creds.items() if v is not None}
+        return session.client('s3', endpoint_url=self.endpoint_url)
 
     def get_etag(self, bucket, key):
         try:


Test patch: