Repository: spotify/luigi

Issue:
Delay instantiating S3Client when generating S3Targets
I was profiling luigi code that had recursive dependencies (i.e., Task A grabs outputs from Task B which grabs outputs from Task C, etc) I noticed that there was a big performance hit when generating too many S3Targets (because boto has to hit the filesystem to check all config files and then a connection object [I believe] is generated to AWS).  This also means that you open a ton of file descriptors (so you can hit a ulimit if you fork too many times ).

I am happy to put up a PR implementing this change, but wanted to see if anybody had feedback on this before putting up a PR.

Rather than instantiating `S3Client` in the `__init__` method, I'd like to suggest changing `self.client` to a cached property that only instantiates the client at the moment we need it.

Pros:

- outputs() method is faster
- prereq for using a connection pool for communicating with S3
- caching S3Targets doesn't surprisingly leak file descriptors

Cons:

- validation errors on client kwargs only come out once you start trying to access targets, (currently you see it as soon as S3Target is instantiated)


Discussion:

Resolving patch:
diff --git a/luigi/contrib/s3.py b/luigi/contrib/s3.py
--- a/luigi/contrib/s3.py
+++ b/luigi/contrib/s3.py
@@ -73,15 +73,35 @@ class S3Client(FileSystem):
     boto-powered S3 client.
     """
 
+    _s3 = None
+
     def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                  **kwargs):
+        from boto.s3.key import Key
+        options = self._get_s3_config()
+        options.update(kwargs)
+        if aws_access_key_id:
+            options['aws_access_key_id'] = aws_access_key_id
+        if aws_secret_access_key:
+            options['aws_secret_access_key'] = aws_secret_access_key
+
+        self.Key = Key
+        self._options = options
+
+    @property
+    def s3(self):
         # only import boto when needed to allow top-lvl s3 module import
         import boto
         import boto.s3.connection
-        from boto.s3.key import Key
 
-        options = self._get_s3_config()
-        options.update(kwargs)
+        options = dict(self._options)
+
+        if self._s3:
+            return self._s3
+
+        aws_access_key_id = options.get('aws_access_key_id')
+        aws_secret_access_key = options.get('aws_secret_access_key')
+
         # Removing key args would break backwards compability
         role_arn = options.get('aws_role_arn')
         role_session_name = options.get('aws_role_session_name')
@@ -97,22 +117,18 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
             aws_access_key_id = assumed_role.credentials.access_key
             aws_session_token = assumed_role.credentials.session_token
 
-        else:
-            if not aws_access_key_id:
-                aws_access_key_id = options.get('aws_access_key_id')
-
-            if not aws_secret_access_key:
-                aws_secret_access_key = options.get('aws_secret_access_key')
-
         for key in ['aws_access_key_id', 'aws_secret_access_key', 'aws_role_session_name', 'aws_role_arn']:
             if key in options:
                 options.pop(key)
-
-        self.s3 = boto.s3.connection.S3Connection(aws_access_key_id,
-                                                  aws_secret_access_key,
-                                                  security_token=aws_session_token,
-                                                  **options)
-        self.Key = Key
+        self._s3 = boto.s3.connection.S3Connection(aws_access_key_id,
+                                                   aws_secret_access_key,
+                                                   security_token=aws_session_token,
+                                                   **options)
+        return self._s3
+
+    @s3.setter
+    def s3(self, value):
+        self._s3 = value
 
     def exists(self, path):
         """


Test patch: