Repository: spotify/luigi

Issue:
"missing boto" warning for hadoop import
Currently when the `luigi.contrib.hadoop` module is imported we get a WARNING about s3 being imported without boto module available  (also manifests as "No handler for logger" errors if hadoop imported outside the task-running interface context).

I assume plenty of folks are using hadoop without s3, so we should get rid of the warning somehow.



Discussion:
great if you can send a PR resolving this issue :)


Resolving patch:
diff --git a/luigi/s3.py b/luigi/s3.py
--- a/luigi/s3.py
+++ b/luigi/s3.py
@@ -16,7 +16,8 @@
 #
 """
 Implementation of Simple Storage Service support.
-:py:class:`S3Target` is a subclass of the Target class to support S3 file system operations
+:py:class:`S3Target` is a subclass of the Target class to support S3 file
+system operations. The `boto` library is required to use S3 targets.
 """
 
 from __future__ import division
@@ -46,13 +47,6 @@
 
 logger = logging.getLogger('luigi-interface')
 
-try:
-    import boto
-    from boto.s3.key import Key
-except ImportError:
-    logger.warning("Loading s3 module without boto installed. Will crash at "
-                   "runtime if s3 functionality is used.")
-
 
 # two different ways of marking a directory
 # with a suffix in S3
@@ -75,6 +69,10 @@ class S3Client(FileSystem):
 
     def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                  **kwargs):
+        # only import boto when needed to allow top-lvl s3 module import
+        import boto
+        from boto.s3.key import Key
+
         options = self._get_s3_config()
         options.update(kwargs)
         # Removing key args would break backwards compability
@@ -88,6 +86,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
         self.s3 = boto.connect_s3(aws_access_key_id,
                                   aws_secret_access_key,
                                   **options)
+        self.Key = Key
 
     def exists(self, path):
         """
@@ -177,7 +176,7 @@ def put(self, local_path, destination_s3_path, **kwargs):
         s3_bucket = self.s3.get_bucket(bucket, validate=True)
 
         # put the file
-        s3_key = Key(s3_bucket)
+        s3_key = self.Key(s3_bucket)
         s3_key.key = key
         s3_key.set_contents_from_filename(local_path, **kwargs)
 
@@ -192,7 +191,7 @@ def put_string(self, content, destination_s3_path, **kwargs):
         s3_bucket = self.s3.get_bucket(bucket, validate=True)
 
         # put the content
-        s3_key = Key(s3_bucket)
+        s3_key = self.Key(s3_bucket)
         s3_key.key = key
         s3_key.set_contents_from_string(content, **kwargs)
 
@@ -261,7 +260,7 @@ def get(self, s3_path, destination_local_path):
         s3_bucket = self.s3.get_bucket(bucket, validate=True)
 
         # download the file
-        s3_key = Key(s3_bucket)
+        s3_key = self.Key(s3_bucket)
         s3_key.key = key
         s3_key.get_contents_to_filename(destination_local_path)
 
@@ -275,7 +274,7 @@ def get_as_string(self, s3_path):
         s3_bucket = self.s3.get_bucket(bucket, validate=True)
 
         # get the content
-        s3_key = Key(s3_bucket)
+        s3_key = self.Key(s3_bucket)
         s3_key.key = key
         contents = s3_key.get_contents_as_string()
 


Test patch: