Repository: spotify/luigi

Issue:
Use of KerberosClient with webhdfs
Currently webhdfs uses InsecureClient by default. However, the hdfs package also has an KerberosClient which can be used to do authenticated requests. 

Our hdfs system uses Kerberos and I would like it if KerberosClient is added as an option. 

The KerberosClient does not need any additional settings (as far as I can test). And thus only an additional parameter is needed: `use_kerberos_client`. 

Are you willing to accept a pull request for this? 


Discussion:
I, personally, do not use the HDFS/Hadoop side of Luigi. But so long as your suggested PR doesn't change the default behavior, please feel free to submit for review!
> Are you willing to accept a pull request for this?

Yep, for sure.

Resolving patch:
diff --git a/luigi/contrib/hdfs/webhdfs_client.py b/luigi/contrib/hdfs/webhdfs_client.py
--- a/luigi/contrib/hdfs/webhdfs_client.py
+++ b/luigi/contrib/hdfs/webhdfs_client.py
@@ -43,6 +43,8 @@ class webhdfs(luigi.Config):
                               description='Port for webhdfs')
     user = luigi.Parameter(default='', description='Defaults to $USER envvar',
                            config_path=dict(section='hdfs', name='user'))
+    client_type = luigi.ChoiceParameter(var_type=str, choices=['insecure', 'kerberos'],
+                                        default='insecure', description='Type of hdfs client to use.')
 
 
 class WebHdfsClient(hdfs_abstract_client.HdfsFileSystem):
@@ -53,10 +55,11 @@ class WebHdfsClient(hdfs_abstract_client.HdfsFileSystem):
     <https://hdfscli.readthedocs.io/en/latest/api.html>`__.
     """
 
-    def __init__(self, host=None, port=None, user=None):
+    def __init__(self, host=None, port=None, user=None, client_type=None):
         self.host = host or hdfs_config.hdfs().namenode_host
         self.port = port or webhdfs().port
         self.user = user or webhdfs().user or os.environ['USER']
+        self.client_type = client_type or webhdfs().client_type
 
     @property
     def url(self):
@@ -73,8 +76,12 @@ def client(self):
         # not urgent to memoize it. Note that it *might* be issues with process
         # forking and whatnot (as the one in the snakebite client) if we
         # memoize it too trivially.
-        import hdfs
-        return hdfs.InsecureClient(url=self.url, user=self.user)
+        if self.client_type == 'kerberos':
+            from hdfs.ext.kerberos import KerberosClient
+            return KerberosClient(url=self.url)
+        else:
+            import hdfs
+            return hdfs.InsecureClient(url=self.url, user=self.user)
 
     def walk(self, path, depth=1):
         return self.client.walk(path, depth=depth)


Test patch:
diff --git a/test/contrib/hdfs/webhdfs_client_test.py b/test/contrib/hdfs/webhdfs_client_test.py
--- a/test/contrib/hdfs/webhdfs_client_test.py
+++ b/test/contrib/hdfs/webhdfs_client_test.py
@@ -14,7 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import unittest
 
+from hdfs import InsecureClient
+from hdfs.ext.kerberos import KerberosClient
 from nose.plugins.attrib import attr
 
 from helpers import with_config
@@ -44,3 +47,16 @@ def test_actually_using_webhdfs(self):
 
     # This one fails when run together with the whole test suite
     test_write_cleanup_no_close = None
+
+
+class TestWebHdfsClient(unittest.TestCase):
+
+    @with_config({'webhdfs': {'client_type': 'insecure'}})
+    def test_insecure_client_type(self):
+        client = WebHdfsClient(host='localhost').client
+        self.assertIsInstance(client, InsecureClient)
+
+    @with_config({'webhdfs': {'client_type': 'kerberos'}})
+    def test_kerberos_client_type(self):
+        client = WebHdfsClient(host='localhost').client
+        self.assertIsInstance(client, KerberosClient)