Repository: ibis-project/ibis

Issue:
Using http for webhdfs not possible when kerberos is enabled 
The hdfs_connect function has no option to choose http or https protocol when connecting to webhdfs.
We are running CDH 5.5 with kerberos but without ssl enabled for webhdfs.

Could you maybe add an option to signal the use of http when using kerberos?

code snippet.

the hdfs_connect function code in ibis/ibis/**init**.py always uses https when auth_mechanism in ['GSSAPI', 'LDAP']

example:

def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
                 auth_mechanism='NOSASL', verify=True, use_https=True, **kwds):

if auth_mechanism in ['GSSAPI', 'LDAP']:
        try:
            import requests_kerberos
        except ImportError:
            raise IbisError(
                "Unable to import requests-kerberos, which is required for "
                "Kerberos HDFS support. Install it by executing `pip install "
                "requests-kerberos` or `pip install hdfs[kerberos]`.")
        from hdfs.ext.kerberos import KerberosClient
        if use_https:
            url = 'https://{0}:{1}'.format(host, port) # use SSL
        else:
            url = 'http://{0}:{1}'.format(host, port) # no SSL
        kwds.setdefault('mutual_auth', 'OPTIONAL')
        hdfs_client = KerberosClient(url, *_kwds)
    else:
        from hdfs.client import InsecureClient
        url = 'http://{0}:{1}'.format(host, port)
        hdfs_client = InsecureClient(url, *_kwds)



Discussion:
Sure thing. I'll take a look when I can


Resolving patch:
diff --git a/ibis/__init__.py b/ibis/__init__.py
--- a/ibis/__init__.py
+++ b/ibis/__init__.py
@@ -71,7 +71,8 @@ def make_client(db, hdfs_client=None):
 
 
 def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
-                 auth_mechanism='NOSASL', verify=True, **kwds):
+                 use_https='default', auth_mechanism='NOSASL',
+                 verify=True, **kwds):
     """
     Connect to HDFS
 
@@ -80,6 +81,9 @@ def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
     host : string, Host name of the HDFS NameNode
     port : int, NameNode's WebHDFS port (default 50070)
     protocol : {'webhdfs'}
+    use_https : boolean, default 'default'
+        Connect to WebHDFS with HTTPS, otherwise plain HTTP. For secure
+        authentication, the default for this is True, otherwise False
     auth_mechanism : string, Set to NOSASL or PLAIN for non-secure clusters.
         Set to GSSAPI or LDAP for Kerberos-secured clusters.
     verify : boolean, Set to False to turn off verifying SSL certificates.
@@ -95,6 +99,10 @@ def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
     session = kwds.setdefault('session', requests.Session())
     session.verify = verify
     if auth_mechanism in ['GSSAPI', 'LDAP']:
+        if use_https == 'default':
+            prefix = 'https'
+        else:
+            prefix = 'https' if use_https else 'http'
         try:
             import requests_kerberos
         except ImportError:
@@ -103,12 +111,17 @@ def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
                 "Kerberos HDFS support. Install it by executing `pip install "
                 "requests-kerberos` or `pip install hdfs[kerberos]`.")
         from hdfs.ext.kerberos import KerberosClient
-        url = 'https://{0}:{1}'.format(host, port) # note SSL
+        # note SSL
+        url = '{0}://{1}:{2}'.format(prefix, host, port)
         kwds.setdefault('mutual_auth', 'OPTIONAL')
         hdfs_client = KerberosClient(url, **kwds)
     else:
+        if use_https == 'default':
+            prefix = 'http'
+        else:
+            prefix = 'https' if use_https else 'http'
         from hdfs.client import InsecureClient
-        url = 'http://{0}:{1}'.format(host, port)
+        url = '{0}://{1}:{2}'.format(prefix, host, port)
         hdfs_client = InsecureClient(url, **kwds)
     return WebHDFS(hdfs_client)
 
diff --git a/ibis/impala/api.py b/ibis/impala/api.py
--- a/ibis/impala/api.py
+++ b/ibis/impala/api.py
@@ -46,9 +46,9 @@ def verify(expr):
 
 
 def connect(host='localhost', port=21050, database='default', timeout=45,
-            use_ssl=False, ca_cert=None, user=None, password=None,
-            auth_mechanism='NOSASL', kerberos_service_name='impala',
-            pool_size=8, hdfs_client=None):
+            use_ssl=False, ca_cert=None, user=None,
+            password=None, auth_mechanism='NOSASL',
+            kerberos_service_name='impala', pool_size=8, hdfs_client=None):
     """
     Create an ImpalaClient for use with Ibis.
 


Test patch: