Repository: spotify/luigi
File: luigi/contrib/hive.py (Python)

Code before:
    def __init__(self, hdfs_client=None, warehouse_location=None):
        self.hdfs_client = hdfs_client or get_autoconfig_client()
        self.warehouse_location = warehouse_location or get_hive_warehouse_location()

    def table_schema(self, table, database='default'):
        return NotImplemented

    def table_location(self, table, database='default', partition=None):
        return os.path.join(
            self.warehouse_location,
            database + '.db',
            table,
            self.partition_spec(partition)
        )

    def table_exists(self, table, database='default', partition=None):
        """
        The table/partition is considered existing if corresponding path in hdfs exists
        and contains file except those which match pattern set in  `ignored_file_masks`
        """
        path = self.table_location(table, database, partition)
        if self.hdfs_client.exists(path):
            ignored_files = get_ignored_file_masks()
            filenames = self.hdfs_client.listdir(path)
            if ignored_files is None:
                return True

            pattern = re.compile(ignored_files)
            for filename in filenames:
                if not pattern.match(filename):
                    return True

        return False

    def partition_spec(self, partition):
        _validate_partition(partition)
        return '/'.join([
            '{}={}'.format(k, v) for (k, v) in six.iteritems(partition or {})
        ])


def get_default_client():
    syntax = get_hive_syntax()
    if syntax == "apache":
        return ApacheHiveCommandClient()
    elif syntax == "metastore":
        return MetastoreClient()
    elif syntax == 'warehouse':
        return WarehouseHiveClient()
    else:
        return HiveCommandClient()

Review comment (suggestion):
Is it possible to check ignored_files before calling listdir? It will be a little faster.
I am not familiar with hive warehouse, so I do not know is it needed to call listdir for existence.

Code after:
    def __init__(self, hdfs_client=None, warehouse_location=None):
        self.hdfs_client = hdfs_client or get_autoconfig_client()
        self.warehouse_location = warehouse_location or get_hive_warehouse_location()

    def table_schema(self, table, database='default'):
        return NotImplemented

    def table_location(self, table, database='default', partition=None):
        return os.path.join(
            self.warehouse_location,
            database + '.db',
            table,
            self.partition_spec(partition)
        )

    def table_exists(self, table, database='default', partition=None):
        """
        The table/partition is considered existing if corresponding path in hdfs exists
        and contains file except those which match pattern set in  `ignored_file_masks`
        """
        path = self.table_location(table, database, partition)
        if self.hdfs_client.exists(path):
            ignored_files = get_ignored_file_masks()
            if ignored_files is None:
                return True

            filenames = self.hdfs_client.listdir(path)
            pattern = re.compile(ignored_files)
            for filename in filenames:
                if not pattern.match(filename):
                    return True

        return False

    def partition_spec(self, partition):
        _validate_partition(partition)
        return '/'.join([
            '{}={}'.format(k, v) for (k, v) in six.iteritems(partition or {})
        ])


def get_default_client():
    syntax = get_hive_syntax()
    if syntax == "apache":
        return ApacheHiveCommandClient()
    elif syntax == "metastore":
        return MetastoreClient()
    elif syntax == 'warehouse':
        return WarehouseHiveClient()
    else:
        return HiveCommandClient()