Repository: dask/dask
File: dask/dataframe/io/parquet/arrow.py (Python)

Code before:
        if filename:
            result[fs.sep.join([path, filename])] += 1
        else:
            return None  # File path is missing, abort
    return tuple(result.values())


def _merge_statistics(stats, s):
    """ Update `stats` with vaules in `s`
    """
    stats[-1]["total_byte_size"] += s["total_byte_size"]
    stats[-1]["num-rows"] += s["num-rows"]
    ncols = len(stats[-1]["columns"])
    ncols_n = len(s["columns"])
    if ncols != ncols_n:
        raise ValueError(f"Column count not equal ({ncols} vs {ncols_n})")
    for i in range(ncols):
        name = stats[-1]["columns"][i]["name"]
        j = i
        for ii in range(ncols):
            if name == s["columns"][j]["name"]:
                break
            if ii == ncols - 1:
                raise KeyError(f"Column statistics missing for {name}")
            j = (j + 1) % ncols

        min_n = s["columns"][j]["min"]
        max_n = s["columns"][j]["max"]
        null_count_n = s["columns"][j]["null_count"]

        min_i = stats[-1]["columns"][i]["min"]
        max_i = stats[-1]["columns"][i]["max"]
        stats[-1]["columns"][i]["min"] = min(min_i, min_n)
        stats[-1]["columns"][i]["max"] = min(max_i, max_n)
        stats[-1]["columns"][i]["null_count"] += null_count_n
    return True


def _determine_dataset_parts(fs, paths, gather_statistics, filters, dataset_kwargs):
    """ Determine how to access metadata and break read into ``parts``

    This logic is mostly to handle `gather_statistics=False` cases,
    because this also means we should avoid scanning every file in the
    dataset.
    """
    parts = []
    if len(paths) > 1:
        if gather_statistics is not False:
            # This scans all the files
            dataset = pq.ParquetDataset(
                paths, filesystem=fs, filters=filters, **dataset_kwargs

Review comment (suggestion):
It has recently been brought to my attention that max/min are [deprecated](https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L199) and the recommendation is to use max/min_value, which are the same except maybe reversed if the column_ordering value says so. I don't know why they chose this duplication! In any case, the code should probably look for the min/max_value first and fall back to min/max.

Code after:
            result[fs.sep.join([path, filename])] += 1
        else:
            return None  # File path is missing, abort
    return tuple(result.values())


def _merge_statistics(stats, s):
    """ Update `stats` with vaules in `s`
    """
    stats[-1]["total_byte_size"] += s["total_byte_size"]
    stats[-1]["num-rows"] += s["num-rows"]
    ncols = len(stats[-1]["columns"])
    ncols_n = len(s["columns"])
    if ncols != ncols_n:
        raise ValueError(f"Column count not equal ({ncols} vs {ncols_n})")
    for i in range(ncols):
        name = stats[-1]["columns"][i]["name"]
        j = i
        for ii in range(ncols):
            if name == s["columns"][j]["name"]:
                break
            if ii == ncols - 1:
                raise KeyError(f"Column statistics missing for {name}")
            j = (j + 1) % ncols

        min_n = s["columns"][j]["min"]
        max_n = s["columns"][j]["max"]
        null_count_n = s["columns"][j]["null_count"]

        min_i = stats[-1]["columns"][i]["min"]
        max_i = stats[-1]["columns"][i]["max"]
        stats[-1]["columns"][i]["min"] = min(min_i, min_n)
        stats[-1]["columns"][i]["max"] = min(max_i, max_n)
        stats[-1]["columns"][i]["null_count"] += null_count_n
    return True


def _determine_dataset_parts(fs, paths, gather_statistics, filters, dataset_kwargs):
    """ Determine how to access metadata and break read into ``parts``

    This logic is mostly to handle `gather_statistics=False` cases,
    because this also means we should avoid scanning every file in the
    dataset.
    """
    parts = []
    if len(paths) > 1:
        if gather_statistics is not False:
            # This scans all the files
            dataset = pq.ParquetDataset(
                paths, filesystem=fs, filters=filters, **dataset_kwargs
            )