Repository: huggingface/datasets
File: src/datasets/table.py (Python)

Code before:
    # To more efficiently read big data from disk, we can memory map the file,
    # so that Arrow can directly reference the data mapped from disk and avoid
    # having to allocate its own memory. In such case the operating system will
    # be able to page in the mapped memory lazily and page it out without any
    # write back cost when under pressure, allowing to more easily read arrays
    # bigger than the total memory.
    return pa.memory_map(filename)


def arrow_table_schema_from_file(filename: str) -> pa.Schema:
    """
    Infer arrow table schema from file without loading whole file into memory.
    Usefull especially while having very big files.
    """
    with _memory_mapped_arrow_stream_from_file(filename) as memory_mapped_stream:
        schema = pa.ipc.open_stream(memory_mapped_stream).schema
    return schema


def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table:
    with _memory_mapped_arrow_stream_from_file(filename) as memory_mapped_stream:
        pa_table = pa.ipc.open_stream(memory_mapped_stream).read_all()
    return pa_table


def arrow_table_batches_from_file(filename: str, max_chunksize: Optional[int] = None):
    pa_table = _memory_mapped_arrow_table_from_file(filename)
    return pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER)


def _write_table_to_file(table: pa.Table, filename: str) -> int:
    with open(filename, "wb") as sink:
        writer = pa.RecordBatchStreamWriter(sink=sink, schema=table.schema)
        batches: List[pa.RecordBatch] = table.to_batches()
        for batch in batches:
            writer.write_batch(batch)
        writer.close()
        return sum(batch.nbytes for batch in batches)


def _deepcopy(x, memo: dict):
    """deepcopy a regular class instance"""
    cls = x.__class__
    result = cls.__new__(cls)
    memo[id(x)] = result
    for k, v in x.__dict__.items():
        setattr(result, k, copy.deepcopy(v, memo))
    return result


def _interpolation_search(arr: List[int], x: int) -> int:

Review comment (performance):
for consistency with the other loading functions, you can name it _memory_mapped_record_batch_reader_from_file

Code after:
    """
    Infer arrow table schema from file without loading whole file into memory.
    Usefull especially while having very big files.
    """
    with pa.memory_map(filename) as memory_mapped_stream:
        schema = pa.ipc.open_stream(memory_mapped_stream).schema
    return schema


def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table:
    memory_mapped_stream = pa.memory_map(filename)
    opened_stream = pa.ipc.open_stream(memory_mapped_stream)
    pa_table = opened_stream.read_all()
    return pa_table


def _memory_mapped_record_batch_reader_from_file(filename: str, max_chunksize: Optional[int] = None):
    pa_table = _memory_mapped_arrow_table_from_file(filename)
    return pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER)


def _write_table_to_file(table: pa.Table, filename: str) -> int:
    with open(filename, "wb") as sink:
        writer = pa.RecordBatchStreamWriter(sink=sink, schema=table.schema)
        batches: List[pa.RecordBatch] = table.to_batches()
        for batch in batches:
            writer.write_batch(batch)
        writer.close()
        return sum(batch.nbytes for batch in batches)


def _deepcopy(x, memo: dict):
    """deepcopy a regular class instance"""
    cls = x.__class__
    result = cls.__new__(cls)
    memo[id(x)] = result
    for k, v in x.__dict__.items():
        setattr(result, k, copy.deepcopy(v, memo))
    return result


def _interpolation_search(arr: List[int], x: int) -> int:
    """
    Return the position i of a sorted array so that arr[i] <= x < arr[i+1]

    Args:
        arr (`List[int]`): non-empty sorted list of integers
        x (`int`): query

    Returns:
        `int`: the position i so that arr[i] <= x < arr[i+1]