import copy
import os
import tempfile
from functools import wraps
from itertools import groupby
from typing import List, Optional, Tuple, TypeVar, Union
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
from . import config
from .utils.logging import get_logger
logger = get_logger(__name__)
def inject_arrow_table_documentation(arrow_table_method):
def wrapper(method):
out = wraps(arrow_table_method)(method)
out.__doc__ = out.__doc__.replace("pyarrow.Table", "Table")
return out
return wrapper
def _in_memory_arrow_table_from_file(filename: str) -> pa.Table:
in_memory_stream = pa.input_stream(filename)
opened_stream = pa.ipc.open_stream(in_memory_stream)
pa_table = opened_stream.read_all()
return pa_table
def _in_memory_arrow_table_from_buffer(buffer: pa.Buffer) -> pa.Table:
stream = pa.BufferReader(buffer)
opened_stream = pa.ipc.open_stream(stream)
table = opened_stream.read_all()
return table
def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table:
memory_mapped_stream = pa.memory_map(filename)
opened_stream = pa.ipc.open_stream(memory_mapped_stream)
pa_table = opened_stream.read_all()
return pa_table
def _write_table_to_file(table: pa.Table, filename: str) -> int:
with open(filename, "wb") as sink:
writer = pa.RecordBatchStreamWriter(sink=sink, schema=table.schema)
batches: List[pa.RecordBatch] = table.to_batches()
for batch in batches:
writer.write_batch(batch)
writer.close()
return sum(batch.nbytes for batch in batches)
def _deepcopy(x, memo: dict):
"""deepcopy a regular class instance"""
cls = x.__class__
result = cls.__new__(cls)
memo[id(x)] = result
for k, v in x.__dict__.items():
setattr(result, k, copy.deepcopy(v, memo))
return result
def _interpolation_search(arr: List[int], x: int) -> int:
"""
Return the position i of a sorted array so that arr[i] <= x < arr[i+1]
Args:
arr (:obj:`List[int]`): non-empty sorted list of integers
x (:obj:`int`): query
Returns:
`int`: the position i so that arr[i] <= x < arr[i+1]
Raises:
`IndexError`: if the array is empty or if the query is outside the array values
"""
i, j = 0, len(arr) - 1
while i < j and arr[i] <= x < arr[j]:
k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i]))
if arr[k] <= x < arr[k + 1]:
return k
elif arr[k] < x:
i, j = k + 1, j
else:
i, j = i, k
raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.")
class IndexedTableMixin:
def __init__(self, table: pa.Table):
self._schema = table.schema
self._batches = [recordbatch for recordbatch in table.to_batches() if len(recordbatch) > 0]
self._offsets: np.ndarray = np.cumsum([0] + [len(b) for b in self._batches], dtype=np.int64)
def fast_gather(self, indices: Union[List[int], np.ndarray]) -> pa.Table:
"""
Create a pa.Table by gathering the records at the records at the specified indices. Should be faster
than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute
the binary searches in parallel, highly optimized C
"""
assert len(indices), "Indices must be non-empty"
batch_indices = np.searchsorted(self._offsets, indices, side="right") - 1
return pa.Table.from_batches(
[
self._batches[batch_idx].slice(i - self._offsets[batch_idx], 1)
for batch_idx, i in zip(batch_indices, indices)
],
schema=self._schema,
)
def fast_slice(self, offset=0, length=None) -> pa.Table:
"""
Slice the Table using interpolation search.
The behavior is the same as :obj:`pyarrow.Table.slice` but it's significantly faster.
Interpolation search is used to find the start and end indexes of the batches we want to keep.
The batches to keep are then concatenated to form the sliced Table.
"""
if offset < 0:
raise IndexError("Offset must be non-negative")
elif offset >= self._offsets[-1] or (length is not None and length <= 0):
return pa.Table.from_batches([], schema=self._schema)
i = _interpolation_search(self._offsets, offset)
if length is None or length + offset >= self._offsets[-1]:
batches = self._batches[i:]
batches[0] = batches[0].slice(offset - self._offsets[i])
else:
j = _interpolation_search(self._offsets, offset + length - 1)
batches = self._batches[i : j + 1]
batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j])
batches[0] = batches[0].slice(offset - self._offsets[i])
return pa.Table.from_batches(batches, schema=self._schema)
[docs]class Table(IndexedTableMixin):
"""
Wraps a pyarrow Table by using composition.
This is the base class for InMemoryTable, MemoryMappedTable and ConcatenationTable.
It implements all the basic attributes/methods of the pyarrow Table class except
the Table transforms: slice, filter, flatten, combine_chunks, cast, add_column,
append_column, remove_column, set_column, rename_columns and drop.
The implementation of these methods differs for the subclasses.
"""
def __init__(self, table: pa.Table):
super().__init__(table)
self.table = table
def __deepcopy__(self, memo: dict):
# arrow tables are immutable, so there's no need to copy self.table
# moreover calling deepcopy on a pyarrow table seems to make pa.total_allocated_bytes() decrease for some reason
# by adding it to the memo, self.table won't be copied
memo[id(self.table)] = self.table
# same for the recordbatches used by the index
memo[id(self._batches)] = list(self._batches)
return _deepcopy(self, memo)
def __getstate__(self):
# We can't pickle objects that are bigger than 4GiB, or it causes OverflowError
# So we write the table on disk instead
if self.table.nbytes >= config.MAX_TABLE_NBYTES_FOR_PICKLING:
table = self.table
with tempfile.NamedTemporaryFile("wb", delete=False, suffix=".arrow") as tmp_file:
filename = tmp_file.name
logger.debug(
f"Attempting to pickle a table bigger than 4GiB. Writing it on the disk instead at {filename}"
)
_write_table_to_file(table=table, filename=filename)
return {"path": filename}
else:
return {"table": self.table}
def __setstate__(self, state):
if "path" in state:
filename = state["path"]
logger.debug(f"Unpickling a big table from the disk at {filename}")
table = _in_memory_arrow_table_from_file(filename)
logger.debug(f"Removing temporary table file at {filename}")
os.remove(filename)
else:
table = state["table"]
Table.__init__(self, table)
[docs] @inject_arrow_table_documentation(pa.Table.validate)
def validate(self, *args, **kwargs):
return self.table.validate(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.equals)
def equals(self, *args, **kwargs):
args = tuple(arg.table if isinstance(arg, Table) else arg for arg in args)
kwargs = {k: v.table if isinstance(v, Table) else v for k, v in kwargs}
return self.table.equals(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.to_batches)
def to_batches(self, *args, **kwargs):
return self.table.to_batches(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.to_pydict)
def to_pydict(self, *args, **kwargs):
return self.table.to_pydict(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.to_pandas)
def to_pandas(self, *args, **kwargs):
return self.table.to_pandas(*args, **kwargs)
def to_string(self, *args, **kwargs):
return self.table.to_string(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.field)
def field(self, *args, **kwargs):
return self.table.field(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.column)
def column(self, *args, **kwargs):
return self.table.column(*args, **kwargs)
[docs] @inject_arrow_table_documentation(pa.Table.itercolumns)
def itercolumns(self, *args, **kwargs):
return self.table.itercolumns(*args, **kwargs)
@property
def schema(self):
return self.table.schema
@property
def columns(self):
return self.table.columns
@property
def num_columns(self):
return self.table.num_columns
@property
def num_rows(self):
return self.table.num_rows
@property
def shape(self):
return self.table.shape
@property
def nbytes(self):
return self.table.nbytes
@property
def column_names(self):
return self.table.column_names
def __eq__(self, other):
return self.equals(other)
def __getitem__(self, i):
return self.table[i]
def __len__(self):
return len(self.table)
def __repr__(self):
return self.table.__repr__().replace("pyarrow.Table", self.__class__.__name__)
def __str__(self):
return self.table.__str__().replace("pyarrow.Table", self.__class__.__name__)
@inject_arrow_table_documentation(pa.Table.slice)
def slice(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.filter)
def filter(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.flatten)
def flatten(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.combine_chunks)
def combine_chunks(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.cast)
def cast(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.add_column)
def add_column(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.append_column)
def append_column(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.remove_column)
def remove_column(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.set_column)
def set_column(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.rename_columns)
def rename_columns(self, *args, **kwargs):
raise NotImplementedError()
@inject_arrow_table_documentation(pa.Table.drop)
def drop(self, *args, **kwargs):
raise NotImplementedError()
class TableBlock(Table):
"""
TableBlock is the allowed class inside a ConcanetationTable.
Only MemoryMappedTable and InMemoryTable are TableBlock.
This is because we don't want a ConcanetationTable made out of other ConcanetationTables.
"""
pass
[docs]class InMemoryTable(TableBlock):
"""
The table is said in-memory when it is loaded into the user's RAM.
Pickling it does copy all the data using memory.
Its implementation is simple and uses the underlying pyarrow Table methods directly.
This is different from the MemoryMapped table, for which pickling doesn't copy all the
data in memory. For a MemoryMapped, unpickling instead reloads the table from the disk.
InMemoryTable must be used when data fit in memory, while MemoryMapped are reserved for
data bigger than memory or when you want the memory footprint of your application to
stay low.
"""
@classmethod
def from_file(cls, filename: str):
table = _in_memory_arrow_table_from_file(filename)
return cls(table)
@classmethod
def from_buffer(cls, buffer: pa.Buffer):
table = _in_memory_arrow_table_from_buffer(buffer)
return cls(table)
@inject_arrow_table_documentation(pa.Table.from_pandas)
@classmethod
def from_pandas(cls, *args, **kwargs):
return cls(pa.Table.from_pandas(*args, **kwargs))
@inject_arrow_table_documentation(pa.Table.from_arrays)
@classmethod
def from_arrays(cls, *args, **kwargs):
return cls(pa.Table.from_arrays(*args, **kwargs))
@inject_arrow_table_documentation(pa.Table.from_pydict)
@classmethod
def from_pydict(cls, *args, **kwargs):
return cls(pa.Table.from_pydict(*args, **kwargs))
@inject_arrow_table_documentation(pa.Table.from_batches)
@classmethod
def from_batches(cls, *args, **kwargs):
return cls(pa.Table.from_batches(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.slice)
def slice(self, offset=0, length=None):
# Use fast slicing here
return InMemoryTable(self.fast_slice(offset=offset, length=length))
[docs] @inject_arrow_table_documentation(pa.Table.filter)
def filter(self, *args, **kwargs):
return InMemoryTable(self.table.filter(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.flatten)
def flatten(self, *args, **kwargs):
return InMemoryTable(self.table.flatten(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.combine_chunks)
def combine_chunks(self, *args, **kwargs):
return InMemoryTable(self.table.combine_chunks(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.cast)
def cast(self, *args, **kwargs):
return InMemoryTable(self.table.cast(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.add_column)
def add_column(self, *args, **kwargs):
return InMemoryTable(self.table.add_column(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.append_column)
def append_column(self, *args, **kwargs):
return InMemoryTable(self.table.append_column(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.remove_column)
def remove_column(self, *args, **kwargs):
return InMemoryTable(self.table.remove_column(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.set_column)
def set_column(self, *args, **kwargs):
return InMemoryTable(self.table.set_column(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.rename_columns)
def rename_columns(self, *args, **kwargs):
return InMemoryTable(self.table.rename_columns(*args, **kwargs))
[docs] @inject_arrow_table_documentation(pa.Table.drop)
def drop(self, *args, **kwargs):
return InMemoryTable(self.table.drop(*args, **kwargs))
# The MemoryMappedTable needs replays to properly reload tables from the disk
Replay = Tuple[str, tuple, dict]
[docs]class MemoryMappedTable(TableBlock):
"""
The table is said memory mapped when it doesn't use the user's RAM but loads the data
from the disk instead.
Pickling it doesn't copy the data into memory.
Instead, only the path to the memory mapped arrow file is pickled, as well as the list
of transforms to "replay" when reloading the table from the disk.
Its implementation requires to store an history of all the transforms that were applied
to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table
from the disk.
This is different from the InMemoryTable table, for which pickling does copy all the
data in memory.
InMemoryTable must be used when data fit in memory, while MemoryMapped are reserved for
data bigger than memory or when you want the memory footprint of your application to
stay low.
"""
def __init__(self, table: pa.Table, path: str, replays: Optional[List[Replay]] = None):
super().__init__(table)
self.path = path
self.replays: List[Replay] = replays if replays is not None else []
@classmethod
def from_file(cls, filename: str, replays=None):
table = _memory_mapped_arrow_table_from_file(filename)
table = cls._apply_replays(table, replays)
return cls(table, filename, replays)
def __getstate__(self):
return {"path": self.path, "replays": self.replays}
def __setstate__(self, state):
path = state["path"]
replays = state["replays"]
table = _memory_mapped_arrow_table_from_file(path)
table = self._apply_replays(table, replays)
MemoryMappedTable.__init__(self, table, path=path, replays=replays)
@staticmethod
def _apply_replays(table: pa.Table, replays: Optional[List[Replay]] = None) -> pa.Table:
if replays is not None:
for name, args, kwargs in replays:
table = getattr(table, name)(*args, **kwargs)
return table
def _append_replay(self, replay: Replay) -> List[Replay]:
replays = copy.deepcopy(self.replays)
replays.append(replay)
return replays
[docs] @inject_arrow_table_documentation(pa.Table.slice)
def slice(self, offset=0, length=None):
replay = ("slice", (offset, length), {})
replays = self._append_replay(replay)
# Use fast slicing here
return MemoryMappedTable(self.fast_slice(offset=offset, length=length), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.filter)
def filter(self, *args, **kwargs):
replay = ("filter", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.filter(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.flatten)
def flatten(self, *args, **kwargs):
replay = ("flatten", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.flatten(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.combine_chunks)
def combine_chunks(self, *args, **kwargs):
replay = ("combine_chunks", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.combine_chunks(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.cast)
def cast(self, *args, **kwargs):
replay = ("cast", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.cast(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.add_column)
def add_column(self, *args, **kwargs):
replay = ("add_column", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.add_column(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.append_column)
def append_column(self, *args, **kwargs):
replay = ("append_column", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.append_column(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.remove_column)
def remove_column(self, *args, **kwargs):
replay = ("remove_column", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.remove_column(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.set_column)
def set_column(self, *args, **kwargs):
replay = ("set_column", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.set_column(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.rename_columns)
def rename_columns(self, *args, **kwargs):
replay = ("rename_columns", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.rename_columns(*args, **kwargs), self.path, replays)
[docs] @inject_arrow_table_documentation(pa.Table.drop)
def drop(self, *args, **kwargs):
replay = ("drop", copy.deepcopy(args), copy.deepcopy(kwargs))
replays = self._append_replay(replay)
return MemoryMappedTable(self.table.drop(*args, **kwargs), self.path, replays)
# A ConcatenationTable is the concatenation of several tables.
# The ``blocks`` attributes stores a list of list of blocks.
# The first axis concatenates the tables along the axis 0 (it appends rows),
# while the second axis concatenates tables along the axis 1 (it appends columns).
TableBlockContainer = TypeVar("TableBlockContainer", TableBlock, List[TableBlock], List[List[TableBlock]])
[docs]class ConcatenationTable(Table):
"""
The table comes from the concatenation of several tables called blocks.
It enables concatenation on both axis 0 (append rows) and axis 1 (append columns).
The underlying tables are called "blocks" and can be either InMemoryTable
or MemoryMappedTable objects.
This allows to combine tables that come from memory or that are memory mapped.
When a ConcatenationTable is pickled, then each block is pickled:
- the InMemoryTable objects are pickled by copying all the data in memory;
- the MemoryMappedTable objects are pickled without copying the data into memory.
Instead, only the path to the memory mapped arrow file is pickled, as well as the list
of transforms to "replays" when reloading the table from the disk.
Its implementation requires to store each block separately.
The ``blocks`` attributes stores a list of list of blocks.
The first axis concatenates the tables along the axis 0 (it appends rows),
while the second axis concatenates tables along the axis 1 (it appends columns).
You can access the fully combined table by accessing the ConcatenationTable.table attribute,
and the blocks by accessing the ConcatenationTable.blocks attribute.
"""
def __init__(self, table: pa.Table, blocks: List[List[TableBlock]]):
super().__init__(table)
self.blocks = blocks
# Check that all the blocks have the right type.
# Only InMemoryTable and MemoryMappedTable are allowed.
for subtables in blocks:
for subtable in subtables:
if not isinstance(subtable, TableBlock):
raise TypeError(
"The blocks of a ConcatenationTable must be InMemoryTable or MemoryMappedTable objects"
f", but got {subtable}."
)
def __getstate__(self):
return {"blocks": self.blocks}
def __setstate__(self, state):
blocks = state["blocks"]
table = self._concat_blocks_horizontally_and_vertically(blocks)
ConcatenationTable.__init__(self, table, blocks=blocks)
@staticmethod
def _concat_blocks(blocks: List[Union[TableBlock, pa.Table]], axis: int = 0) -> pa.Table:
pa_tables = [table.table if hasattr(table, "table") else table for table in blocks]
if axis == 0:
# Align schemas: re-order the columns to make the schemas match before concatenating over rows
schema = pa_tables[0].schema
pa_tables = [
table
if table.schema == schema
else pa.Table.from_arrays([table[name] for name in schema.names], names=schema.names)
for table in pa_tables
]
return pa.concat_tables(pa_tables)
elif axis == 1:
for i, table in enumerate(pa_tables):
if i == 0:
pa_table = table
else:
for name, col in zip(table.column_names, table.columns):
pa_table = pa_table.append_column(name, col)
return pa_table
else:
raise ValueError("'axis' must be either 0 or 1")
@classmethod
def _concat_blocks_horizontally_and_vertically(cls, blocks: List[List[TableBlock]]) -> pa.Table:
pa_tables_to_concat_vertically = []
for i, tables in enumerate(blocks):
if not tables:
continue
pa_table_horizontally_concatenated = cls._concat_blocks(tables, axis=1)
pa_tables_to_concat_vertically.append(pa_table_horizontally_concatenated)
return cls._concat_blocks(pa_tables_to_concat_vertically, axis=0)
@classmethod
def _merge_blocks(cls, blocks: TableBlockContainer, axis: Optional[int] = None) -> TableBlockContainer:
if axis is not None:
merged_blocks = []
for is_in_memory, block_group in groupby(blocks, key=lambda x: isinstance(x, InMemoryTable)):
if is_in_memory:
block_group = [InMemoryTable(cls._concat_blocks(list(block_group), axis=axis))]
merged_blocks += list(block_group)
else: # both
merged_blocks = [cls._merge_blocks(row_block, axis=1) for row_block in blocks]
if all(len(row_block) == 1 for row_block in merged_blocks):
merged_blocks = cls._merge_blocks(
[block for row_block in merged_blocks for block in row_block], axis=0
)
return merged_blocks
@classmethod
def _consolidate_blocks(cls, blocks: TableBlockContainer) -> TableBlockContainer:
if isinstance(blocks, TableBlock):
return blocks
elif isinstance(blocks[0], TableBlock):
return cls._merge_blocks(blocks, axis=0)
else:
return cls._merge_blocks(blocks)
@classmethod
def from_blocks(cls, blocks: TableBlockContainer) -> "ConcatenationTable":
blocks = cls._consolidate_blocks(blocks)
if isinstance(blocks, TableBlock):
table = blocks
return cls(table.table, [[table]])
elif isinstance(blocks[0], TableBlock):
table = cls._concat_blocks(blocks, axis=0)
blocks = [[t] for t in blocks]
return cls(table, blocks)
else:
table = cls._concat_blocks_horizontally_and_vertically(blocks)
return cls(table, blocks)
[docs] @classmethod
def from_tables(cls, tables: List[Union[pa.Table, Table]], axis: int = 0) -> "ConcatenationTable":
"""Create ConcatenationTable from list of tables.
Args:
tables (list of :class:`Table` or list of :obj:`pyarrow.Table`): List of tables.
axis: (``{0, 1}``, default ``0``, meaning over rows):
Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
(horizontally).
.. versionadded:: 1.6.0
"""
def to_blocks(table):
if isinstance(table, pa.Table):
return [[InMemoryTable(table)]]
elif isinstance(table, ConcatenationTable):
return copy.deepcopy(table.blocks)
else:
return [[table]]
def _split_like(blocks_to_split, blocks_like):
splits = []
offset = 0
for block_row in blocks_like:
length = block_row[0].num_rows
splits.append((offset, length))
offset += length
return [
[block.slice(offset=split[0], length=split[1]) for block in blocks_to_split[0]] for split in splits
]
def _extend_blocks(result, blocks: List[List[TableBlock]], axis: int = 0):
if axis == 0:
result.extend(blocks)
elif axis == 1:
if len(result) == 1 and len(blocks) > 1:
result = _split_like(result, blocks) # Split result
elif len(blocks) == 1 and len(result) > 1:
blocks = _split_like(blocks, result) # Split blocks
# TODO: This assumes each block_row has the same num_rows
for i, row_blocks in enumerate(blocks):
result[i].extend(row_blocks)
return result
blocks = to_blocks(tables[0])
for table in tables[1:]:
table_blocks = to_blocks(table)
blocks = _extend_blocks(blocks, table_blocks, axis=axis)
return cls.from_blocks(blocks)
@property
def _slices(self):
offset = 0
for tables in self.blocks:
length = len(tables[0])
yield (offset, length)
offset += length
[docs] @inject_arrow_table_documentation(pa.Table.slice)
def slice(self, offset=0, length=None):
table = self.table.slice(offset, length=length)
length = length if length is not None else self.num_rows - offset
blocks = []
for tables in self.blocks:
n_rows = len(tables[0])
if length == 0:
break
elif n_rows <= offset:
offset = offset - n_rows
elif n_rows <= offset + length:
blocks.append([t.slice(offset) for t in tables])
length, offset = length + offset - n_rows, 0
else:
blocks.append([t.slice(offset, length) for t in tables])
length, offset = 0, 0
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.filter)
def filter(self, mask, *args, **kwargs):
table = self.table.filter(mask, *args, **kwargs)
blocks = []
for (offset, length), tables in zip(self._slices, self.blocks):
submask = mask.slice(offset, length)
blocks.append([t.filter(submask, *args, **kwargs) for t in tables])
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.flatten)
def flatten(self, *args, **kwargs):
table = self.table.flatten(*args, **kwargs)
blocks = []
for tables in self.blocks:
blocks.append([t.flatten(*args, **kwargs) for t in tables])
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.combine_chunks)
def combine_chunks(self, *args, **kwargs):
table = self.table.combine_chunks(*args, **kwargs)
blocks = []
for tables in self.blocks:
blocks.append([t.combine_chunks(*args, **kwargs) for t in tables])
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.cast)
def cast(self, target_schema, *args, **kwargs):
table = self.table.cast(target_schema, *args, **kwargs)
blocks = []
for subtables in self.blocks:
new_tables = []
fields = list(target_schema)
for subtable in subtables:
subfields = []
for name in subtable.column_names:
subfields.append(fields.pop(next(i for i, field in enumerate(fields) if field.name == name)))
subschema = pa.schema(subfields)
new_tables.append(subtable.cast(subschema, *args, **kwargs))
blocks.append(new_tables)
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.add_column)
def add_column(self, *args, **kwargs):
raise NotImplementedError()
[docs] @inject_arrow_table_documentation(pa.Table.append_column)
def append_column(self, *args, **kwargs):
raise NotImplementedError()
[docs] @inject_arrow_table_documentation(pa.Table.remove_column)
def remove_column(self, i, *args, **kwargs):
table = self.table.remove_column(i, *args, **kwargs)
name = self.table.column_names[i]
blocks = []
for tables in self.blocks:
blocks.append(
[
t.remove_column(t.column_names.index(name), *args, **kwargs) if name in t.column_names else t
for t in tables
]
)
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.set_column)
def set_column(self, *args, **kwargs):
raise NotImplementedError()
[docs] @inject_arrow_table_documentation(pa.Table.rename_columns)
def rename_columns(self, names, *args, **kwargs):
table = self.table.rename_columns(names, *args, **kwargs)
names = dict(zip(self.table.column_names, names))
blocks = []
for tables in self.blocks:
blocks.append(
[t.rename_columns([names[name] for name in t.column_names], *args, **kwargs) for t in tables]
)
return ConcatenationTable(table, blocks)
[docs] @inject_arrow_table_documentation(pa.Table.drop)
def drop(self, columns, *args, **kwargs):
table = self.table.drop(columns)
blocks = []
for tables in self.blocks:
blocks.append([t.drop([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables])
return ConcatenationTable(table, blocks)
[docs]def concat_tables(tables: List[Table], axis: int = 0) -> Table:
"""
Concatenate tables.
Args:
tables (list of :class:`Table`): List of tables to be concatenated.
axis (``{0, 1}``, default ``0``, meaning over rows):
Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
(horizontally).
.. versionadded:: 1.6.0
Returns:
:obj:`datasets.table.Table` that is the concatenated table:
If the number of input tables is > 1, then the returned table is a :obj:`datasets.table.ConcatenationTable`.
Otherwise if there's only one table, it is returned as is.
"""
tables = list(tables)
if len(tables) == 1:
return tables[0]
return ConcatenationTable.from_tables(tables, axis=axis)
[docs]def list_table_cache_files(table: Table) -> List[str]:
"""
Get the cache files that are loaded by the table.
Cache file are used when parts of the table come from the disk via memory mapping.
Returns:
:obj:`List[str]`: a list of paths to the cache files loaded by the table
"""
if isinstance(table, ConcatenationTable):
cache_files = []
for subtables in table.blocks:
for subtable in subtables:
cache_files += list_table_cache_files(subtable)
return cache_files
elif isinstance(table, MemoryMappedTable):
return [table.path]
else:
return []
def cast_with_sliced_list_support(pa_table: pa.Table, schema: pa.Schema) -> pa.Table:
"""Same as pyarrow.Table.cast, except it works for sliced list arrays"""
def wrap_for_chunked_arrays(func):
"""Apply the function on each chunk of a pyarrow.ChunkedArray, or on the array directly"""
def wrapper(array):
if isinstance(array, pa.ChunkedArray):
return pa.chunked_array([func(chunk) for chunk in array.chunks])
else:
return func(array)
return wrapper
@wrap_for_chunked_arrays
def reset_sliced_list_offset(array: pa.ListArray):
"""Return the same pyarrow.ListArray but with array.offset == 0 for compatibility with cast"""
if array.offset == 0:
return array
elif len(array) == 0:
return array.values.slice(0, 0)
else:
values_offset = array.offsets[0] # the relevant values start at this index
new_values = array.values.slice(values_offset.as_py()) # get the values to start at the right position
new_offsets = pc.subtract(array.offsets, values_offset) # update the offsets accordingly
return pa.ListArray.from_arrays(new_offsets, new_values)
arrays = [reset_sliced_list_offset(array) if isinstance(array.type, pa.ListType) else array for array in pa_table]
return pa.Table.from_arrays(arrays, schema=schema)