if isinstance(name, six.string_types):<EOL><INDENT>if isinstance(value, Expression):<EOL><INDENT>value = value.expression<EOL><DEDENT>if isinstance(value, np.ndarray):<EOL><INDENT>self.add_column(name, value)<EOL><DEDENT>else:<EOL><INDENT>self.add_virtual_column(name, value)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(name)))<EOL><DEDENT>


if isinstance(item, int):<EOL><INDENT>names = self.get_column_names()<EOL>return [self.evaluate(name, item, item+<NUM_LIT:1>)[<NUM_LIT:0>] for name in names]<EOL><DEDENT>elif isinstance(item, six.string_types):<EOL><INDENT>if hasattr(self, item) and isinstance(getattr(self, item), Expression):<EOL><INDENT>return getattr(self, item)<EOL><DEDENT>return Expression(self, item)  <EOL><DEDENT>elif isinstance(item, Expression):<EOL><INDENT>expression = item.expression<EOL>df = self.copy()<EOL>df.select(expression, name=FILTER_SELECTION_NAME, mode='<STR_LIT>')<EOL>return df<EOL><DEDENT>elif isinstance(item, (tuple, list)):<EOL><INDENT>df = self.copy(column_names=item)<EOL>return df<EOL><DEDENT>elif isinstance(item, slice):<EOL><INDENT>df = self.extract()<EOL>start, stop, step = item.start, item.stop, item.step<EOL>start = start or <NUM_LIT:0><EOL>stop = stop or len(df)<EOL>assert step in [None, <NUM_LIT:1>]<EOL>df.set_active_range(start, stop)<EOL>return df.trim()<EOL><DEDENT>

Convenient way to get expressions, (shallow) copies of a few columns, or to apply filtering.

        Example:

        >>> df['Lz']  # the expression 'Lz
        >>> df['Lz/2'] # the expression 'Lz/2'
        >>> df[["Lz", "E"]] # a shallow copy with just two columns
        >>> df[df.Lz < 0]  # a shallow copy with the filter Lz < 0 applied


if isinstance(item, Expression):<EOL><INDENT>name = item.expression<EOL><DEDENT>else:<EOL><INDENT>name = item<EOL><DEDENT>if name in self.columns:<EOL><INDENT>del self.columns[name]<EOL>self.column_names.remove(name)<EOL><DEDENT>elif name in self.virtual_columns:<EOL><INDENT>del self.virtual_columns[name]<EOL>self.column_names.remove(name)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError('<STR_LIT>' % name)<EOL><DEDENT>if hasattr(self, name):<EOL><INDENT>try:<EOL><INDENT>if isinstance(getattr(self, name), Expression):<EOL><INDENT>delattr(self, name)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Removes a (virtual) column from the DataFrame.

        Note: this does not remove check if the column is used in a virtual expression or in the filter\
            and may lead to issues. It is safer to use :meth:`drop`.


@docsubst<EOL><INDENT>def drop(self, columns, inplace=False, check=True):<DEDENT>

columns = _ensure_list(columns)<EOL>columns = _ensure_strings_from_expressions(columns)<EOL>df = self if inplace else self.copy()<EOL>depending_columns = df._depending_columns(columns_exclude=columns)<EOL>for column in columns:<EOL><INDENT>if check and column in depending_columns:<EOL><INDENT>df._hide_column(column)<EOL><DEDENT>else:<EOL><INDENT>del df[column]<EOL><DEDENT><DEDENT>return df<EOL>

Drop columns (or a single column).

        :param columns: List of columns or a single column name
        :param inplace: {inplace}
        :param check: When true, it will check if the column is used in virtual columns or the filter, and hide it instead.


column = _ensure_string_from_expression(column)<EOL>new_name = self._find_valid_name('<STR_LIT>' + column)<EOL>self._rename(column, new_name)<EOL>

Hides a column by prefixing the name with \'__\


return vaex.utils.find_valid_name(initial_name, used=self.get_column_names(hidden=True))<EOL>

Finds a non-colliding name by optional postfixing


def _depending_columns(self, columns=None, columns_exclude=None, check_filter=True):

columns = set(columns or self.get_column_names(hidden=True))<EOL>if columns_exclude:<EOL><INDENT>columns -= set(columns_exclude)<EOL><DEDENT>depending_columns = set()<EOL>for column in columns:<EOL><INDENT>expression = self._expr(column)<EOL>depending_columns |= expression.variables()<EOL><DEDENT>depending_columns -= set(columns)<EOL>if check_filter:<EOL><INDENT>if self.filtered:<EOL><INDENT>selection = self.get_selection(FILTER_SELECTION_NAME)<EOL>depending_columns |= selection._depending_columns(self)<EOL><DEDENT><DEDENT>return depending_columns<EOL>

Find all depending column for a set of column (default all), minus the excluded ones


return iter(list(self.get_column_names()))<EOL>


<EOL>root_nodes = []<EOL>leafes = []<EOL>def walk(node):<EOL><INDENT>if isinstance(node, six.string_types):<EOL><INDENT>leafes.append(node)<EOL>if node in root_nodes:  <EOL><INDENT>root_nodes.remove(node)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>node_repr, fname, fobj, deps = node<EOL>if node_repr in self.virtual_columns:<EOL><INDENT>leafes.append(node_repr)<EOL>if node_repr in root_nodes:<EOL><INDENT>root_nodes.remove(node_repr)<EOL><DEDENT><DEDENT>for dep in deps:<EOL><INDENT>walk(dep)<EOL><DEDENT><DEDENT><DEDENT>for column in self.virtual_columns.keys():<EOL><INDENT>if column not in leafes:<EOL><INDENT>root_nodes.append(column)<EOL><DEDENT>node = self[column]._graph()<EOL>node_repr, fname, fobj, deps = node<EOL>for dep in deps:<EOL><INDENT>walk(dep)<EOL><DEDENT><DEDENT>return root_nodes<EOL>

Returns a list of string which are the virtual columns that are not used in any other virtual column.


from graphviz import Digraph<EOL>dot = dot or Digraph(comment='<STR_LIT>')<EOL>root_nodes = self._root_nodes()<EOL>for column in root_nodes:<EOL><INDENT>self[column]._graphviz(dot=dot)<EOL><DEDENT>return dot<EOL>

Return a graphviz.Digraph object with a graph of all virtual columns


def categorize(self, column, labels=None, check=True):

column = _ensure_string_from_expression(column)<EOL>if check:<EOL><INDENT>vmin, vmax = self.minmax(column)<EOL>if labels is None:<EOL><INDENT>N = int(vmax + <NUM_LIT:1>)<EOL>labels = list(map(str, range(N)))<EOL><DEDENT>if (vmax - vmin) >= len(labels):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(vmax, len(labels)))<EOL><DEDENT><DEDENT>self._categories[column] = dict(labels=labels, N=len(labels))<EOL>

Mark column as categorical, with given labels, assuming zero indexing


def ordinal_encode(self, column, values=None, inplace=False):

column = _ensure_string_from_expression(column)<EOL>df = self if inplace else self.copy()<EOL>df_unfiltered = df.copy()<EOL>df_unfiltered.select_nothing(name=FILTER_SELECTION_NAME)<EOL>df_unfiltered._length_unfiltered = df._length_original<EOL>df_unfiltered.set_active_range(<NUM_LIT:0>, df._length_original)<EOL>found_values, codes = df_unfiltered.unique(column, return_inverse=True)<EOL>if values is None:<EOL><INDENT>values = found_values<EOL><DEDENT>else:<EOL><INDENT>translation = np.zeros(len(found_values), dtype=np.uint64)<EOL>missing_value = len(found_values)<EOL>for i, found_value in enumerate(found_values):<EOL><INDENT>try:<EOL><INDENT>found_value = found_value.decode('<STR_LIT:ascii>')<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>if found_value not in values:  <EOL><INDENT>translation[i] = missing_value<EOL><DEDENT>else:<EOL><INDENT>translation[i] = values.index(found_value)<EOL><DEDENT><DEDENT>codes = translation[codes]<EOL>if missing_value in translation:<EOL><INDENT>codes = np.ma.masked_array(codes, codes==missing_value)<EOL><DEDENT><DEDENT>original_column = df.rename_column(column, '<STR_LIT>' + column, unique=True)<EOL>labels = [str(k) for k in values]<EOL>df.add_column(column, codes)<EOL>df._categories[column] = dict(labels=labels, N=len(values), values=values)<EOL>return df<EOL>

Encode column as ordinal values and mark it as categorical.

        The existing column is renamed to a hidden column and replaced by a numerical columns
        with values between [0, len(values)-1].


class Datas(object):<EOL><INDENT>pass<EOL><DEDENT>datas = Datas()<EOL>for name, array in self.columns.items():<EOL><INDENT>setattr(datas, name, array)<EOL><DEDENT>return datas<EOL>

Gives direct access to the data as numpy arrays.

        Convenient when working with IPython in combination with small DataFrames, since this gives tab-completion.
        Only real columns (i.e. no virtual) columns can be accessed, for getting the data from virtual columns, use
        DataFrame.evalulate(...).

        Columns can be accesed by there names, which are attributes. The attribues are of type numpy.ndarray.

        Example:

        >>> df = vaex.example()
        >>> r = np.sqrt(df.data.x**2 + df.data.y**2)


def shallow_copy(self, virtual=True, variables=True):

df = DataFrameLocal(self.name, self.path, self.column_names)<EOL>df.columns.update(self.columns)<EOL>df._length_unfiltered = self._length_unfiltered<EOL>df._length_original = self._length_original<EOL>df._index_end = self._index_end<EOL>df._index_start = self._index_start<EOL>df._active_fraction = self._active_fraction<EOL>if virtual:<EOL><INDENT>df.virtual_columns.update(self.virtual_columns)<EOL><DEDENT>if variables:<EOL><INDENT>df.variables.update(self.variables)<EOL><DEDENT>return df<EOL>

Creates a (shallow) copy of the DataFrame.

        It will link to the same data, but will have its own state, e.g. virtual columns, variables, selection etc.


return True<EOL>

The local implementation of :func:`DataFrame.evaluate`, always returns True.


if selection:<EOL><INDENT>return <NUM_LIT:0> if self.mask is None else np.sum(self.mask)<EOL><DEDENT>else:<EOL><INDENT>return len(self)<EOL><DEDENT>

Get the length of the DataFrames, for the selection of the whole DataFrame.

        If selection is False, it returns len(df).

        TODO: Implement this in DataFrameRemote, and move the method up in :func:`DataFrame.length`

        :param selection: When True, will return the number of selected rows
        :return:


@_hidden<EOL><INDENT>def __call__(self, *expressions, **kwargs):<DEDENT>

import vaex.legacy<EOL>return vaex.legacy.SubspaceLocal(self, expressions, kwargs.get("<STR_LIT>") or self.executor, delay=kwargs.get("<STR_LIT>", False))<EOL>

The local implementation of :func:`DataFrame.__call__`


if dtype is None:<EOL><INDENT>dtype = np.float64<EOL><DEDENT>chunks = []<EOL>for name in self.get_column_names(strings=False):<EOL><INDENT>if not np.can_cast(self.dtype(name), dtype):<EOL><INDENT>if self.dtype(name) != dtype:<EOL><INDENT>raise ValueError("<STR_LIT>" % (name, self.dtype(name), dtype))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>chunks.append(self.evaluate(name))<EOL><DEDENT><DEDENT>return np.array(chunks, dtype=dtype).T<EOL>

Gives a full memory copy of the DataFrame into a 2d numpy array of shape (n_rows, n_columns).
        Note that the memory order is fortran, so all values of 1 column are contiguous in memory for performance reasons.

        Note this returns the same result as:

        >>> np.array(ds)

        If any of the columns contain masked arrays, the masks are ignored (i.e. the masked elements are returned as well).


@vaex.utils.deprecated('<STR_LIT>')<EOL><INDENT>def _hstack(self, other, prefix=None):<DEDENT>

assert len(self) == len(other), "<STR_LIT>"<EOL>for name in other.get_column_names():<EOL><INDENT>if prefix:<EOL><INDENT>new_name = prefix + name<EOL><DEDENT>else:<EOL><INDENT>new_name = name<EOL><DEDENT>self.add_column(new_name, other.columns[name])<EOL><DEDENT>

Join the columns of the other DataFrame to this one, assuming the ordering is the same


dfs = []<EOL>if isinstance(self, DataFrameConcatenated):<EOL><INDENT>dfs.extend(self.dfs)<EOL><DEDENT>else:<EOL><INDENT>dfs.extend([self])<EOL><DEDENT>if isinstance(other, DataFrameConcatenated):<EOL><INDENT>dfs.extend(other.dfs)<EOL><DEDENT>else:<EOL><INDENT>dfs.extend([other])<EOL><DEDENT>return DataFrameConcatenated(dfs)<EOL>

Concatenates two DataFrames, adding the rows of one the other DataFrame to the current, returned in a new DataFrame.

        No copy of the data is made.

        :param other: The other DataFrame that is concatenated with this DataFrame
        :return: New DataFrame with the rows concatenated
        :rtype: DataFrameConcatenated


def evaluate(self, expression, i1=None, i2=None, out=None, selection=None, filtered=True, internal=False):

expression = _ensure_string_from_expression(expression)<EOL>selection = _ensure_strings_from_expressions(selection)<EOL>i1 = i1 or <NUM_LIT:0><EOL>i2 = i2 or (len(self) if (self.filtered and filtered) else self.length_unfiltered())<EOL>mask = None<EOL>if self.filtered and filtered:  <EOL><INDENT>indices = self._filtered_range_to_unfiltered_indices(i1, i2)<EOL>i1 = indices[<NUM_LIT:0>]<EOL>i2 = indices[-<NUM_LIT:1>] + <NUM_LIT:1>  <EOL><DEDENT>if selection is not None or (self.filtered and filtered):<EOL><INDENT>mask = self.evaluate_selection_mask(selection, i1, i2)<EOL><DEDENT>scope = scopes._BlockScope(self, i1, i2, mask=mask, **self.variables)<EOL>if out is not None:<EOL><INDENT>scope.buffers[expression] = out<EOL><DEDENT>value = scope.evaluate(expression)<EOL>if isinstance(value, ColumnString) and not internal:<EOL><INDENT>value = value.to_numpy()<EOL><DEDENT>return value<EOL>

The local implementation of :func:`DataFrame.evaluate`


def compare(self, other, report_missing=True, report_difference=False, show=<NUM_LIT:10>, orderby=None, column_names=None):

if column_names is None:<EOL><INDENT>column_names = self.get_column_names(virtual=False)<EOL>for other_column_name in other.get_column_names(virtual=False):<EOL><INDENT>if other_column_name not in column_names:<EOL><INDENT>column_names.append(other_column_name)<EOL><DEDENT><DEDENT><DEDENT>different_values = []<EOL>missing = []<EOL>type_mismatch = []<EOL>meta_mismatch = []<EOL>assert len(self) == len(other)<EOL>if orderby:<EOL><INDENT>index1 = np.argsort(self.columns[orderby])<EOL>index2 = np.argsort(other.columns[orderby])<EOL><DEDENT>for column_name in column_names:<EOL><INDENT>if column_name not in self.get_column_names(virtual=False):<EOL><INDENT>missing.append(column_name)<EOL>if report_missing:<EOL><INDENT>print("<STR_LIT>" % column_name)<EOL><DEDENT><DEDENT>elif column_name not in other.get_column_names(virtual=False):<EOL><INDENT>missing.append(column_name)<EOL>if report_missing:<EOL><INDENT>print("<STR_LIT>" % column_name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ucd1 = self.ucds.get(column_name)<EOL>ucd2 = other.ucds.get(column_name)<EOL>if ucd1 != ucd2:<EOL><INDENT>print("<STR_LIT>" % (ucd1, ucd2, column_name))<EOL>meta_mismatch.append(column_name)<EOL><DEDENT>unit1 = self.units.get(column_name)<EOL>unit2 = other.units.get(column_name)<EOL>if unit1 != unit2:<EOL><INDENT>print("<STR_LIT>" % (unit1, unit2, column_name))<EOL>meta_mismatch.append(column_name)<EOL><DEDENT>type1 = self.dtype(column_name)<EOL>if type1 != str_type:<EOL><INDENT>type1 = type1.type<EOL><DEDENT>type2 = other.dtype(column_name)<EOL>if type2 != str_type:<EOL><INDENT>type2 = type2.type<EOL><DEDENT>if type1 != type2:<EOL><INDENT>print("<STR_LIT>" % (self.dtype(column_name), other.dtype(column_name), column_name))<EOL>type_mismatch.append(column_name)<EOL><DEDENT>else:<EOL><INDENT>a = self.evaluate(column_name)<EOL>b = other.evaluate(column_name)<EOL>if orderby:<EOL><INDENT>a = a[index1]<EOL>b = b[index2]<EOL><DEDENT>def normalize(ar):<EOL><INDENT>if ar.dtype == str_type:<EOL><INDENT>return ar<EOL><DEDENT>if ar.dtype.kind == "<STR_LIT:f>" and hasattr(ar, "<STR_LIT>"):<EOL><INDENT>mask = ar.mask<EOL>ar = ar.copy()<EOL>ar[mask] = np.nan<EOL><DEDENT>if ar.dtype.kind in "<STR_LIT>":<EOL><INDENT>if hasattr(ar, "<STR_LIT>"):<EOL><INDENT>data = ar.data<EOL><DEDENT>else:<EOL><INDENT>data = ar<EOL><DEDENT>values = [value.strip() for value in data.tolist()]<EOL>if hasattr(ar, "<STR_LIT>"):<EOL><INDENT>ar = np.ma.masked_array(values, ar.mask)<EOL><DEDENT>else:<EOL><INDENT>ar = np.array(values)<EOL><DEDENT><DEDENT>return ar<EOL><DEDENT>def equal_mask(a, b):<EOL><INDENT>a = normalize(a)<EOL>b = normalize(b)<EOL>boolean_mask = (a == b)<EOL>if self.dtype(column_name) != str_type and self.dtype(column_name).kind == '<STR_LIT:f>':  <EOL><INDENT>boolean_mask |= (np.isnan(a) & np.isnan(b))<EOL><DEDENT>return boolean_mask<EOL><DEDENT>boolean_mask = equal_mask(a, b)<EOL>all_equal = np.all(boolean_mask)<EOL>if not all_equal:<EOL><INDENT>count = np.sum(~boolean_mask)<EOL>print("<STR_LIT>" % (column_name, count, len(self)))<EOL>different_values.append(column_name)<EOL>if report_difference:<EOL><INDENT>indices = np.arange(len(self))[~boolean_mask]<EOL>values1 = self.columns[column_name][:][~boolean_mask]<EOL>values2 = other.columns[column_name][:][~boolean_mask]<EOL>print("<STR_LIT>")<EOL>for i in range(min(len(values1), show)):<EOL><INDENT>try:<EOL><INDENT>diff = values1[i] - values2[i]<EOL><DEDENT>except:<EOL><INDENT>diff = "<STR_LIT>"<EOL><DEDENT>print("<STR_LIT>" % (column_name, indices[i], values1[i], values2[i], column_name, indices[i], diff))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return different_values, missing, type_mismatch, meta_mismatch<EOL>

Compare two DataFrames and report their difference, use with care for large DataFrames


@docsubst<EOL><INDENT>def join(self, other, on=None, left_on=None, right_on=None, lsuffix='<STR_LIT>', rsuffix='<STR_LIT>', how='<STR_LIT:left>', inplace=False):<DEDENT>

ds = self if inplace else self.copy()<EOL>if how == '<STR_LIT:left>':<EOL><INDENT>left = ds<EOL>right = other<EOL><DEDENT>elif how == '<STR_LIT:right>':<EOL><INDENT>left = other<EOL>right = ds<EOL>lsuffix, rsuffix = rsuffix, lsuffix<EOL>left_on, right_on = right_on, left_on<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(how))<EOL><DEDENT>for name in right:<EOL><INDENT>if name in left and name + rsuffix == name + lsuffix:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>.format(name))<EOL><DEDENT><DEDENT>right = right.extract()  <EOL>assert left.length_unfiltered() == left.length_original()<EOL>N = left.length_unfiltered()<EOL>N_other = len(right)<EOL>left_on = left_on or on<EOL>right_on = right_on or on<EOL>if left_on is None and right_on is None:<EOL><INDENT>for name in right:<EOL><INDENT>right_name = name<EOL>if name in left:<EOL><INDENT>left.rename_column(name, name + lsuffix)<EOL>right_name = name + rsuffix<EOL><DEDENT>if name in right.virtual_columns:<EOL><INDENT>left.add_virtual_column(right_name, right.virtual_columns[name])<EOL><DEDENT>else:<EOL><INDENT>left.add_column(right_name, right.columns[name])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>left_values = left.evaluate(left_on, filtered=False)<EOL>right_values = right.evaluate(right_on)<EOL>if np.ma.isMaskedArray(left_values):<EOL><INDENT>mask = ~left_values.mask<EOL>left_values = left_values.data<EOL>index_left = dict(zip(left_values[mask], np.arange(N)[mask]))<EOL><DEDENT>else:<EOL><INDENT>index_left = dict(zip(left_values, np.arange(N)))<EOL><DEDENT>if np.ma.isMaskedArray(right_values):<EOL><INDENT>mask = ~right_values.mask<EOL>right_values = right_values.data<EOL>index_other = dict(zip(right_values[mask], np.arange(N_other)[mask]))<EOL><DEDENT>else:<EOL><INDENT>index_other = dict(zip(right_values, np.arange(N_other)))<EOL><DEDENT>left_mask = np.ones(N, dtype=np.bool)<EOL>left_row_to_right = np.zeros(N, dtype=np.int64) - <NUM_LIT:1><EOL>for i in range(N_other):<EOL><INDENT>left_row = index_left.get(right_values[i])<EOL>if left_row is not None:<EOL><INDENT>left_mask[left_row] = False  <EOL>left_row_to_right[left_row] = i<EOL><DEDENT><DEDENT>lookup = np.ma.array(left_row_to_right, mask=left_mask)<EOL>for name in right:<EOL><INDENT>right_name = name<EOL>if name in left:<EOL><INDENT>left.rename_column(name, name + lsuffix)<EOL>right_name = name + rsuffix<EOL><DEDENT>if name in right.virtual_columns:<EOL><INDENT>left.add_virtual_column(right_name, right.virtual_columns[name])<EOL><DEDENT>else:<EOL><INDENT>left.add_column(right_name, ColumnIndexed(right, lookup, name))<EOL><DEDENT><DEDENT><DEDENT>return left<EOL>

Return a DataFrame joined with other DataFrames, matched by columns/expression on/left_on/right_on

        If neither on/left_on/right_on is given, the join is done by simply adding the columns (i.e. on the implicit
        row index).

        Note: The filters will be ignored when joining, the full DataFrame will be joined (since filters may
        change). If either DataFrame is heavily filtered (contains just a small number of rows) consider running
        :func:`DataFrame.extract` first.

        Example:

        >>> a = np.array(['a', 'b', 'c'])
        >>> x = np.arange(1,4)
        >>> ds1 = vaex.from_arrays(a=a, x=x)
        >>> b = np.array(['a', 'b', 'd'])
        >>> y = x**2
        >>> ds2 = vaex.from_arrays(b=b, y=y)
        >>> ds1.join(ds2, left_on='a', right_on='b')

        :param other: Other DataFrame to join with (the right side)
        :param on: default key for the left table (self)
        :param left_on: key for the left table (self), overrides on
        :param right_on: default key for the right table (other), overrides on
        :param lsuffix: suffix to add to the left column names in case of a name collision
        :param rsuffix: similar for the right
        :param how: how to join, 'left' keeps all rows on the left, and adds columns (with possible missing values)
                'right' is similar with self and other swapped.
        :param inplace: {inplace}
        :return:


def export(self, path, column_names=None, byteorder="<STR_LIT:=>", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):

if path.endswith('<STR_LIT>'):<EOL><INDENT>self.export_arrow(path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL><DEDENT>elif path.endswith('<STR_LIT>'):<EOL><INDENT>self.export_hdf5(path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL><DEDENT>elif path.endswith('<STR_LIT>'):<EOL><INDENT>self.export_fits(path, column_names, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL><DEDENT>if path.endswith('<STR_LIT>'):<EOL><INDENT>self.export_parquet(path, column_names, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL><DEDENT>

Exports the DataFrame to a file written with arrow

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian (not supported for fits)
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:


def export_arrow(self, path, column_names=None, byteorder="<STR_LIT:=>", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):

import vaex_arrow.export<EOL>vaex_arrow.export.export(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL>

Exports the DataFrame to a file written with arrow

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:


def export_parquet(self, path, column_names=None, byteorder="<STR_LIT:=>", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):

import vaex_arrow.export<EOL>vaex_arrow.export.export_parquet(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL>

Exports the DataFrame to a parquet file

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:


def export_hdf5(self, path, column_names=None, byteorder="<STR_LIT:=>", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):

import vaex.export<EOL>vaex.export.export_hdf5(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL>

Exports the DataFrame to a vaex hdf5 file

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:


def export_fits(self, path, column_names=None, shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):

import vaex.export<EOL>vaex.export.export_fits(self, path, column_names, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)<EOL>

Exports the DataFrame to a fits file that is compatible with TOPCAT colfits format

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:


return int(self.count(selection=selection).item())<EOL>

The local implementation of :func:`DataFrame.selected_length`


from .groupby import GroupBy<EOL>groupby = GroupBy(self, by=by)<EOL>if agg is None:<EOL><INDENT>return groupby<EOL><DEDENT>else:<EOL><INDENT>return groupby.agg(agg)<EOL><DEDENT>

Return a :class:`GroupBy` or :class:`DataFrame` object when agg is not None

        Examples:

        >>> import vaex
        >>> import numpy as np
        >>> np.random.seed(42)
        >>> x = np.random.randint(1, 5, 10)
        >>> y = x**2
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df.groupby(df.x, agg='count')
        #    x    y_count
        0    3          4
        1    4          2
        2    1          3
        3    2          1
        >>> df.groupby(df.x, agg=[vaex.agg.count('y'), vaex.agg.mean('y')])
        #    x    y_count    y_mean
        0    3          4         9
        1    4          2        16
        2    1          3         1
        3    2          1         4
        >>> df.groupby(df.x, agg={'z': [vaex.agg.count('y'), vaex.agg.mean('y')]})
        #    x    z_count    z_mean
        0    3          4         9
        1    4          2        16
        2    1          3         1
        3    2          1         4

        Example using datetime:

        >>> import vaex
        >>> import numpy as np
        >>> t = np.arange('2015-01-01', '2015-02-01', dtype=np.datetime64)
        >>> y = np.arange(len(t))
        >>> df = vaex.from_arrays(t=t, y=y)
        >>> df.groupby(vaex.BinnerTime.per_week(df.t)).agg({'y' : 'sum'})
        #  t                      y
        0  2015-01-01 00:00:00   21
        1  2015-01-08 00:00:00   70
        2  2015-01-15 00:00:00  119
        3  2015-01-22 00:00:00  168
        4  2015-01-29 00:00:00   87

        :param dict, list or agg agg: Aggregate operation in the form of a string, vaex.agg object, a dictionary
            where the keys indicate the target column names, and the values the operations, or the a list of aggregates.
            When not given, it will return the groupby object.
        :return: :class:`DataFrame` or :class:`GroupBy` object.


from .groupby import BinBy<EOL>binby = BinBy(self, by=by)<EOL>if agg is None:<EOL><INDENT>return binby<EOL><DEDENT>else:<EOL><INDENT>return binby.agg(agg)<EOL><DEDENT>

Return a :class:`BinBy` or :class:`DataArray` object when agg is not None

        The binby operations does not return a 'flat' DataFrame, instead it returns an N-d grid
        in the form of an xarray.

        :param dict, list or agg agg: Aggregate operation in the form of a string, vaex.agg object, a dictionary
            where the keys indicate the target column names, and the values the operations, or the a list of aggregates.
            When not given, it will return the binby object.
        :return: :class:`DataArray` or :class:`BinBy` object.


@docsubst<EOL><INDENT>@stat_1d<EOL>def _agg(self, aggregator, grid, selection=False, delay=False, progress=None):<DEDENT>

task_agg = self._get_task_agg(grid)<EOL>sub_task = aggregator.add_operations(task_agg)<EOL>return self._delay(delay, sub_task)<EOL>

:param selection: {selection}
:param delay: {delay}
:param progress: {progress}
:return: {return_stat_scalar}


<EOL>super(DataFrameArrays, self).add_column(name, data)<EOL>self._length_unfiltered = int(round(self._length_original * self._active_fraction))<EOL>

Add a column to the DataFrame

        :param str name: name of column
        :param data: numpy array with the data


return self.__array__()<EOL>


return AggregatorDescriptorBasic('<STR_LIT>', expression, '<STR_LIT:count>')<EOL>


return AggregatorDescriptorBasic('<STR_LIT>', expression, '<STR_LIT>')<EOL>


return AggregatorDescriptorMean('<STR_LIT>', expression, '<STR_LIT>')<EOL>


return AggregatorDescriptorBasic('<STR_LIT>', [expression, order_expression], '<STR_LIT>', multi_args=True)<EOL>


return _Statistic('<STR_LIT>', expression)<EOL>

Creates a standard deviation aggregation


return _Statistic('<STR_LIT>', x, y)<EOL>


def then(self, success=None, failure=None):

ret = self.create_next()<EOL>def callAndFulfill(v):<EOL><INDENT>"""<STR_LIT>"""<EOL>try:<EOL><INDENT>if aplus._isFunction(success):<EOL><INDENT>ret.fulfill(success(v))<EOL><DEDENT>else:<EOL><INDENT>ret.fulfill(v)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>Promise.last_exc_info = sys.exc_info()<EOL>e.exc_info = sys.exc_info()<EOL>ret.reject(e)<EOL><DEDENT><DEDENT>def callAndReject(r):<EOL><INDENT>"""<STR_LIT>"""<EOL>try:<EOL><INDENT>if aplus._isFunction(failure):<EOL><INDENT>ret.fulfill(failure(r))<EOL><DEDENT>else:<EOL><INDENT>ret.reject(r)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>Promise.last_exc_info = sys.exc_info()<EOL>e.exc_info = sys.exc_info()<EOL>ret.reject(e)<EOL><DEDENT><DEDENT>self.done(callAndFulfill, callAndReject)<EOL>return ret<EOL>

This method takes two optional arguments.  The first argument
is used if the "self promise" is fulfilled and the other is
used if the "self promise" is rejected.  In either case, this
method returns another promise that effectively represents
the result of either the first of the second argument (in the
case that the "self promise" is fulfilled or rejected,
respectively).

Each argument can be either:
  * None - Meaning no action is taken
  * A function - which will be called with either the value
        of the "self promise" or the reason for rejection of
        the "self promise".  The function may return:
        * A value - which will be used to fulfill the promise
          returned by this method.
        * A promise - which, when fulfilled or rejected, will
          cascade its value or reason to the promise returned
          by this method.
  * A value - which will be assigned as either the value
        or the reason for the promise returned by this method
        when the "self promise" is either fulfilled or rejected,
        respectively.

:type success: (object) -> object
:type failure: (object) -> object
:rtype : Promise


def monochrome(I, color, vmin=None, vmax=None):

if vmin is None:<EOL><INDENT>vmin = np.nanmin(I)<EOL><DEDENT>if vmax is None:<EOL><INDENT>vmax = np.nanmax(I)<EOL><DEDENT>normalized = (I - vmin) / (vmax - vmin)<EOL>return np.clip(normalized[..., np.newaxis], <NUM_LIT:0>, <NUM_LIT:1>) * np.array(color)<EOL>

Turns a intensity array to a monochrome 'image' by replacing each intensity by a scaled 'color'

    Values in I between vmin  and vmax get scaled between 0 and 1, and values outside this range are clipped to this.

    Example
    >>> I = np.arange(16.).reshape(4,4)
    >>> color = (0, 0, 1) # red
    >>> rgb = vx.image.monochrome(I, color) # shape is (4,4,3)

    :param I: ndarray of any shape (2d for image)
    :param color: sequence of a (r, g and b) value
    :param vmin: normalization minimum for I, or np.nanmin(I) when None
    :param vmax: normalization maximum for I, or np.nanmax(I) when None
    :return:


def polychrome(I, colors, vmin=None, vmax=None, axis=-<NUM_LIT:1>):

axes_length = len(I.shape)<EOL>allaxes = list(range(axes_length))<EOL>otheraxes = list(allaxes)<EOL>otheraxes.remove((axis + axes_length) % axes_length)<EOL>otheraxes = tuple(otheraxes)<EOL>if vmin is None:<EOL><INDENT>vmin = np.nanmin(I, axis=otheraxes)<EOL><DEDENT>if vmax is None:<EOL><INDENT>vmax = np.nanmax(I, axis=otheraxes)<EOL><DEDENT>normalized = (I - vmin) / (vmax - vmin)<EOL>return np.clip(normalized, <NUM_LIT:0>, <NUM_LIT:1>).dot(colors)<EOL>

Similar to monochrome, but now do it for multiple colors

    Example
    >>> I = np.arange(32.).reshape(4,4,2)
    >>> colors = [(0, 0, 1), (0, 1, 0)] # red and green
    >>> rgb = vx.image.polychrome(I, colors) # shape is (4,4,3)

    :param I: ndarray of any shape (3d will result in a 2d image)
    :param colors: sequence of [(r,g,b), ...] values
    :param vmin: normalization minimum for I, or np.nanmin(I) when None
    :param vmax: normalization maximum for I, or np.nanmax(I) when None
    :param axis: axis which to sum over, by default the last
    :return:


return mapping(iter_properties(fh))<EOL>

Reads properties from a Java .properties file.

Returns a dict (or provided mapping) of properties.

:param fh: a readable file-like object
:param mapping: mapping type to load properties into


def store_properties(fh, props, comment=None, timestamp=True):

if comment is not None:<EOL><INDENT>write_comment(fh, comment)<EOL><DEDENT>if timestamp:<EOL><INDENT>write_comment(fh, time.strftime('<STR_LIT>'))<EOL><DEDENT>if hasattr(props, '<STR_LIT>'):<EOL><INDENT>for key in props:<EOL><INDENT>write_property(fh, key, props[key])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for key, value in props:<EOL><INDENT>write_property(fh, key, value)<EOL><DEDENT><DEDENT>

Writes properties to the file in Java properties format.

:param fh: a writable file-like object
:param props: a mapping (dict) or iterable of key/value pairs
:param comment: comment to write to the beginning of the file
:param timestamp: boolean indicating whether to write a timestamp comment


_require_string(comment, '<STR_LIT>')<EOL>fh.write(_escape_comment(comment))<EOL>fh.write(b'<STR_LIT:\n>')<EOL>

Writes a comment to the file in Java properties format.

Newlines in the comment text are automatically turned into a continuation
of the comment by adding a "#" to the beginning of each line.

:param fh: a writable file-like object
:param comment: comment string to write


if key is COMMENT:<EOL><INDENT>write_comment(fh, value)<EOL>return<EOL><DEDENT>_require_string(key, '<STR_LIT>')<EOL>_require_string(value, '<STR_LIT>')<EOL>fh.write(_escape_key(key))<EOL>fh.write(b'<STR_LIT:=>')<EOL>fh.write(_escape_value(value))<EOL>fh.write(b'<STR_LIT:\n>')<EOL>

Write a single property to the file in Java properties format.

:param fh: a writable file-like object
:param key: the key to write
:param value: the value to write


for line in _property_lines(fh):<EOL><INDENT>key, value = _split_key_value(line)<EOL>if key is not COMMENT:<EOL><INDENT>key = _unescape(key)<EOL><DEDENT>elif not comments:<EOL><INDENT>continue<EOL><DEDENT>yield key, _unescape(value)<EOL><DEDENT>

Incrementally read properties from a Java .properties file.

Yields tuples of key/value pairs.

If ``comments`` is `True`, comments will be included with ``jprops.COMMENT``
in place of the key.

:param fh: a readable file-like object
:param comments: should include comments (default: False)


<EOL>if '<STR_LIT>' in getattr(fp, '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>for line in fp:<EOL><INDENT>yield line<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for line in fp:<EOL><INDENT>line = line.replace(b'<STR_LIT:\r\n>', b'<STR_LIT:\n>').replace(b'<STR_LIT:\r>', b'<STR_LIT:\n>')<EOL>for piece in line.split(b'<STR_LIT:\n>'):<EOL><INDENT>yield piece<EOL><DEDENT><DEDENT><DEDENT>

Wrap a file to convert newlines regardless of whether the file was opened
with the "universal newlines" option or not.


with open(filename,'<STR_LIT:r>') as f:<EOL><INDENT>f = f.readlines()<EOL>null=[f.pop(<NUM_LIT:0>) for i in range(skipline)]<EOL>commentfilter = make_commentfilter(comment)<EOL>if not asStruct:<EOL><INDENT>asStruct = asRecArray<EOL><DEDENT>if namecomment is False and (names or asdict or asStruct):<EOL><INDENT>while <NUM_LIT:1>:<EOL><INDENT>line = f.pop(<NUM_LIT:0>)<EOL>if line[<NUM_LIT:0>] != comment:<EOL><INDENT>nameline = line<EOL>if header_badchars:<EOL><INDENT>for c in header_badchars:<EOL><INDENT>nameline = nameline.replace(c,'<STR_LIT:U+0020>')<EOL><DEDENT><DEDENT>nms=nameline.split(fsep)<EOL>break<EOL><DEDENT>elif len(f) == <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if names or asdict or asStruct:<EOL><INDENT>if type(names) == type(<NUM_LIT:1>):<EOL><INDENT>nameline = f.pop(names)<EOL><DEDENT>else:<EOL><INDENT>nameline = f.pop(<NUM_LIT:0>)<EOL><DEDENT>if nameline[<NUM_LIT:0>]==comment:<EOL><INDENT>nameline = nameline[<NUM_LIT:1>:]<EOL><DEDENT>if header_badchars:<EOL><INDENT>for c in header_badchars:<EOL><INDENT>nameline = nameline.replace(c,'<STR_LIT:U+0020>')<EOL><DEDENT><DEDENT>nms=list([name.strip() for name in nameline.split(fsep)])<EOL><DEDENT><DEDENT>null=[f.pop(<NUM_LIT:0>) for i in range(skipafter)]<EOL>if fixedformat:<EOL><INDENT>myreadff = lambda x: readff(x,fixedformat)<EOL>splitarr = list(map(myreadff,f))<EOL>splitarr = list(filter(commentfilter,splitarr))<EOL><DEDENT>else:<EOL><INDENT>fstrip = list(map(str.strip,f))<EOL>fseps = [ fsep for i in range(len(f)) ]<EOL>splitarr = list(map(str.split,fstrip,fseps))<EOL>if removeblanks:<EOL><INDENT>for i in range(splitarr.count(['<STR_LIT>'])):<EOL><INDENT>splitarr.remove(['<STR_LIT>'])<EOL><DEDENT><DEDENT>splitarr = list(filter(commentfilter,splitarr))<EOL>nperline = list(map(len,splitarr))<EOL>if hasmode:<EOL><INDENT>ncols,nrows = mode(nperline)<EOL>if nrows != len(splitarr):<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>" % (len(splitarr) - nrows,ncols,nrows))<EOL><DEDENT>for i in range(len(splitarr)-<NUM_LIT:1>,-<NUM_LIT:1>,-<NUM_LIT:1>):  <EOL><INDENT>if nperline[i] != ncols:<EOL><INDENT>splitarr.pop(i)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>x = numpy.asarray( splitarr , dtype='<STR_LIT:float>')<EOL><DEDENT>except ValueError:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>" % '<STR_LIT:float>')<EOL><DEDENT>try:<EOL><INDENT>x = numpy.asarray( splitarr , dtype='<STR_LIT:S>')<EOL><DEDENT>except ValueError:<EOL><INDENT>if hasmode:<EOL><INDENT>raise Exception( "<STR_LIT>" +"<STR_LIT>" +"<STR_LIT>" )<EOL><DEDENT>else:<EOL><INDENT>raise Exception( "<STR_LIT>" +"<STR_LIT>" +"<STR_LIT>" )<EOL><DEDENT><DEDENT><DEDENT>if nullval is not None:<EOL><INDENT>x[x==nullval] = numpy.nan<EOL>x = get_autotype(x)<EOL><DEDENT>if asdict or asStruct:<EOL><INDENT>mydict = OrderedDict(zip(nms,x.T))<EOL>for k,v in mydict.items():<EOL><INDENT>mydict[k] = get_autotype(v)<EOL><DEDENT>if asdict:<EOL><INDENT>return mydict<EOL><DEDENT>elif asRecArray:<EOL><INDENT>return Struct(mydict).as_recarray()<EOL><DEDENT>elif asStruct:<EOL><INDENT>return Struct(mydict)<EOL><DEDENT><DEDENT>elif names and twod:<EOL><INDENT>return nms,x<EOL><DEDENT>elif names:<EOL><INDENT>return nms,[ get_autotype(x.T[i]) for i in range(x.shape[<NUM_LIT:1>]) ]<EOL><DEDENT>else:<EOL><INDENT>if twod:<EOL><INDENT>return x<EOL><DEDENT>else:<EOL><INDENT>return [ get_autotype(x.T[i]) for i in range(x.shape[<NUM_LIT:1>]) ]<EOL><DEDENT><DEDENT><DEDENT>

The default return is a two dimensional float array.  If you want a list of
columns output instead of a 2D array, pass 'twod=False'.  In this case,
each column's data type will be automatically detected.

Example usage:
CASE 1) a table has the format:
 X    Y    Z
0.0  2.4  8.2
1.0  3.4  5.6
0.7  3.2  2.1
...
names,(x,y,z)=readcol("myfile.tbl",names=True,twod=False)
or
x,y,z=readcol("myfile.tbl",skipline=1,twod=False)
or
names,xx = readcol("myfile.tbl",names=True)
or
xxdict = readcol("myfile.tbl",asdict=True)
or
xxstruct = readcol("myfile.tbl",asStruct=True)

CASE 2) no title is contained into the table, then there is
no need to skipline:
x,y,z=readcol("myfile.tbl")

CASE 3) there is a names column and then more descriptive text:
 X      Y     Z
(deg) (deg) (km/s)
0.0    2.4   8.2
1.0    3.4.  5.6
...
then use:
names,x,y,z=readcol("myfile.tbl",names=True,skipline=1,twod=False)
or
x,y,z=readcol("myfile.tbl",skipline=2,twod=False)

INPUTS:
    fsep - field separator, e.g. for comma separated value (csv) files
    skipline - number of lines to ignore at the start of the file
    names - read / don't read in the first line as a list of column names
            can specify an integer line number too, though it will be
            the line number after skipping lines
    twod - two dimensional or one dimensional output
    nullval - if specified, all instances of this value will be replaced
       with a floating NaN
    asdict - zips names with data to create a dict with column headings
        tied to column data.  If asdict=True, names will be set to True
    asStruct - same as asdict, but returns a structure instead of a dictionary
        (i.e. you call struct.key instead of struct['key'])
    fixedformat - if you have a fixed format file, this is a python list of
        column lengths.  e.g. the first table above would be [3,5,5].  Note
        that if you specify the wrong fixed format, you will get junk; if your
        format total is greater than the line length, the last entries will all
        be blank but readcol will not report an error.
    namecomment - assumed that "Name" row is on a comment line.  If it is not -
        e.g., it is the first non-comment line, change this to False
    removeblanks - remove all blank entries from split lines.  This can cause lost
        data if you have blank entries on some lines.
    header_badchars - remove these characters from a header before parsing it
        (helpful for IPAC tables that are delimited with | )

If you get this error: "scipy could not be imported.  Your table must have
full rows." it means readcol cannot automatically guess which columns
contain data.  If you have scipy and columns of varying length, readcol will
read in all of the rows with length=mode(row lengths).


try:<EOL><INDENT>narr = arr.astype('<STR_LIT:float>')<EOL>if (narr < sys.maxsize).all() and (narr % <NUM_LIT:1>).sum() == <NUM_LIT:0>:<EOL><INDENT>return narr.astype('<STR_LIT:int>')<EOL><DEDENT>else:<EOL><INDENT>return narr<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>return arr<EOL><DEDENT>

Attempts to return a numpy array converted to the most sensible dtype
Value errors will be caught and simply return the original array
Tries to make dtype int, then float, then no change


F = numpy.array([<NUM_LIT:0>]+format).cumsum()<EOL>bothF = zip(F[:-<NUM_LIT:1>],F[<NUM_LIT:1>:])<EOL>strarr = [s[l:u] for l,u in bothF]<EOL>return strarr<EOL>

Fixed-format reader
Pass in a single line string (s) and a format list,
which needs to be a python list of string lengths


self.__dict__[name] = data<EOL>

Add a new column (attribute) to the struct
(will overwrite anything with the same name)


dtype = [(k,v.dtype) for k,v in self.__dict__.iteritems()]<EOL>R = numpy.recarray(len(self.__dict__[k]),dtype=dtype)<EOL>for key in self.__dict__:<EOL><INDENT>R[key] = self.__dict__[key]<EOL><DEDENT>return R<EOL>


DESC = '<STR_LIT>'                                <EOL>HEAD = '<STR_LIT>'        <EOL>keys = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:filename>')<EOL>f = open(filename, '<STR_LIT:rb>')<EOL>"""<STR_LIT>"""<EOL>firstbytes = struct.unpack('<STR_LIT:I>',f.read(<NUM_LIT:4>))<EOL>if firstbytes[<NUM_LIT:0>] == <NUM_LIT:8>:<EOL><INDENT>gtype = <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>gtype = <NUM_LIT:1><EOL><DEDENT>if gtype == <NUM_LIT:2>:<EOL><INDENT>f.seek(<NUM_LIT:16>)<EOL><DEDENT>else:<EOL><INDENT>f.seek(<NUM_LIT:0>)<EOL><DEDENT>if seek is not None:<EOL><INDENT>f.seek(seek)<EOL><DEDENT>raw = struct.unpack(HEAD,f.read(<NUM_LIT>))[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>values = (raw[:<NUM_LIT:6>], raw[<NUM_LIT:6>:<NUM_LIT:12>]) + raw[<NUM_LIT:12>:<NUM_LIT:16>] + (raw[<NUM_LIT:16>:<NUM_LIT>],) + raw[<NUM_LIT>:<NUM_LIT:30>] + (raw[<NUM_LIT:30>:<NUM_LIT>], raw[<NUM_LIT>], filename)<EOL>header = dict(list(zip(keys, values)))<EOL>f.close()<EOL>if gtype == <NUM_LIT:2>:<EOL><INDENT>posoffset = (<NUM_LIT:2>*<NUM_LIT:16> + (<NUM_LIT:8> + <NUM_LIT>))<EOL><DEDENT>else:<EOL><INDENT>posoffset = (<NUM_LIT:8> + <NUM_LIT>)<EOL><DEDENT>Npart = sum(header['<STR_LIT>'])<EOL>if gtype == <NUM_LIT:2>:<EOL><INDENT>veloffset = <NUM_LIT:3>*<NUM_LIT:16> + (<NUM_LIT:8> + <NUM_LIT>) + (<NUM_LIT:8> + <NUM_LIT:3>*<NUM_LIT:4>*Npart)<EOL><DEDENT>else:<EOL><INDENT>veloffset= (<NUM_LIT:8> + <NUM_LIT>) + (<NUM_LIT:8> + <NUM_LIT:3>*<NUM_LIT:4>*Npart)<EOL><DEDENT>return Npart, posoffset+<NUM_LIT:4>, veloffset+<NUM_LIT:4>, header<EOL>

Read header data from Gadget data file 'filename' with Gadget file
    type 'gtype'. Returns offsets of positions and velocities.


def parallelize(cores=None, fork=True, flatten=False, info=False, infoclass=InfoThreadProgressBar, init=None, *args, **kwargs):

if cores is None:<EOL><INDENT>cores = multiprocessing.cpu_count()<EOL><DEDENT>def wrapper(f):<EOL><INDENT>def execute(*multiargs):<EOL><INDENT>results = []<EOL>len(list(zip(*multiargs)))<EOL>N = len(multiargs[<NUM_LIT:0>])<EOL>if info:<EOL><INDENT>print("<STR_LIT>" % (N, cores))<EOL><DEDENT>taskQueue = queue.Queue(len(multiargs[<NUM_LIT:0>]))<EOL>for tasknr, _args in enumerate(zip(*multiargs)):<EOL><INDENT>taskQueue.put((tasknr, list(_args)))<EOL><DEDENT>executions = [Execution(taskQueue, fork, f, init, corenr, args, kwargs) for corenr in range(cores)]<EOL>if info:<EOL><INDENT>infoobj = infoclass(len(multiargs[<NUM_LIT:0>]), executions)<EOL>infoobj.start()<EOL><DEDENT>for i, execution in enumerate(executions):<EOL><INDENT>execution.setName("<STR_LIT>" % i)<EOL>execution.start()<EOL><DEDENT>error = False<EOL>for execution in executions:<EOL><INDENT>log("<STR_LIT>", execution.getName())<EOL>try:<EOL><INDENT>execution.join()<EOL><DEDENT>except BaseException:<EOL><INDENT>error = True<EOL><DEDENT>results.extend(execution.results)<EOL>if execution.error:<EOL><INDENT>error = True<EOL><DEDENT><DEDENT>if info:<EOL><INDENT>infoobj.join()<EOL><DEDENT>if error:<EOL><INDENT>print("<STR_LIT:error>", file=sys.stderr)<EOL>results = None<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>results.sort(cmp=lambda a, b: cmp(a[<NUM_LIT:0>], b[<NUM_LIT:0>]))<EOL>results = [k[<NUM_LIT:1>] for k in results]<EOL>if flatten:<EOL><INDENT>flatresults = []<EOL>for result in results:<EOL><INDENT>flatresults.extend(result)<EOL><DEDENT>results = flatresults<EOL><DEDENT><DEDENT>return results<EOL><DEDENT>return execute<EOL><DEDENT>return wrapper<EOL>

Function decorater that executes the function in parallel

    Usage::

            @parallelize(cores=10, info=True)
            def f(x):
                    return x**2

            x = numpy.arange(0, 100, 0.1)
            y = f(x) # this gets executed parallel

    :param cores: number of cpus/cores to use (if None, it counts the cores using /proc/cpuinfo)
    :param fork: fork a process (should always be true since of the GIT, but can be used with c modules that release the GIT)
    :param flatten: if False and each return value is a list, final result will be a list of lists, if True, all lists are combined to one big list
    :param info: show progress bar (see infoclass)
    :param infoclass: class to instantiate that shows the progress (default shows progressbar)
    :param init: function to be called in each forked process before executing, can be used to set the seed, takes a integer as parameter (number that identifies the process)
    :param args: extra arguments passed to function
    :param kwargs: extra keyword arguments passed to function

    Example::

            @parallelize(cores=10, info=True, n=2)
            def f(x, n):
                    return x**n

            x = numpy.arange(0, 100, 0.1)
            y = f(x) # this gets executed parallel


if cores == None:<EOL><INDENT>cores = multiprocessing.cpu_count()<EOL><DEDENT>def wrapper(f):<EOL><INDENT>def execute(*multiargs):<EOL><INDENT>results = []<EOL>len(list(zip(*multiargs)))<EOL>N = len(multiargs[<NUM_LIT:0>])<EOL>if info:<EOL><INDENT>print("<STR_LIT>" % (N, cores))<EOL><DEDENT>taskQueue = queue.Queue(len(multiargs[<NUM_LIT:0>]))<EOL>for tasknr, _args in enumerate(zip(*multiargs)):<EOL><INDENT>taskQueue.put((tasknr, list(_args)))<EOL><DEDENT>executions = [Execution(taskQueue, fork, f, init, corenr, args, kwargs) for corenr in range(cores)]<EOL>if info:<EOL><INDENT>infoobj = infoclass(len(multiargs[<NUM_LIT:0>]), executions)<EOL>infoobj.start()<EOL><DEDENT>for i, execution in enumerate(executions):<EOL><INDENT>execution.setName("<STR_LIT>" % i)<EOL>execution.start()<EOL><DEDENT>error = False<EOL>for execution in executions:<EOL><INDENT>log("<STR_LIT>",execution.getName())<EOL>try:<EOL><INDENT>execution.join()<EOL><DEDENT>except BaseException:<EOL><INDENT>error = True<EOL><DEDENT>results.extend(execution.results)<EOL>if execution.error:<EOL><INDENT>error = True <EOL><DEDENT><DEDENT>if info:<EOL><INDENT>infoobj.join()<EOL><DEDENT>if error:<EOL><INDENT>print("<STR_LIT:error>", file=sys.stderr)<EOL>results = None<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>results.sort(cmp=lambda a, b: cmp(a[<NUM_LIT:0>], b[<NUM_LIT:0>]))<EOL>results = [k[<NUM_LIT:1>] for k in results]<EOL>if flatten:<EOL><INDENT>flatresults = []<EOL>for result in results:<EOL><INDENT>flatresults.extend(result)<EOL><DEDENT>results = flatresults<EOL><DEDENT><DEDENT>return results<EOL><DEDENT>return execute<EOL><DEDENT>return wrapper<EOL>


def __init__(self, min_value, max_value, format="<STR_LIT>", width=<NUM_LIT>, barchar="<STR_LIT:#>", emptychar="<STR_LIT:->", output=sys.stdout):

self.min_value = min_value<EOL>self.max_value = max_value<EOL>self.format = format<EOL>self.width = width<EOL>self.barchar = barchar<EOL>self.emptychar = emptychar<EOL>self.output = output<EOL>self.firsttime = True<EOL>self.prevtime = time.time()<EOL>self.starttime = self.prevtime<EOL>self.prevfraction = <NUM_LIT:0><EOL>self.firsttimedone = False<EOL>self.value = self.min_value<EOL>

:param min_value: minimum value for update(..)
:param format: format specifier for the output
:param width: width of the progress bar's (excluding extra text)
:param barchar: character used to print the bar
:param output: where to write the output to


def _export(dataset_input, dataset_output, random_index_column, path, column_names=None, byteorder="<STR_LIT:=>", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):

if selection:<EOL><INDENT>if selection == True:  <EOL><INDENT>selection = "<STR_LIT:default>"<EOL><DEDENT><DEDENT>N = len(dataset_input) if not selection else dataset_input.selected_length(selection)<EOL>if N == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if shuffle and sort:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if shuffle:<EOL><INDENT>shuffle_array = dataset_output.columns[random_index_column]<EOL><DEDENT>partial_shuffle = shuffle and len(dataset_input) != N<EOL>order_array = None<EOL>order_array_inverse = None<EOL>has_strings = any([dataset_input.dtype(k) == str_type for k in column_names])<EOL>if partial_shuffle:<EOL><INDENT>shuffle_array_full = np.random.choice(len(dataset_input), len(dataset_input), replace=False)<EOL>shuffle_array[:] = shuffle_array_full[shuffle_array_full < N]<EOL>del shuffle_array_full<EOL>order_array = shuffle_array<EOL><DEDENT>elif shuffle:<EOL><INDENT>shuffle_array_memory = np.random.choice(N, N, replace=False)<EOL>shuffle_array[:] = shuffle_array_memory<EOL>order_array = shuffle_array<EOL><DEDENT>if order_array is not None:<EOL><INDENT>indices_r = np.zeros_like(order_array)<EOL>indices_r[order_array] = np.arange(len(order_array))<EOL>order_array_inverse = indices_r<EOL>del indices_r<EOL><DEDENT>if sort:<EOL><INDENT>if selection:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>indices = np.argsort(dataset_input.evaluate(sort))<EOL>indices_r = np.zeros_like(indices)<EOL>indices_r[indices] = np.arange(len(indices))<EOL>if has_strings:<EOL><INDENT>order_array_inverse = indices if ascending else indices[:--<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>del indices<EOL><DEDENT>order_array = indices_r if ascending else indices_r[::-<NUM_LIT:1>]<EOL>logger.info("<STR_LIT>")<EOL><DEDENT>if progress == True:<EOL><INDENT>progress = vaex.utils.progressbar_callable(title="<STR_LIT>")<EOL><DEDENT>progress = progress or (lambda value: True)<EOL>progress_total = len(column_names) * len(dataset_input)<EOL>progress_status = ProgressStatus()<EOL>progress_status.cancelled = False<EOL>progress_status.value = <NUM_LIT:0><EOL>if selection:<EOL><INDENT>full_mask = dataset_input.evaluate_selection_mask(selection)<EOL><DEDENT>else:<EOL><INDENT>full_mask = None<EOL><DEDENT>sparse_groups = collections.defaultdict(list)<EOL>sparse_matrices = {}  <EOL>string_columns = []<EOL>futures = []<EOL>thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=<NUM_LIT:1>) <EOL>if True:<EOL><INDENT>for column_name in column_names:<EOL><INDENT>sparse_matrix = dataset_output._sparse_matrix(column_name)<EOL>if sparse_matrix is not None:<EOL><INDENT>sparse_groups[id(sparse_matrix)].append(column_name)<EOL>sparse_matrices[id(sparse_matrix)] = sparse_matrix<EOL>continue<EOL><DEDENT>logger.debug("<STR_LIT>" % column_name)<EOL>future = thread_pool.submit(_export_column, dataset_input, dataset_output, column_name, full_mask,<EOL>shuffle, sort, selection, N, order_array, order_array_inverse, progress_status)<EOL>futures.append(future)<EOL><DEDENT><DEDENT>done = False<EOL>while not done:<EOL><INDENT>done = True<EOL>for future in futures:<EOL><INDENT>try:<EOL><INDENT>future.result(<NUM_LIT:0.1>/<NUM_LIT:4>)<EOL><DEDENT>except concurrent.futures.TimeoutError:<EOL><INDENT>done = False<EOL>break<EOL><DEDENT><DEDENT>if not done:<EOL><INDENT>if not progress(progress_status.value / float(progress_total)):<EOL><INDENT>progress_status.cancelled = True<EOL><DEDENT><DEDENT><DEDENT>for sparse_matrix_id, column_names in sparse_groups.items():<EOL><INDENT>sparse_matrix = sparse_matrices[sparse_matrix_id]<EOL>for column_name in column_names:<EOL><INDENT>assert not shuffle<EOL>assert selection in [None, False]<EOL>column = dataset_output.columns[column_name]<EOL>column.matrix.data[:] = dataset_input.columns[column_name].matrix.data<EOL>column.matrix.indptr[:] = dataset_input.columns[column_name].matrix.indptr<EOL>column.matrix.indices[:] = dataset_input.columns[column_name].matrix.indices<EOL><DEDENT><DEDENT>return column_names<EOL>

:param DatasetLocal dataset: dataset to export
:param str path: path for file
:param lis[str] column_names: list of column names to export or None for all columns
:param str byteorder: = for native, < for little endian and > for big endian
:param bool shuffle: export rows in random order
:param bool selection: export selection or not
:param progress: progress callback that gets a progress fraction as argument and should return True to continue
:return:


def export_fits(dataset, path, column_names=None, shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):

if shuffle:<EOL><INDENT>random_index_name = "<STR_LIT>"<EOL>while random_index_name in dataset.get_column_names():<EOL><INDENT>random_index_name += "<STR_LIT>"<EOL><DEDENT><DEDENT>column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True)<EOL>logger.debug("<STR_LIT>" % column_names)<EOL>N = len(dataset) if not selection else dataset.selected_length(selection)<EOL>data_types = []<EOL>data_shapes = []<EOL>ucds = []<EOL>units = []<EOL>for column_name in column_names:<EOL><INDENT>if column_name in dataset.get_column_names(strings=True):<EOL><INDENT>column = dataset.columns[column_name]<EOL>shape = (N,) + column.shape[<NUM_LIT:1>:]<EOL>dtype = column.dtype<EOL>if dataset.dtype(column_name) == str_type:<EOL><INDENT>max_length = dataset[column_name].apply(lambda x: len(x)).max(selection=selection)<EOL>dtype = np.dtype('<STR_LIT:S>'+str(int(max_length)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>dtype = np.float64().dtype<EOL>shape = (N,)<EOL><DEDENT>ucds.append(dataset.ucds.get(column_name))<EOL>units.append(dataset.units.get(column_name))<EOL>data_types.append(dtype)<EOL>data_shapes.append(shape)<EOL><DEDENT>if shuffle:<EOL><INDENT>column_names.append(random_index_name)<EOL>data_types.append(np.int64().dtype)<EOL>data_shapes.append((N,))<EOL>ucds.append(None)<EOL>units.append(None)<EOL><DEDENT>else:<EOL><INDENT>random_index_name = None<EOL><DEDENT>null_values = {key: dataset.columns[key].fill_value for key in dataset.get_column_names() if dataset.is_masked(key) and dataset.dtype(key).kind != "<STR_LIT:f>"}<EOL>vaex.file.colfits.empty(path, N, column_names, data_types, data_shapes, ucds, units, null_values=null_values)<EOL>if shuffle:<EOL><INDENT>del column_names[-<NUM_LIT:1>]<EOL>del data_types[-<NUM_LIT:1>]<EOL>del data_shapes[-<NUM_LIT:1>]<EOL><DEDENT>dataset_output = vaex.file.other.FitsBinTable(path, write=True)<EOL>_export(dataset_input=dataset, dataset_output=dataset_output, path=path, random_index_column=random_index_name,<EOL>column_names=column_names, selection=selection, shuffle=shuffle,<EOL>progress=progress, sort=sort, ascending=ascending)<EOL>dataset_output.close_files()<EOL>

:param DatasetLocal dataset: dataset to export
:param str path: path for file
:param lis[str] column_names: list of column names to export or None for all columns
:param bool shuffle: export rows in random order
:param bool selection: export selection or not
:param progress: progress callback that gets a progress fraction as argument and should return True to continue,
        or a default progress bar when progress=True
:param: bool virtual: When True, export virtual columns
:return:


depending = set()<EOL>for expression in self.expressions:<EOL><INDENT>expression = ds._expr(expression)  <EOL>depending |= expression.variables()<EOL><DEDENT>if self.previous_selection:<EOL><INDENT>depending |= self.previous_selection._depending_columns(ds)<EOL><DEDENT>return depending<EOL>

Find all columns that this selection depends on for df ds


return StringOperations(self)<EOL>


return StringOperationsPandas(self)<EOL>

Gives access to string operations (using Pandas Series)


expression = self.expression<EOL>def walk(node):<EOL><INDENT>if isinstance(node, six.string_types):<EOL><INDENT>if node in self.ds.virtual_columns:<EOL><INDENT>ex = Expression(self.ds, self.ds.virtual_columns[node])<EOL>return [node, None, None, [ex._graph()]]<EOL><DEDENT>else:<EOL><INDENT>return node<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fname, node_repr, deps = node<EOL>if len(node_repr) > <NUM_LIT:30>:  <EOL><INDENT>node_repr = node_repr[:<NUM_LIT>] + '<STR_LIT>'<EOL><DEDENT>deps = [walk(dep) for dep in deps]<EOL>obj = self.ds.functions.get(fname)<EOL>if isinstance(obj, Function):<EOL><INDENT>obj = obj.f<EOL><DEDENT>if isinstance(obj, FunctionSerializablePickle):<EOL><INDENT>obj = obj.f<EOL><DEDENT>return [node_repr, fname, obj, deps]<EOL><DEDENT><DEDENT>return walk(expresso._graph(expression))<EOL>

Return a graph containing the dependencies of this expression
        Structure is:
            [<string expression>, <function name if callable>, <function object if callable>, [subgraph/dependencies, ....]]


from graphviz import Graph, Digraph<EOL>node = self._graph()<EOL>dot = dot or Digraph(comment=self.expression)<EOL>def walk(node):<EOL><INDENT>if isinstance(node, six.string_types):<EOL><INDENT>dot.node(node, node)<EOL>return node, node<EOL><DEDENT>else:<EOL><INDENT>node_repr, fname, fobj, deps = node<EOL>node_id = node_repr<EOL>dot.node(node_id, node_repr)<EOL>for dep in deps:<EOL><INDENT>dep_id, dep = walk(dep)<EOL>dot.edge(node_id, dep_id)<EOL><DEDENT>return node_id, node<EOL><DEDENT><DEDENT>walk(node)<EOL>return dot<EOL>

Return a graphviz.Digraph object with a graph of the expression


return self.evaluate().tolist()<EOL>


def count(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, edges=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.count(**kwargs)<EOL>

Shortcut for ds.count(expression, ...), see `Dataset.count`


def sum(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.sum(**kwargs)<EOL>

Shortcut for ds.sum(expression, ...), see `Dataset.sum`


def mean(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.mean(**kwargs)<EOL>

Shortcut for ds.mean(expression, ...), see `Dataset.mean`


def std(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.std(**kwargs)<EOL>

Shortcut for ds.std(expression, ...), see `Dataset.std`


def var(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.var(**kwargs)<EOL>

Shortcut for ds.std(expression, ...), see `Dataset.var`


def minmax(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.minmax(**kwargs)<EOL>

Shortcut for ds.minmax(expression, ...), see `Dataset.minmax`


def min(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.min(**kwargs)<EOL>

Shortcut for ds.min(expression, ...), see `Dataset.min`


def max(self, binby=[], limits=None, shape=default_shape, selection=False, delay=False, progress=None):

kwargs = dict(locals())<EOL>del kwargs['<STR_LIT>']<EOL>kwargs['<STR_LIT>'] = self.expression<EOL>return self.ds.max(**kwargs)<EOL>

Shortcut for ds.max(expression, ...), see `Dataset.max`


return self.ds.nop(self.expression)<EOL>

Evaluates expression, and drop the result, usefull for benchmarking, since vaex is usually lazy


return self.expand().expression not in self.ds.columns<EOL>

If this expression is not transient (e.g. on disk) optimizations can be made


return self.ds.is_masked(self.expression)<EOL>


def value_counts(self, dropna=False, dropnull=True, ascending=False, progress=False):

from pandas import Series<EOL>dtype = self.dtype<EOL>transient = self.transient or self.ds.filtered or self.ds.is_masked(self.expression)<EOL>if self.dtype == str_type and not transient:<EOL><INDENT>ar = self.ds.columns[self.expression]<EOL>if not isinstance(ar, ColumnString):<EOL><INDENT>transient = True<EOL><DEDENT><DEDENT>counter_type = counter_type_from_dtype(self.dtype, transient)<EOL>counters = [None] * self.ds.executor.thread_pool.nthreads<EOL>def map(thread_index, i1, i2, ar):<EOL><INDENT>if counters[thread_index] is None:<EOL><INDENT>counters[thread_index] = counter_type()<EOL><DEDENT>if dtype == str_type:<EOL><INDENT>previous_ar = ar<EOL>ar = _to_string_sequence(ar)<EOL>if not transient:<EOL><INDENT>assert ar is previous_ar.string_sequence<EOL><DEDENT><DEDENT>if np.ma.isMaskedArray(ar):<EOL><INDENT>mask = np.ma.getmaskarray(ar)<EOL>counters[thread_index].update(ar, mask)<EOL><DEDENT>else:<EOL><INDENT>counters[thread_index].update(ar)<EOL><DEDENT>return <NUM_LIT:0><EOL><DEDENT>def reduce(a, b):<EOL><INDENT>return a+b<EOL><DEDENT>self.ds.map_reduce(map, reduce, [self.expression], delay=False, progress=progress, name='<STR_LIT>', info=True, to_numpy=False)<EOL>counters = [k for k in counters if k is not None]<EOL>counter0 = counters[<NUM_LIT:0>]<EOL>for other in counters[<NUM_LIT:1>:]:<EOL><INDENT>counter0.merge(other)<EOL><DEDENT>value_counts = counter0.extract()<EOL>index = np.array(list(value_counts.keys()))<EOL>counts = np.array(list(value_counts.values()))<EOL>order = np.argsort(counts)<EOL>if not ascending:<EOL><INDENT>order = order[::-<NUM_LIT:1>]<EOL><DEDENT>counts = counts[order]<EOL>index = index[order]<EOL>if not dropna or not dropnull:<EOL><INDENT>index = index.tolist()<EOL>counts = counts.tolist()<EOL>if not dropna and counter0.nan_count:<EOL><INDENT>index = [np.nan] + index<EOL>counts = [counter0.nan_count] + counts<EOL><DEDENT>if not dropnull and counter0.null_count:<EOL><INDENT>index = ['<STR_LIT:null>'] + index<EOL>counts = [counter0.null_count] + counts<EOL><DEDENT><DEDENT>return Series(counts, index=index)<EOL>

Computes counts of unique values.

         WARNING:
          * If the expression/column is not categorical, it will be converted on the fly
          * dropna is False by default, it is True by default in pandas

        :param dropna: when True, it will not report the missing values
        :param ascending: when False (default) it will report the most frequent occuring item first
        :returns: Pandas series containing the counts


def map(self, mapper, nan_mapping=None, null_mapping=None):

assert isinstance(mapper, collectionsAbc.Mapping), "<STR_LIT>"<EOL>df = self.ds<EOL>mapper_keys = np.array(list(mapper.keys()))<EOL>key_set = df._set(self.expression)<EOL>found_keys = key_set.keys()<EOL>mapper_has_nan = any([key != key for key in mapper_keys])<EOL>if not set(mapper_keys).issuperset(found_keys):<EOL><INDENT>missing = set(found_keys).difference(mapper_keys)<EOL>missing0 = list(missing)[<NUM_LIT:0>]<EOL>if missing0 == missing0:  <EOL><INDENT>raise ValueError('<STR_LIT>' % missing)<EOL><DEDENT><DEDENT>choices = [mapper[key] for key in found_keys]<EOL>if key_set.has_nan:<EOL><INDENT>if mapper_has_nan:<EOL><INDENT>choices = [mapper[np.nan]] + choices<EOL><DEDENT>else:<EOL><INDENT>choices = [nan_mapping] + choices<EOL><DEDENT><DEDENT>if key_set.has_null:<EOL><INDENT>choices = [null_mapping] + choices<EOL><DEDENT>choices = np.array(choices)<EOL>key_set_name = df.add_variable('<STR_LIT>', key_set, unique=True)<EOL>choices_name = df.add_variable('<STR_LIT>', choices, unique=True)<EOL>expr = '<STR_LIT>'.format(self, key_set_name, choices_name)<EOL>return Expression(df, expr)<EOL>

Map values of an expression or in memory column accoring to an input
        dictionary or a custom callable function.

        Example:

        >>> import vaex
        >>> df = vaex.from_arrays(color=['red', 'red', 'blue', 'red', 'green'])
        >>> mapper = {'red': 1, 'blue': 2, 'green': 3}
        >>> df['color_mapped'] = df.color.map(mapper)
        >>> df
        #  color      color_mapped
        0  red                   1
        1  red                   1
        2  blue                  2
        3  red                   1
        4  green                 3
        >>> import numpy as np
        >>> df = vaex.from_arrays(type=[0, 1, 2, 2, 2, np.nan])
        >>> df['role'] = df['type'].map({0: 'admin', 1: 'maintainer', 2: 'user', np.nan: 'unknown'})
        >>> df
        #    type  role
        0       0  admin
        1       1  maintainer
        2       2  user
        3       2  user
        4       2  user
        5     nan  unknown

        :param mapper: dict like object used to map the values from keys to values
        :param nan_mapping: value to be used when a nan is present (and not in the mapper)
        :param null_mapping: value to use used when there is a missing value
        :return: A vaex expression
        :rtype: vaex.expression.Expression


return self.f(*args, **kwargs)<EOL>


def register_function(scope=None, as_property=False, name=None):

prefix = '<STR_LIT>'<EOL>if scope:<EOL><INDENT>prefix = scope + "<STR_LIT:_>"<EOL>if scope not in scopes:<EOL><INDENT>raise KeyError("<STR_LIT>")<EOL><DEDENT><DEDENT>def wrapper(f, name=name):<EOL><INDENT>name = name or f.__name__<EOL>if name.startswith(prefix):<EOL><INDENT>name = name[len(prefix):]<EOL><DEDENT>full_name = prefix + name<EOL>if scope:<EOL><INDENT>def closure(name=name, full_name=full_name, function=f):<EOL><INDENT>def wrapper(self, *args, **kwargs):<EOL><INDENT>lazy_func = getattr(self.expression.ds.func, full_name)<EOL>args = (self.expression, ) + args<EOL>return lazy_func(*args, **kwargs)<EOL><DEDENT>return functools.wraps(function)(wrapper)<EOL><DEDENT>if as_property:<EOL><INDENT>setattr(scopes[scope], name, property(closure()))<EOL><DEDENT>else:<EOL><INDENT>setattr(scopes[scope], name, closure())<EOL><DEDENT><DEDENT>else:<EOL><INDENT>def closure(name=name, full_name=full_name, function=f):<EOL><INDENT>def wrapper(self, *args, **kwargs):<EOL><INDENT>lazy_func = getattr(self.ds.func, full_name)<EOL>args = (self, ) + args<EOL>return lazy_func(*args, **kwargs)<EOL><DEDENT>return functools.wraps(function)(wrapper)<EOL><DEDENT>setattr(vaex.expression.Expression, name, closure())<EOL><DEDENT>vaex.expression.expression_namespace[prefix + name] = f<EOL>return f  <EOL><DEDENT>return wrapper<EOL>

Decorator to register a new function with vaex.

    Example:

    >>> import vaex
    >>> df = vaex.example()
    >>> @vaex.register_function()
    >>> def invert(x):
    >>>     return 1/x
    >>> df.x.invert()

    >>> import numpy as np
    >>> df = vaex.from_arrays(departure=np.arange('2015-01-01', '2015-12-05', dtype='datetime64'))
    >>> @vaex.register_function(as_property=True, scope='dt')
    >>> def dt_relative_day(x):
    >>>     return vaex.functions.dt_dayofyear(x)/365.
    >>> df.departure.dt.relative_day


@register_function()<EOL>def fillna(ar, value, fill_nan=True, fill_masked=True):

ar = ar if not isinstance(ar, column.Column) else ar.to_numpy()<EOL>if ar.dtype.kind in '<STR_LIT:O>' and fill_nan:<EOL><INDENT>strings = ar.astype(str)<EOL>mask = strings == '<STR_LIT>'<EOL>ar = ar.copy()<EOL>ar[mask] = value<EOL><DEDENT>elif ar.dtype.kind in '<STR_LIT:f>' and fill_nan:<EOL><INDENT>mask = np.isnan(ar)<EOL>if np.any(mask):<EOL><INDENT>ar = ar.copy()<EOL>ar[mask] = value<EOL><DEDENT><DEDENT>if fill_masked and np.ma.isMaskedArray(ar):<EOL><INDENT>mask = ar.mask<EOL>if np.any(mask):<EOL><INDENT>ar = ar.data.copy()<EOL>ar[mask] = value<EOL><DEDENT><DEDENT>return ar<EOL>

Returns an array where missing values are replaced by value.

    If the dtype is object, nan values and 'nan' string values
    are replaced by value when fill_nan==True.


import pandas as pd<EOL>return pd.Series(x).dt.dayofweek.values<EOL>

Obtain the day of the week with Monday=0 and Sunday=6

    :returns: an expression containing the day of week.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.dayofweek
    Expression = dt_dayofweek(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  0
    1  3
    2  3


import pandas as pd<EOL>return pd.Series(x).dt.dayofyear.values<EOL>

The ordinal day of the year.

    :returns: an expression containing the ordinal day of the year.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.dayofyear
    Expression = dt_dayofyear(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  285
    1   42
    2  316


import pandas as pd<EOL>return pd.Series(x).dt.is_leap_year.values<EOL>

Check whether a year is a leap year.

    :returns: an expression which evaluates to True if a year is a leap year, and to False otherwise.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.is_leap_year
    Expression = dt_is_leap_year(date)
    Length: 3 dtype: bool (expression)
    ----------------------------------
    0  False
    1   True
    2  False


import pandas as pd<EOL>return pd.Series(x).dt.year.values<EOL>

Extracts the year out of a datetime sample.

    :returns: an expression containing the year extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.year
    Expression = dt_year(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  2009
    1  2016
    2  2015


import pandas as pd<EOL>return pd.Series(x).dt.month.values<EOL>

Extracts the month out of a datetime sample.

    :returns: an expression containing the month extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.month
    Expression = dt_month(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  10
    1   2
    2  11


import pandas as pd<EOL>return pd.Series(_pandas_dt_fix(x)).dt.month_name().values.astype(str)<EOL>

Returns the month names of a datetime sample in English.

    :returns: an expression containing the month names extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.month_name
    Expression = dt_month_name(date)
    Length: 3 dtype: str (expression)
    ---------------------------------
    0   October
    1  February
    2  November


import pandas as pd<EOL>return pd.Series(x).dt.day.values<EOL>

Extracts the day from a datetime sample.

    :returns: an expression containing the day extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.day
    Expression = dt_day(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  12
    1  11
    2  12


import pandas as pd<EOL>return pd.Series(_pandas_dt_fix(x)).dt.day_name().values.astype(str)<EOL>

Returns the day names of a datetime sample in English.

    :returns: an expression containing the day names extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.day_name
    Expression = dt_day_name(date)
    Length: 3 dtype: str (expression)
    ---------------------------------
    0    Monday
    1  Thursday
    2  Thursday


import pandas as pd<EOL>return pd.Series(x).dt.weekofyear.values<EOL>

Returns the week ordinal of the year.

    :returns: an expression containing the week ordinal of the year, extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.weekofyear
    Expression = dt_weekofyear(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  42
    1   6
    2  46


import pandas as pd<EOL>return pd.Series(x).dt.hour.values<EOL>

Extracts the hour out of a datetime samples.

    :returns: an expression containing the hour extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.hour
    Expression = dt_hour(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0   3
    1  10
    2  11


import pandas as pd<EOL>return pd.Series(x).dt.minute.values<EOL>

Extracts the minute out of a datetime samples.

    :returns: an expression containing the minute extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.minute
    Expression = dt_minute(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0  31
    1  17
    2  34


import pandas as pd<EOL>return pd.Series(x).dt.second.values<EOL>

Extracts the second out of a datetime samples.

    :returns: an expression containing the second extracted from a datetime column.

    Example:

    >>> import vaex
    >>> import numpy as np
    >>> date = np.array(['2009-10-12T03:31:00', '2016-02-11T10:17:34', '2015-11-12T11:34:22'], dtype=np.datetime64)
    >>> df = vaex.from_arrays(date=date)
    >>> df
      #  date
      0  2009-10-12 03:31:00
      1  2016-02-11 10:17:34
      2  2015-11-12 11:34:22

    >>> df.date.dt.second
    Expression = dt_second(date)
    Length: 3 dtype: int64 (expression)
    -----------------------------------
    0   0
    1  34
    2  22


sl = _to_string_sequence(x).capitalize()<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Capitalize the first letter of a string sample.

    :returns: an expression containing the capitalized strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.capitalize()
    Expression = str_capitalize(text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    Something
    1  Very pretty
    2    Is coming
    3          Our
    4         Way.


sl1 = _to_string_sequence(x)<EOL>sl2 = _to_string_sequence(other)<EOL>sl = sl1.concat(sl2)<EOL>return column.ColumnStringArrow.from_string_sequence(sl)<EOL>

Concatenate two string columns on a row-by-row basis.

    :param expression other: The expression of the other column to be concatenated.
    :returns: an expression containing the concatenated columns.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.cat(df.text)
    Expression = str_cat(text, text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0      SomethingSomething
    1  very prettyvery pretty
    2      is comingis coming
    3                  ourour
    4                way.way.


@register_function(scope='<STR_LIT:str>')<EOL>def str_center(x, width, fillchar='<STR_LIT:U+0020>'):

sl = _to_string_sequence(x).pad(width, fillchar, True, True)<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Fills the left and right side of the strings with additional characters, such that the sample has a total of `width`
    characters.

    :param int width: The total number of characters of the resulting string sample.
    :param str fillchar: The character used for filling.
    :returns: an expression containing the filled strings.

    Example:
    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.center(width=11, fillchar='!')
    Expression = str_center(text, width=11, fillchar='!')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0  !Something!
    1  very pretty
    2  !is coming!
    3  !!!!our!!!!
    4  !!!!way.!!!


@register_function(scope='<STR_LIT:str>')<EOL>def str_contains(x, pattern, regex=True):

return _to_string_sequence(x).search(pattern, regex)<EOL>

Check if a string pattern or regex is contained within a sample of a string column.

    :param str pattern: A string or regex pattern
    :param bool regex: If True,
    :returns: an expression which is evaluated to True if the pattern is found in a given sample, and it is False otherwise.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.contains('very')
    Expression = str_contains(text, 'very')
    Length: 5 dtype: bool (expression)
    ----------------------------------
    0  False
    1   True
    2  False
    3  False
    4  False


@register_function(scope='<STR_LIT:str>')<EOL>def str_count(x, pat, regex=False):

return _to_string_sequence(x).count(pat, regex)<EOL>

Count the occurences of a pattern in sample of a string column.

    :param str pat: A string or regex pattern
    :param bool regex: If True,
    :returns: an expression containing the number of times a pattern is found in each sample.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.count(pat="et", regex=False)
    Expression = str_count(text, pat='et', regex=False)
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0  1
    1  1
    2  0
    3  0
    4  0


return _to_string_sequence(x).endswith(pat)<EOL>

Check if the end of each string sample matches the specified pattern.

    :param str pat: A string pattern or a regex
    :returns: an expression evaluated to True if the pattern is found at the end of a given sample, False otherwise.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.endswith(pat="ing")
    Expression = str_endswith(text, pat='ing')
    Length: 5 dtype: bool (expression)
    ----------------------------------
    0   True
    1  False
    2   True
    3  False
    4  False


@register_function(scope='<STR_LIT:str>')<EOL>def str_find(x, sub, start=<NUM_LIT:0>, end=None):

return _to_string_sequence(x).find(sub, start, <NUM_LIT:0> if end is None else end, end is None, True)<EOL>

Returns the lowest indices in each string in a column, where the provided substring is fully contained between within a
    sample. If the substring is not found, -1 is returned.

    :param str sub: A substring to be found in the samples
    :param int start:
    :param int end:
    :returns: an expression containing the lowest indices specifying the start of the substring.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.find(sub="et")
    Expression = str_find(text, sub='et')
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0   3
    1   7
    2  -1
    3  -1
    4  -1


x = _to_string_sequence(x)<EOL>if i == -<NUM_LIT:1>:<EOL><INDENT>sl = x.slice_string_end(-<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>sl = x.slice_string(i, i+<NUM_LIT:1>)<EOL><DEDENT>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Extract a character from each sample at the specified position from a string column.
    Note that if the specified position is out of bound of the string sample, this method returns '', while pandas retunrs nan.

    :param int i: The index location, at which to extract the character.
    :returns: an expression containing the extracted characters.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.get(5)
    Expression = str_get(text, 5)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    h
    1    p
    2    m
    3
    4


@register_function(scope='<STR_LIT:str>')<EOL>def str_index(x, sub, start=<NUM_LIT:0>, end=None):

return str_find(x, sub, start, end)<EOL>

Returns the lowest indices in each string in a column, where the provided substring is fully contained between within a
    sample. If the substring is not found, -1 is returned. It is the same as `str.find`.

    :param str sub: A substring to be found in the samples
    :param int start:
    :param int end:
    :returns: an expression containing the lowest indices specifying the start of the substring.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.index(sub="et")
    Expression = str_find(text, sub='et')
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0   3
    1   7
    2  -1
    3  -1
    4  -1


sl = _to_string_list_sequence(x).join(sep)<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Same as find (difference with pandas is that it does not raise a ValueError)


return _to_string_sequence(x).len()<EOL>

Returns the length of a string sample.

    :returns: an expression contains the length of each sample of a string column.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.len()
    Expression = str_len(text)
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0   9
    1  11
    2   9
    3   3
    4   4


return _to_string_sequence(x).byte_length()<EOL>

Returns the number of bytes in a string sample.

    :returns: an expression contains the number of bytes in each sample of a string column.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.byte_length()
    Expression = str_byte_length(text)
    Length: 5 dtype: int64 (expression)
    -----------------------------------
    0   9
    1  11
    2   9
    3   3
    4   4


@register_function(scope='<STR_LIT:str>')<EOL>def str_ljust(x, width, fillchar='<STR_LIT:U+0020>'):

sl = _to_string_sequence(x).pad(width, fillchar, False, True)<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Fills the right side of string samples with a specified character such that the strings are right-hand justified.

    :param int width: The minimal width of the strings.
    :param str fillchar: The character used for filling.
    :returns: an expression containing the filled strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.ljust(width=10, fillchar='!')
    Expression = str_ljust(text, width=10, fillchar='!')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0   Something!
    1  very pretty
    2   is coming!
    3   our!!!!!!!
    4   way.!!!!!!


sl = _to_string_sequence(x).lower()<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Converts string samples to lower case.

    :returns: an expression containing the converted strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.lower()
    Expression = str_lower(text)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    something
    1  very pretty
    2    is coming
    3          our
    4         way.


<EOL>sl = _to_string_sequence(x).lstrip('<STR_LIT>' if to_strip is None else to_strip) if to_strip != '<STR_LIT>' else x<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Remove leading characters from a string sample.

    :param str to_strip: The string to be removed
    :returns: an expression containing the modified string column.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.lstrip(to_strip='very ')
    Expression = str_lstrip(text, to_strip='very ')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0  Something
    1     pretty
    2  is coming
    3        our
    4       way.


return _to_string_sequence(x).match(pattern)<EOL>

Check if a string sample matches a given regular expression.

    :param str pattern: a string or regex to match to a string sample.
    :returns: an expression which is evaluated to True if a match is found, False otherwise.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.match(pattern='our')
    Expression = str_match(text, pattern='our')
    Length: 5 dtype: bool (expression)
    ----------------------------------
    0  False
    1  False
    2  False
    3   True
    4  False


@register_function(scope='<STR_LIT:str>')<EOL>def str_pad(x, width, side='<STR_LIT:left>', fillchar='<STR_LIT:U+0020>'):

sl = _to_string_sequence(x).pad(width, fillchar, side in ['<STR_LIT:left>', '<STR_LIT>'], side in ['<STR_LIT:right>', '<STR_LIT>'])<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Pad strings in a given column.

    :param int width: The total width of the string
    :param str side: If 'left' than pad on the left, if 'right' than pad on the right side the string.
    :param str fillchar: The character used for padding.
    :returns: an expression containing the padded strings.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.pad(width=10, side='left', fillchar='!')
    Expression = str_pad(text, width=10, side='left', fillchar='!')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0   !Something
    1  very pretty
    2   !is coming
    3   !!!!!!!our
    4   !!!!!!way.


sl = _to_string_sequence(x).repeat(repeats)<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Duplicate each string in a column.

    :param int repeats: number of times each string sample is to be duplicated.
    :returns: an expression containing the duplicated strings

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.repeat(3)
    Expression = str_repeat(text, 3)
    Length: 5 dtype: str (expression)
    ---------------------------------
    0        SomethingSomethingSomething
    1  very prettyvery prettyvery pretty
    2        is comingis comingis coming
    3                          ourourour
    4                       way.way.way.


@register_function(scope='<STR_LIT:str>')<EOL>def str_replace(x, pat, repl, n=-<NUM_LIT:1>, flags=<NUM_LIT:0>, regex=False):

sl = _to_string_sequence(x).replace(pat, repl, n, flags, regex)<EOL>return column.ColumnStringArrow(sl.bytes, sl.indices, sl.length, sl.offset, string_sequence=sl)<EOL>

Replace occurences of a pattern/regex in a column with some other string.

    :param str pattern: string or a regex pattern
    :param str replace: a replacement string
    :param int n: number of replacements to be made from the start. If -1 make all replacements.
    :param int flags: ??
    :param bool regex: If True, ...?
    :returns: an expression containing the string replacements.

    Example:

    >>> import vaex
    >>> text = ['Something', 'very pretty', 'is coming', 'our', 'way.']
    >>> df = vaex.from_arrays(text=text)
    >>> df
      #  text
      0  Something
      1  very pretty
      2  is coming
      3  our
      4  way.

    >>> df.text.str.replace(pat='et', repl='__')
    Expression = str_replace(text, pat='et', repl='__')
    Length: 5 dtype: str (expression)
    ---------------------------------
    0    Som__hing
    1  very pr__ty
    2    is coming
    3          our
    4         way.