@docsubst<EOL><INDENT>def percentile_approx(self, expression, percentage=<NUM_LIT>, binby=[], limits=None, shape=default_shape, percentile_shape=<NUM_LIT>, percentile_limits="<STR_LIT>", selection=False, delay=False):<DEDENT>

waslist, [expressions, ] = vaex.utils.listify(expression)<EOL>if not isinstance(binby, (tuple, list)):<EOL><INDENT>binby = [binby]<EOL><DEDENT>else:<EOL><INDENT>binby = binby<EOL><DEDENT>@delayed<EOL>def calculate(expression, shape, limits):<EOL><INDENT>return self.count(binby=list(binby) + [expression], shape=shape, limits=limits, selection=selection, delay=True, edges=True)<EOL><DEDENT>@delayed<EOL>def finish(percentile_limits, counts_list):<EOL><INDENT>results = []<EOL>for i, counts in enumerate(counts_list):<EOL><INDENT>counts = counts.astype(np.float)<EOL>nonnans = list([slice(<NUM_LIT:2>, -<NUM_LIT:1>, None) for k in range(len(counts.shape) - <NUM_LIT:1>)])<EOL>nonnans.append(slice(<NUM_LIT:1>, None, None))  <EOL>nonnans = tuple(nonnans)<EOL>cumulative_grid = np.cumsum(counts.__getitem__(nonnans), -<NUM_LIT:1>)  <EOL>totalcounts = np.sum(counts.__getitem__(nonnans), -<NUM_LIT:1>)<EOL>empty = totalcounts == <NUM_LIT:0><EOL>original_shape = counts.shape<EOL>shape = cumulative_grid.shape  <EOL>counts = np.sum(counts, -<NUM_LIT:1>)<EOL>edges_floor = np.zeros(shape[:-<NUM_LIT:1>] + (<NUM_LIT:2>,), dtype=np.int64)<EOL>edges_ceil = np.zeros(shape[:-<NUM_LIT:1>] + (<NUM_LIT:2>,), dtype=np.int64)<EOL>values = np.array((totalcounts + <NUM_LIT:1>) * percentage / <NUM_LIT>)  <EOL>values[empty] = <NUM_LIT:0><EOL>floor_values = np.array(np.floor(values))<EOL>ceil_values = np.array(np.ceil(values))<EOL>vaex.vaexfast.grid_find_edges(cumulative_grid, floor_values, edges_floor)<EOL>vaex.vaexfast.grid_find_edges(cumulative_grid, ceil_values, edges_ceil)<EOL>def index_choose(a, indices):<EOL><INDENT>out = np.zeros(a.shape[:-<NUM_LIT:1>])<EOL>for i in np.ndindex(out.shape):<EOL><INDENT>out[i] = a[i + (indices[i],)]<EOL><DEDENT>return out<EOL><DEDENT>def calculate_x(edges, values):<EOL><INDENT>left, right = edges[..., <NUM_LIT:0>], edges[..., <NUM_LIT:1>]<EOL>left_value = index_choose(cumulative_grid, left)<EOL>right_value = index_choose(cumulative_grid, right)<EOL>u = np.array((values - left_value) / (right_value - left_value))<EOL>xleft, xright = percentile_limits[i][<NUM_LIT:0>] + (left - <NUM_LIT:0.5>) * (percentile_limits[i][<NUM_LIT:1>] - percentile_limits[i][<NUM_LIT:0>]) / (shape[-<NUM_LIT:1>] - <NUM_LIT:3>),percentile_limits[i][<NUM_LIT:0>] + (right - <NUM_LIT:0.5>) * (percentile_limits[i][<NUM_LIT:1>] - percentile_limits[i][<NUM_LIT:0>]) / (shape[-<NUM_LIT:1>] - <NUM_LIT:3>)<EOL>x = xleft + (xright - xleft) * u  <EOL>return x<EOL><DEDENT>x1 = calculate_x(edges_floor, floor_values)<EOL>x2 = calculate_x(edges_ceil, ceil_values)<EOL>u = values - floor_values<EOL>x = x1 + (x2 - x1) * u<EOL>results.append(x)<EOL><DEDENT>return results<EOL><DEDENT>shape = _expand_shape(shape, len(binby))<EOL>percentile_shapes = _expand_shape(percentile_shape, len(expressions))<EOL>if percentile_limits:<EOL><INDENT>percentile_limits = _expand_limits(percentile_limits, len(expressions))<EOL><DEDENT>limits = self.limits(binby, limits, selection=selection, delay=True)<EOL>percentile_limits = self.limits(expressions, percentile_limits, selection=selection, delay=True)<EOL>@delayed<EOL>def calculation(limits, percentile_limits):<EOL><INDENT>tasks = [calculate(expression, tuple(shape) + (percentile_shape, ), list(limits) + [list(percentile_limit)])<EOL>for percentile_shape, percentile_limit, expression<EOL>in zip(percentile_shapes, percentile_limits, expressions)]<EOL>return finish(percentile_limits, delayed_args(*tasks))<EOL><DEDENT>result = calculation(limits, percentile_limits)<EOL>@delayed<EOL>def finish2(grid):<EOL><INDENT>value = vaex.utils.unlistify(waslist, np.array(grid))<EOL>return value<EOL><DEDENT>return self._delay(delay, finish2(result))<EOL>

Calculate the percentile given by percentage, possibly on a grid defined by binby.

        NOTE: this value is approximated by calculating the cumulative distribution on a grid defined by
        percentile_shape and percentile_limits.

        Example:

        >>> df.percentile_approx("x", 10), df.percentile_approx("x", 90)
        (array([-8.3220355]), array([ 7.92080358]))
        >>> df.percentile_approx("x", 50, binby="x", shape=5, limits=[-10, 10])
        array([[-7.56462982],
                   [-3.61036641],
                   [-0.01296306],
                   [ 3.56697863],
                   [ 7.45838367]])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param percentile_limits: {percentile_limits}
        :param percentile_shape: {percentile_shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}


@docsubst<EOL><INDENT>def limits_percentage(self, expression, percentage=<NUM_LIT>, square=False, delay=False):<DEDENT>

<EOL>import scipy<EOL>logger.info("<STR_LIT>", expression, percentage)<EOL>waslist, [expressions, ] = vaex.utils.listify(expression)<EOL>limits = []<EOL>for expr in expressions:<EOL><INDENT>subspace = self(expr)<EOL>limits_minmax = subspace.minmax()<EOL>vmin, vmax = limits_minmax[<NUM_LIT:0>]<EOL>size = <NUM_LIT> * <NUM_LIT:16><EOL>counts = subspace.histogram(size=size, limits=limits_minmax)<EOL>cumcounts = np.concatenate([[<NUM_LIT:0>], np.cumsum(counts)])<EOL>cumcounts /= cumcounts.max()<EOL>f = (<NUM_LIT:1> - percentage / <NUM_LIT>) / <NUM_LIT:2><EOL>x = np.linspace(vmin, vmax, size + <NUM_LIT:1>)<EOL>l = scipy.interp([f, <NUM_LIT:1> - f], cumcounts, x)<EOL>limits.append(l)<EOL><DEDENT>return vaex.utils.unlistify(waslist, limits)<EOL>

Calculate the [min, max] range for expression, containing approximately a percentage of the data as defined
        by percentage.

        The range is symmetric around the median, i.e., for a percentage of 90, this gives the same results as:

        Example:

        >>> df.limits_percentage("x", 90)
        array([-12.35081376,  12.14858052]
        >>> df.percentile_approx("x", 5), df.percentile_approx("x", 95)
        (array([-12.36813152]), array([ 12.13275818]))

        NOTE: this value is approximated by calculating the cumulative distribution on a grid.
        NOTE 2: The values above are not exactly the same, since percentile and limits_percentage do not share the same code

        :param expression: {expression_limits}
        :param float percentage: Value between 0 and 100
        :param delay: {delay}
        :return: {return_limits}


@docsubst<EOL><INDENT>def limits(self, expression, value=None, square=False, selection=None, delay=False, shape=None):<DEDENT>

if expression == []:<EOL><INDENT>return [] if shape is None else ([], [])<EOL><DEDENT>waslist, [expressions, ] = vaex.utils.listify(expression)<EOL>expressions = _ensure_strings_from_expressions(expressions)<EOL>selection = _ensure_strings_from_expressions(selection)<EOL>if value is None:<EOL><INDENT>value = "<STR_LIT>"<EOL><DEDENT>if _is_limit(value) or not _issequence(value):<EOL><INDENT>values = (value,) * len(expressions)<EOL><DEDENT>else:<EOL><INDENT>values = value<EOL><DEDENT>initial_expressions, initial_values = expressions, values<EOL>expression_values = dict()<EOL>expression_shapes = dict()<EOL>for i, (expression, value) in enumerate(zip(expressions, values)):<EOL><INDENT>if _issequence(expression):<EOL><INDENT>expressions = expression<EOL>nested = True<EOL><DEDENT>else:<EOL><INDENT>expressions = [expression]<EOL>nested = False<EOL><DEDENT>if _is_limit(value) or not _issequence(value):<EOL><INDENT>values = (value,) * len(expressions)<EOL><DEDENT>else:<EOL><INDENT>values = value<EOL><DEDENT>for j, (expression, value) in enumerate(zip(expressions, values)):<EOL><INDENT>if shape is not None:<EOL><INDENT>if _issequence(shape):<EOL><INDENT>shapes = shape<EOL><DEDENT>else:<EOL><INDENT>shapes = (shape, ) * (len(expressions) if nested else len(initial_expressions))<EOL><DEDENT><DEDENT>shape_index = j if nested else i<EOL>if not _is_limit(value):  <EOL><INDENT>expression_values[(expression, value)] = None<EOL><DEDENT>if self.is_category(expression):<EOL><INDENT>N = self._categories[_ensure_string_from_expression(expression)]['<STR_LIT:N>']<EOL>expression_shapes[expression] = min(N, shapes[shape_index] if shape is not None else default_shape)<EOL><DEDENT>else:<EOL><INDENT>expression_shapes[expression] = shapes[shape_index] if shape is not None else default_shape<EOL><DEDENT><DEDENT><DEDENT>limits_list = []<EOL>for expression, value in expression_values.keys():<EOL><INDENT>if self.is_category(expression):<EOL><INDENT>N = self._categories[_ensure_string_from_expression(expression)]['<STR_LIT:N>']<EOL>limits = [-<NUM_LIT:0.5>, N-<NUM_LIT:0.5>]<EOL><DEDENT>else:<EOL><INDENT>if isinstance(value, six.string_types):<EOL><INDENT>if value == "<STR_LIT>":<EOL><INDENT>limits = self.minmax(expression, selection=selection, delay=True)<EOL><DEDENT>else:<EOL><INDENT>match = re.match(r"<STR_LIT>", value)<EOL>if match is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>number, type = match.groups()<EOL>import ast<EOL>number = ast.literal_eval(number)<EOL>type = type.strip()<EOL>if type in ["<STR_LIT:s>", "<STR_LIT>"]:<EOL><INDENT>limits = self.limits_sigma(number)<EOL><DEDENT>elif type in ["<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>limits = self.limits_sigma(number, square=True)<EOL><DEDENT>elif type in ["<STR_LIT:%>", "<STR_LIT>"]:<EOL><INDENT>limits = self.limits_percentage(expression, number, delay=False)<EOL><DEDENT>elif type in ["<STR_LIT:%s>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>limits = self.limits_percentage(expression, number, square=True, delay=True)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif value is None:<EOL><INDENT>limits = self.limits_percentage(expression, square=square, delay=True)<EOL><DEDENT>else:<EOL><INDENT>limits = value<EOL><DEDENT><DEDENT>limits_list.append(limits)<EOL>if limits is None:<EOL><INDENT>raise ValueError("<STR_LIT>" % value)<EOL><DEDENT>expression_values[(expression, value)] = limits<EOL>logger.debug("<STR_LIT>", limits, np.array(limits).shape)<EOL>@delayed<EOL>def echo(limits):<EOL><INDENT>logger.debug("<STR_LIT>", limits, np.array(limits).shape)<EOL><DEDENT>echo(limits)<EOL><DEDENT>limits_list = delayed_args(*limits_list)<EOL>@delayed<EOL>def finish(limits_list):<EOL><INDENT>limits_outer = []<EOL>shapes_list = []<EOL>for expression, value in zip(initial_expressions, initial_values):<EOL><INDENT>if _issequence(expression):<EOL><INDENT>expressions = expression<EOL>waslist2 = True<EOL><DEDENT>else:<EOL><INDENT>expressions = [expression]<EOL>waslist2 = False<EOL><DEDENT>if _is_limit(value) or not _issequence(value):<EOL><INDENT>values = (value,) * len(expressions)<EOL><DEDENT>else:<EOL><INDENT>values = value<EOL><DEDENT>limits = []<EOL>shapes = []<EOL>for expression, value in zip(expressions, values):<EOL><INDENT>if not _is_limit(value):<EOL><INDENT>value = expression_values[(expression, value)]<EOL>if not _is_limit(value):<EOL><INDENT>value = value.get()<EOL><DEDENT><DEDENT>limits.append(value)<EOL>shapes.append(expression_shapes[expression])<EOL><DEDENT>if waslist2:<EOL><INDENT>limits_outer.append(limits)<EOL>shapes_list.append(shapes)<EOL><DEDENT>else:<EOL><INDENT>limits_outer.append(limits[<NUM_LIT:0>])<EOL>shapes_list.append(shapes[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>if shape:<EOL><INDENT>return vaex.utils.unlistify(waslist, limits_outer), vaex.utils.unlistify(waslist, shapes_list)<EOL><DEDENT>else:<EOL><INDENT>return vaex.utils.unlistify(waslist, limits_outer)<EOL><DEDENT><DEDENT>return self._delay(delay, finish(limits_list))<EOL>

Calculate the [min, max] range for expression, as described by value, which is '99.7%' by default.

        If value is a list of the form [minvalue, maxvalue], it is simply returned, this is for convenience when using mixed
        forms.

        Example:

        >>> df.limits("x")
        array([-28.86381927,  28.9261226 ])
        >>> df.limits(["x", "y"])
        (array([-28.86381927,  28.9261226 ]), array([-28.60476934,  28.96535249]))
        >>> df.limits(["x", "y"], "minmax")
        (array([-128.293991,  271.365997]), array([ -71.5523682,  146.465836 ]))
        >>> df.limits(["x", "y"], ["minmax", "90%"])
        (array([-128.293991,  271.365997]), array([-13.37438402,  13.4224423 ]))
        >>> df.limits(["x", "y"], ["minmax", [0, 10]])
        (array([-128.293991,  271.365997]), [0, 10])

        :param expression: {expression_limits}
        :param value: {limits}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_limits}


def mode(self, expression, binby=[], limits=None, shape=<NUM_LIT>, mode_shape=<NUM_LIT:64>, mode_limits=None, progressbar=False, selection=None):

if len(binby) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>len(shape)<EOL>shape = tuple(shape)<EOL><DEDENT>except:<EOL><INDENT>shape = len(binby) * (shape,)<EOL><DEDENT>shape = (mode_shape,) + shape<EOL>subspace = self(*(list(binby) + [expression]))<EOL>if selection:<EOL><INDENT>subspace = subspace.selected()<EOL><DEDENT>limits = self.limits(list(binby), limits)<EOL>mode_limits = self.limits([expression], mode_limits)<EOL>limits = list(limits) + list(mode_limits)<EOL>counts = subspace.histogram(limits=limits, size=shape, progressbar=progressbar)<EOL>indices = np.argmax(counts, axis=<NUM_LIT:0>)<EOL>pmin, pmax = limits[-<NUM_LIT:1>]<EOL>centers = np.linspace(pmin, pmax, mode_shape + <NUM_LIT:1>)[:-<NUM_LIT:1>]  <EOL>centers += (centers[<NUM_LIT:1>] - centers[<NUM_LIT:0>]) / <NUM_LIT:2>  <EOL>modes = centers[indices]<EOL>ok = counts.sum(axis=<NUM_LIT:0>) > <NUM_LIT:0><EOL>modes[~ok] = np.nan<EOL>return modes<EOL><DEDENT>


def plot_widget(self, x, y, z=None, grid=None, shape=<NUM_LIT>, limits=None, what="<STR_LIT>", figsize=None,<EOL>f="<STR_LIT>", figure_key=None, fig=None, axes=None, xlabel=None, ylabel=None, title=None,<EOL>show=True, selection=[None, True], colormap="<STR_LIT>", grid_limits=None, normalize="<STR_LIT>",<EOL>grid_before=None,<EOL>what_kwargs={}, type="<STR_LIT:default>",<EOL>scales=None, tool_select=False, bq_cleanup=True,<EOL>backend="<STR_LIT>",<EOL>**kwargs):

import vaex.jupyter.plot<EOL>backend = vaex.jupyter.plot.create_backend(backend)<EOL>cls = vaex.jupyter.plot.get_type(type)<EOL>x = _ensure_strings_from_expressions(x)<EOL>y = _ensure_strings_from_expressions(y)<EOL>z = _ensure_strings_from_expressions(z)<EOL>for name in '<STR_LIT>'.split():<EOL><INDENT>if name in kwargs:<EOL><INDENT>kwargs[name] = _ensure_strings_from_expressions(kwargs[name])<EOL><DEDENT><DEDENT>plot2d = cls(backend=backend, dataset=self, x=x, y=y, z=z, grid=grid, shape=shape, limits=limits, what=what,<EOL>f=f, figure_key=figure_key, fig=fig,<EOL>selection=selection, grid_before=grid_before,<EOL>grid_limits=grid_limits, normalize=normalize, colormap=colormap, what_kwargs=what_kwargs, **kwargs)<EOL>if show:<EOL><INDENT>plot2d.show()<EOL><DEDENT>return plot2d<EOL>

Viz 1d, 2d or 3d in a Jupyter notebook

        .. note::
            This API is not fully settled and may change in the future

        Example:

        >>> df.plot_widget(df.x, df.y, backend='bqplot')
        >>> df.plot_widget(df.pickup_longitude, df.pickup_latitude, backend='ipyleaflet')

        :param backend: Widget backend to use: 'bqplot', 'ipyleaflet', 'ipyvolume', 'matplotlib'


def healpix_count(self, expression=None, healpix_expression=None, healpix_max_level=<NUM_LIT:12>, healpix_level=<NUM_LIT:8>, binby=None, limits=None, shape=default_shape, delay=False, progress=None, selection=None):

<EOL>import healpy as hp<EOL>if healpix_expression is None:<EOL><INDENT>if self.ucds.get("<STR_LIT>", None) == '<STR_LIT>':  <EOL><INDENT>healpix_expression = "<STR_LIT>"<EOL><DEDENT><DEDENT>if healpix_expression is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>reduce_level = healpix_max_level - healpix_level<EOL>NSIDE = <NUM_LIT:2>**healpix_level<EOL>nmax = hp.nside2npix(NSIDE)<EOL>scaling = <NUM_LIT:4>**reduce_level<EOL>expr = "<STR_LIT>" % (healpix_expression, scaling)<EOL>binby = [expr] + ([] if binby is None else _ensure_list(binby))<EOL>shape = (nmax,) + _expand_shape(shape, len(binby) - <NUM_LIT:1>)<EOL>epsilon = <NUM_LIT:1.> / scaling / <NUM_LIT:2><EOL>limits = [[-epsilon, nmax - epsilon]] + ([] if limits is None else limits)<EOL>return self.count(expression, binby=binby, limits=limits, shape=shape, delay=delay, progress=progress, selection=selection)<EOL>

Count non missing value for expression on an array which represents healpix data.

        :param expression: Expression or column for which to count non-missing values, or None or '*' for counting the rows
        :param healpix_expression: {healpix_max_level}
        :param healpix_max_level: {healpix_max_level}
        :param healpix_level: {healpix_level}
        :param binby: {binby}, these dimension follow the first healpix dimension.
        :param limits: {limits}
        :param shape: {shape}
        :param selection: {selection}
        :param delay: {delay}
        :param progress: {progress}
        :return:


def healpix_plot(self, healpix_expression="<STR_LIT>", healpix_max_level=<NUM_LIT:12>, healpix_level=<NUM_LIT:8>, what="<STR_LIT>", selection=None,<EOL>grid=None,<EOL>healpix_input="<STR_LIT>", healpix_output="<STR_LIT>", f=None,<EOL>colormap="<STR_LIT>", grid_limits=None, image_size=<NUM_LIT>, nest=True,<EOL>figsize=None, interactive=False, title="<STR_LIT>", smooth=None, show=False, colorbar=True,<EOL>rotation=(<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>), **kwargs):

<EOL>import healpy as hp<EOL>import pylab as plt<EOL>if grid is None:<EOL><INDENT>reduce_level = healpix_max_level - healpix_level<EOL>NSIDE = <NUM_LIT:2>**healpix_level<EOL>nmax = hp.nside2npix(NSIDE)<EOL>scaling = <NUM_LIT:4>**reduce_level<EOL>epsilon = <NUM_LIT:1.> / scaling / <NUM_LIT:2><EOL>grid = self._stat(what=what, binby="<STR_LIT>" % (healpix_expression, scaling), limits=[-epsilon, nmax - epsilon], shape=nmax, selection=selection)<EOL><DEDENT>if grid_limits:<EOL><INDENT>grid_min, grid_max = grid_limits<EOL><DEDENT>else:<EOL><INDENT>grid_min = grid_max = None<EOL><DEDENT>f_org = f<EOL>f = _parse_f(f)<EOL>if smooth:<EOL><INDENT>if nest:<EOL><INDENT>grid = hp.reorder(grid, inp="<STR_LIT>", out="<STR_LIT>")<EOL>nest = False<EOL><DEDENT>grid = hp.smoothing(grid, sigma=np.radians(smooth))<EOL><DEDENT>fgrid = f(grid)<EOL>coord_map = dict(equatorial='<STR_LIT:C>', galactic='<STR_LIT>', ecliptic="<STR_LIT:E>")<EOL>fig = plt.gcf()<EOL>if figsize is not None:<EOL><INDENT>fig.set_size_inches(*figsize)<EOL><DEDENT>what_label = what<EOL>if f_org:<EOL><INDENT>what_label = f_org + "<STR_LIT:U+0020>" + what_label<EOL><DEDENT>f = hp.mollzoom if interactive else hp.mollview<EOL>with warnings.catch_warnings():<EOL><INDENT>warnings.simplefilter("<STR_LIT:ignore>")<EOL>coord = coord_map[healpix_input], coord_map[healpix_output]<EOL>if coord_map[healpix_input] == coord_map[healpix_output]:<EOL><INDENT>coord = None<EOL><DEDENT>f(fgrid, unit=what_label, rot=rotation, nest=nest, title=title, coord=coord,<EOL>cmap=colormap, hold=True, xsize=image_size, min=grid_min, max=grid_max, cbar=colorbar, **kwargs)<EOL><DEDENT>if show:<EOL><INDENT>plt.show()<EOL><DEDENT>

Viz data in 2d using a healpix column.

        :param healpix_expression: {healpix_max_level}
        :param healpix_max_level: {healpix_max_level}
        :param healpix_level: {healpix_level}
        :param what: {what}
        :param selection: {selection}
        :param grid: {grid}
        :param healpix_input: Specificy if the healpix index is in "equatorial", "galactic" or "ecliptic".
        :param healpix_output: Plot in "equatorial", "galactic" or "ecliptic".
        :param f: function to apply to the data
        :param colormap: matplotlib colormap
        :param grid_limits: Optional sequence [minvalue, maxvalue] that determine the min and max value that map to the colormap (values below and above these are clipped to the the min/max). (default is [min(f(grid)), max(f(grid)))
        :param image_size: size for the image that healpy uses for rendering
        :param nest: If the healpix data is in nested (True) or ring (False)
        :param figsize: If given, modify the matplotlib figure size. Example (14,9)
        :param interactive: (Experimental, uses healpy.mollzoom is True)
        :param title: Title of figure
        :param smooth: apply gaussian smoothing, in degrees
        :param show: Call matplotlib's show (True) or not (False, defaut)
        :param rotation: Rotatate the plot, in format (lon, lat, psi) such that (lon, lat) is the center, and rotate on the screen by angle psi. All angles are degrees.
        :return:


def plot3d(self, x, y, z, vx=None, vy=None, vz=None, vwhat=None, limits=None, grid=None, what="<STR_LIT>", shape=<NUM_LIT>, selection=[None, True], f=None,<EOL>vcount_limits=None,<EOL>smooth_pre=None, smooth_post=None, grid_limits=None, normalize="<STR_LIT>", colormap="<STR_LIT>",<EOL>figure_key=None, fig=None,<EOL>lighting=True, level=[<NUM_LIT:0.1>, <NUM_LIT:0.5>, <NUM_LIT>], opacity=[<NUM_LIT>, <NUM_LIT>, <NUM_LIT:0.1>], level_width=<NUM_LIT:0.1>,<EOL>show=True, **kwargs):

import vaex.ext.ipyvolume<EOL>cls = vaex.ext.ipyvolume.PlotDefault<EOL>plot3d = cls(df=self, x=x, y=y, z=z, vx=vx, vy=vy, vz=vz,<EOL>grid=grid, shape=shape, limits=limits, what=what,<EOL>f=f, figure_key=figure_key, fig=fig,<EOL>selection=selection, smooth_pre=smooth_pre, smooth_post=smooth_post,<EOL>grid_limits=grid_limits, vcount_limits=vcount_limits, normalize=normalize, colormap=colormap, **kwargs)<EOL>if show:<EOL><INDENT>plot3d.show()<EOL><DEDENT>return plot3d<EOL>


class ColumnList(object):<EOL><INDENT>pass<EOL><DEDENT>data = ColumnList()<EOL>for name in self.get_column_names():<EOL><INDENT>expression = getattr(self, name, None)<EOL>if not isinstance(expression, Expression):<EOL><INDENT>expression = Expression(self, name)<EOL><DEDENT>setattr(data, name, expression)<EOL><DEDENT>return data<EOL>

Gives direct access to the columns only (useful for tab completion).

        Convenient when working with ipython in combination with small DataFrames, since this gives tab-completion.

        Columns can be accesed by there names, which are attributes. The attribues are currently expressions, so you can
        do computations with them.

        Example

        >>> ds = vaex.example()
        >>> df.plot(df.col.x, df.col.y)


pass<EOL>

Close any possible open file handles, the DataFrame will not be in a usable state afterwards.


def byte_size(self, selection=False, virtual=False):

bytes_per_row = <NUM_LIT:0><EOL>N = self.count(selection=selection)<EOL>extra = <NUM_LIT:0><EOL>for column in list(self.get_column_names(virtual=virtual)):<EOL><INDENT>dtype = self.dtype(column)<EOL>dtype_internal = self.dtype(column, internal=True)<EOL>if isinstance(self.columns[column], ColumnString):<EOL><INDENT>extra += self.columns[column].nbytes<EOL><DEDENT>else:<EOL><INDENT>bytes_per_row += dtype_internal.itemsize<EOL>if np.ma.isMaskedArray(self.columns[column]):<EOL><INDENT>bytes_per_row += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return bytes_per_row * self.count(selection=selection) + extra<EOL>

Return the size in bytes the whole DataFrame requires (or the selection), respecting the active_fraction.


return self.byte_size()<EOL>

Alias for `df.byte_size()`, see :meth:`DataFrame.byte_size`.


def dtype(self, expression, internal=False):

expression = _ensure_string_from_expression(expression)<EOL>if expression in self.variables:<EOL><INDENT>return np.float64(<NUM_LIT:1>).dtype<EOL><DEDENT>elif expression in self.columns.keys():<EOL><INDENT>column = self.columns[expression]<EOL>data = column[<NUM_LIT:0>:<NUM_LIT:1>]<EOL>dtype = data.dtype<EOL><DEDENT>else:<EOL><INDENT>data = self.evaluate(expression, <NUM_LIT:0>, <NUM_LIT:1>, filtered=False)<EOL>dtype = data.dtype<EOL><DEDENT>if not internal:<EOL><INDENT>if dtype != str_type:<EOL><INDENT>if dtype.kind in '<STR_LIT>':<EOL><INDENT>return str_type<EOL><DEDENT>if dtype.kind == '<STR_LIT:O>':<EOL><INDENT>if isinstance(data[<NUM_LIT:0>], six.string_types):<EOL><INDENT>return str_type<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return dtype<EOL>

Return the numpy dtype for the given expression, if not a column, the first row will be evaluated to get the dtype.


from pandas import Series<EOL>return Series({column_name:self.dtype(column_name) for column_name in self.get_column_names()})<EOL>

Gives a Pandas series object containing all numpy dtypes of all columns (except hidden).


column = _ensure_string_from_expression(column)<EOL>if column in self.columns:<EOL><INDENT>return np.ma.isMaskedArray(self.columns[column])<EOL><DEDENT>return False<EOL>

Return if a column is a masked (numpy.ma) column.


def unit(self, expression, default=None):

expression = _ensure_string_from_expression(expression)<EOL>try:<EOL><INDENT>unit_or_quantity = eval(expression, expression_namespace, scopes.UnitScope(self))<EOL>unit = unit_or_quantity.unit if hasattr(unit_or_quantity, "<STR_LIT>") else unit_or_quantity<EOL>return unit if isinstance(unit, astropy.units.Unit) else None<EOL><DEDENT>except:<EOL><INDENT>try:<EOL><INDENT>return eval(expression, expression_namespace, scopes.UnitScope(self, <NUM_LIT:1.>)).unit<EOL><DEDENT>except:<EOL><INDENT>return default<EOL><DEDENT><DEDENT>

Returns the unit (an astropy.unit.Units object) for the expression.

        Example

        >>> import vaex
        >>> ds = vaex.example()
        >>> df.unit("x")
        Unit("kpc")
        >>> df.unit("x*L")
        Unit("km kpc2 / s")

        :param expression: Expression, which can be a column name
        :param default: if no unit is known, it will return this
        :return: The resulting unit of the expression
        :rtype: astropy.units.Unit


if isinstance(ucds, six.string_types):<EOL><INDENT>ucds = [ucds]<EOL><DEDENT>if len(ucds) == <NUM_LIT:1>:<EOL><INDENT>ucd = ucds[<NUM_LIT:0>]<EOL>if ucd[<NUM_LIT:0>] == "<STR_LIT>":  <EOL><INDENT>ucd = ucd[<NUM_LIT:1>:]<EOL>columns = [name for name in self.get_column_names() if self.ucds.get(name, "<STR_LIT>").startswith(ucd) and name not in exclude]<EOL><DEDENT>else:<EOL><INDENT>columns = [name for name in self.get_column_names() if ucd in self.ucds.get(name, "<STR_LIT>") and name not in exclude]<EOL><DEDENT>return None if len(columns) == <NUM_LIT:0> else columns[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>columns = [self.ucd_find([ucd], exclude=exclude) for ucd in ucds]<EOL>return None if None in columns else columns<EOL><DEDENT>

Find a set of columns (names) which have the ucd, or part of the ucd.

        Prefixed with a ^, it will only match the first part of the ucd.

        Example

        >>> df.ucd_find('pos.eq.ra', 'pos.eq.dec')
        ['RA', 'DEC']
        >>> df.ucd_find('pos.eq.ra', 'doesnotexist')
        >>> df.ucds[df.ucd_find('pos.eq.ra')]
        'pos.eq.ra;meta.main'
        >>> df.ucd_find('meta.main')]
        'dec'
        >>> df.ucd_find('^meta.main')]


if self.is_local():<EOL><INDENT>name = os.path.abspath(self.path).replace(os.path.sep, "<STR_LIT:_>")[:<NUM_LIT>]  <EOL>name = name.replace("<STR_LIT::>", "<STR_LIT:_>")  <EOL><DEDENT>else:<EOL><INDENT>server = self.server<EOL>name = "<STR_LIT>" % (server.hostname, server.port, server.base_path.replace("<STR_LIT:/>", "<STR_LIT:_>"), self.name)<EOL><DEDENT>dir = os.path.join(vaex.utils.get_private_dir(), "<STR_LIT>", name)<EOL>if create and not os.path.exists(dir):<EOL><INDENT>os.makedirs(dir)<EOL><DEDENT>return dir<EOL>

Each DataFrame has a directory where files are stored for metadata etc.

        Example

        >>> import vaex
        >>> ds = vaex.example()
        >>> vaex.get_private_dir()
        '/Users/users/breddels/.vaex/dfs/_Users_users_breddels_vaex-testing_data_helmi-dezeeuw-2000-10p.hdf5'

        :param bool create: is True, it will create the directory if it does not exist


virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())<EOL>units = {key: str(value) for key, value in self.units.items()}<EOL>ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}<EOL>descriptions = {key: value for key, value in self.descriptions.items()}<EOL>import vaex.serialize<EOL>def check(key, value):<EOL><INDENT>if not vaex.serialize.can_serialize(value.f):<EOL><INDENT>warnings.warn('<STR_LIT>'.format(key))<EOL>return False<EOL><DEDENT>return True<EOL><DEDENT>def clean(value):<EOL><INDENT>return vaex.serialize.to_dict(value.f)<EOL><DEDENT>functions = {key: clean(value) for key, value in self.functions.items() if check(key, value)}<EOL>virtual_columns = {key: value for key, value in self.virtual_columns.items()}<EOL>selections = {name: self.get_selection(name) for name, history in self.selection_histories.items()}<EOL>selections = {name: selection.to_dict() if selection is not None else None for name, selection in selections.items()}<EOL><INDENT>selection is not None}<EOL>state = dict(virtual_columns=virtual_columns,<EOL><INDENT>column_names=self.column_names,<EOL>renamed_columns=self._renamed_columns,<EOL>variables=self.variables,<EOL>functions=functions,<EOL>selections=selections,<EOL>ucds=ucds,<EOL>units=units,<EOL>descriptions=descriptions,<EOL>description=self.description,<EOL>active_range=[self._index_start, self._index_end])<EOL>return state<EOL>

Return the internal state of the DataFrame in a dictionary

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> df.state_get()
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}


def state_set(self, state, use_active_range=False):

self.description = state['<STR_LIT:description>']<EOL>if use_active_range:<EOL><INDENT>self._index_start, self._index_end = state['<STR_LIT>']<EOL><DEDENT>self._length_unfiltered = self._index_end - self._index_start<EOL>if '<STR_LIT>' in state:<EOL><INDENT>for old, new in state['<STR_LIT>']:<EOL><INDENT>self._rename(old, new)<EOL><DEDENT><DEDENT>for name, value in state['<STR_LIT>'].items():<EOL><INDENT>self.add_function(name, vaex.serialize.from_dict(value))<EOL><DEDENT>if '<STR_LIT>' in state:<EOL><INDENT>self.column_names = []<EOL>self.virtual_columns = collections.OrderedDict()<EOL>for name, value in state['<STR_LIT>'].items():<EOL><INDENT>self[name] = self._expr(value)<EOL><DEDENT>self.column_names = state['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>self.virtual_columns = collections.OrderedDict()<EOL>for name, value in state['<STR_LIT>'].items():<EOL><INDENT>self[name] = self._expr(value)<EOL><DEDENT><DEDENT>self.variables = state['<STR_LIT>']<EOL>import astropy  <EOL>units = {key: astropy.units.Unit(value) for key, value in state["<STR_LIT>"].items()}<EOL>self.units.update(units)<EOL>for name, selection_dict in state['<STR_LIT>'].items():<EOL><INDENT>if selection_dict is None:<EOL><INDENT>selection = None<EOL><DEDENT>else:<EOL><INDENT>selection = selections.selection_from_dict(selection_dict)<EOL><DEDENT>self.set_selection(selection, name=name)<EOL><DEDENT>

Sets the internal state of the df

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df
          #    x    y        r
          0    1    2  2.23607
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> state = df.state_get()
        >>> state
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}
        >>> df2 = vaex.from_scalars(x=3, y=4)
        >>> df2.state_set(state)  # now the virtual functions are 'copied'
        >>> df2
          #    x    y    r
          0    3    4    5

        :param state: dict as returned by :meth:`DataFrame.state_get`.
        :param bool use_active_range: Whether to use the active range or not.


vaex.utils.write_json_or_yaml(f, self.state_get())<EOL>

Write the internal state to a json or yaml file (see :meth:`DataFrame.state_get`)

        Example

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> df.state_write('state.json')
        >>> print(open('state.json').read())
        {
        "virtual_columns": {
            "r": "(((x ** 2) + (y ** 2)) ** 0.5)"
        },
        "column_names": [
            "x",
            "y",
            "r"
        ],
        "renamed_columns": [],
        "variables": {
            "pi": 3.141592653589793,
            "e": 2.718281828459045,
            "km_in_au": 149597870.7,
            "seconds_per_year": 31557600
        },
        "functions": {},
        "selections": {
            "__filter__": null
        },
        "ucds": {},
        "units": {},
        "descriptions": {},
        "description": null,
        "active_range": [
            0,
            1
        ]
        }
        >>> df.state_write('state.yaml')
        >>> print(open('state.yaml').read())
        active_range:
        - 0
        - 1
        column_names:
        - x
        - y
        - r
        description: null
        descriptions: {}
        functions: {}
        renamed_columns: []
        selections:
        __filter__: null
        ucds: {}
        units: {}
        variables:
        pi: 3.141592653589793
        e: 2.718281828459045
        km_in_au: 149597870.7
        seconds_per_year: 31557600
        virtual_columns:
        r: (((x ** 2) + (y ** 2)) ** 0.5)

        :param str f: filename (ending in .json or .yaml)


def state_load(self, f, use_active_range=False):

state = vaex.utils.read_json_or_yaml(f)<EOL>self.state_set(state, use_active_range=use_active_range)<EOL>

Load a state previously stored by :meth:`DataFrame.state_store`, see also :meth:`DataFrame.state_set`.


dir = self.get_private_dir(create=True)<EOL>path = os.path.join(dir, "<STR_LIT>")<EOL>try:<EOL><INDENT>if os.path.exists(path):<EOL><INDENT>os.remove(path)<EOL><DEDENT>if not os.listdir(dir):<EOL><INDENT>os.rmdir(dir)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>logger.exception("<STR_LIT>", path, dir)<EOL><DEDENT>

Removes the file with the virtual column etc, it does not change the current virtual columns etc.


path = os.path.join(self.get_private_dir(create=True), "<STR_LIT>")<EOL>virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())<EOL>units = {key: str(value) for key, value in self.units.items() if key in virtual_names}<EOL>ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}<EOL>descriptions = {key: value for key, value in self.descriptions.items() if key in virtual_names}<EOL>meta_info = dict(virtual_columns=self.virtual_columns,<EOL>variables=self.variables,<EOL>ucds=ucds, units=units, descriptions=descriptions)<EOL>vaex.utils.write_json_or_yaml(path, meta_info)<EOL>

Writes virtual columns, variables and their ucd,description and units.

        The default implementation is to write this to a file called virtual_meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_virtual_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.


import astropy.units<EOL>try:<EOL><INDENT>path = os.path.join(self.get_private_dir(create=False), "<STR_LIT>")<EOL>if os.path.exists(path):<EOL><INDENT>meta_info = vaex.utils.read_json_or_yaml(path)<EOL>if '<STR_LIT>' not in meta_info:<EOL><INDENT>return<EOL><DEDENT>self.virtual_columns.update(meta_info["<STR_LIT>"])<EOL>self.variables.update(meta_info["<STR_LIT>"])<EOL>self.ucds.update(meta_info["<STR_LIT>"])<EOL>self.descriptions.update(meta_info["<STR_LIT>"])<EOL>units = {key: astropy.units.Unit(value) for key, value in meta_info["<STR_LIT>"].items()}<EOL>self.units.update(units)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>logger.exception("<STR_LIT>")<EOL><DEDENT>

Will read back the virtual column etc, written by :func:`DataFrame.write_virtual_meta`. This will be done when opening a DataFrame.


<EOL>path = os.path.join(self.get_private_dir(create=True), "<STR_LIT>")<EOL>units = {key: str(value) for key, value in self.units.items()}<EOL>meta_info = dict(description=self.description,<EOL>ucds=self.ucds, units=units, descriptions=self.descriptions,<EOL>)<EOL>vaex.utils.write_json_or_yaml(path, meta_info)<EOL>

Writes all meta data, ucd,description and units

        The default implementation is to write this to a file called meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.
        (For instance the vaex hdf5 implementation does this)

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.


import astropy.units<EOL>try:<EOL><INDENT>path = os.path.join(self.get_private_dir(create=False), "<STR_LIT>")<EOL>if os.path.exists(path):<EOL><INDENT>meta_info = vaex.utils.read_json_or_yaml(path)<EOL>self.description = meta_info["<STR_LIT:description>"]<EOL>self.ucds.update(meta_info["<STR_LIT>"])<EOL>self.descriptions.update(meta_info["<STR_LIT>"])<EOL>units = {key: astropy.units.Unit(value) for key, value in meta_info["<STR_LIT>"].items()}<EOL>self.units.update(units)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>logger.exception("<STR_LIT>", path)<EOL><DEDENT>

Will read back the ucd, descriptions, units etc, written by :func:`DataFrame.write_meta`. This will be done when opening a DataFrame.


raise NotImplementedError<EOL>

Returns True if the DataFrame is local, False when a DataFrame is remote.


@_hidden<EOL><INDENT>def subspace(self, *expressions, **kwargs):<DEDENT>

return self(*expressions, **kwargs)<EOL>

Return a :class:`Subspace` for this DataFrame with the given expressions:

        Example:

        >>> subspace_xy = some_df("x", "y")

        :rtype: Subspace
        :param list[str] expressions: list of expressions
        :param kwargs:
        :return:


@_hidden<EOL><INDENT>def subspaces(self, expressions_list=None, dimensions=None, exclude=None, **kwargs):<DEDENT>

if dimensions is not None:<EOL><INDENT>expressions_list = list(itertools.combinations(self.get_column_names(), dimensions))<EOL>if exclude is not None:<EOL><INDENT>import six<EOL>def excluded(expressions):<EOL><INDENT>if callable(exclude):<EOL><INDENT>return exclude(expressions)<EOL><DEDENT>elif isinstance(exclude, six.string_types):<EOL><INDENT>return exclude in expressions<EOL><DEDENT>elif isinstance(exclude, (list, tuple)):<EOL><INDENT>for e in exclude:<EOL><INDENT>if isinstance(e, six.string_types):<EOL><INDENT>if e in expressions:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>elif isinstance(e, (list, tuple)):<EOL><INDENT>if set(e).issubset(expressions):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return False<EOL><DEDENT>expressions_list = [expr for expr in expressions_list if not excluded(expr)]<EOL><DEDENT>logger.debug("<STR_LIT>", expressions_list)<EOL><DEDENT>import vaex.legacy<EOL>return vaex.legacy.Subspaces([self(*expressions, **kwargs) for expressions in expressions_list])<EOL>

Generate a Subspaces object, based on a custom list of expressions or all possible combinations based on
        dimension

        :param expressions_list: list of list of expressions, where the inner list defines the subspace
        :param dimensions: if given, generates a subspace with all possible combinations for that dimension
        :param exclude: list of


def combinations(self, expressions_list=None, dimension=<NUM_LIT:2>, exclude=None, **kwargs):

if dimension is not None:<EOL><INDENT>expressions_list = list(itertools.combinations(self.get_column_names(), dimension))<EOL>if exclude is not None:<EOL><INDENT>import six<EOL>def excluded(expressions):<EOL><INDENT>if callable(exclude):<EOL><INDENT>return exclude(expressions)<EOL><DEDENT>elif isinstance(exclude, six.string_types):<EOL><INDENT>return exclude in expressions<EOL><DEDENT>elif isinstance(exclude, (list, tuple)):<EOL><INDENT>for e in exclude:<EOL><INDENT>if isinstance(e, six.string_types):<EOL><INDENT>if e in expressions:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>elif isinstance(e, (list, tuple)):<EOL><INDENT>if set(e).issubset(expressions):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return False<EOL><DEDENT>expressions_list = [expr for expr in expressions_list if not excluded(expr)]<EOL><DEDENT>logger.debug("<STR_LIT>", expressions_list)<EOL><DEDENT>return expressions_list<EOL>

Generate a list of combinations for the possible expressions for the given dimension.

        :param expressions_list: list of list of expressions, where the inner list defines the subspace
        :param dimensions: if given, generates a subspace with all possible combinations for that dimension
        :param exclude: list of


@vaex.utils.deprecated('<STR_LIT>')<EOL><INDENT>@_hidden<EOL>def __call__(self, *expressions, **kwargs):<DEDENT>

raise NotImplementedError<EOL>


def set_variable(self, name, expression_or_value, write=True):

self.variables[name] = expression_or_value<EOL>

Set the variable to an expression or value defined by expression_or_value.

        Example

        >>> df.set_variable("a", 2.)
        >>> df.set_variable("b", "a**2")
        >>> df.get_variable("b")
        'a**2'
        >>> df.evaluate_variable("b")
        4.0

        :param name: Name of the variable
        :param write: write variable to meta file
        :param expression: value or expression


return self.variables[name]<EOL>

Returns the variable given by name, it will not evaluate it.

        For evaluation, see :func:`DataFrame.evaluate_variable`, see also :func:`DataFrame.set_variable`


if isinstance(self.variables[name], six.string_types):<EOL><INDENT>value = eval(self.variables[name], expression_namespace, self.variables)<EOL>return value<EOL><DEDENT>else:<EOL><INDENT>return self.variables[name]<EOL><DEDENT>


def _evaluate_selection_mask(self, name="<STR_LIT:default>", i1=None, i2=None, selection=None, cache=False):

i1 = i1 or <NUM_LIT:0><EOL>i2 = i2 or len(self)<EOL>scope = scopes._BlockScopeSelection(self, i1, i2, selection, cache=cache)<EOL>return scope.evaluate(name)<EOL>


def evaluate(self, expression, i1=None, i2=None, out=None, selection=None):

raise NotImplementedError<EOL>

Evaluate an expression, and return a numpy array with the results for the full column or a part of it.

        Note that this is not how vaex should be used, since it means a copy of the data needs to fit in memory.

        To get partial results, use i1 and i2

        :param str expression: Name/expression to evaluate
        :param int i1: Start row index, default is the start (0)
        :param int i2: End row index, default is the length of the DataFrame
        :param ndarray out: Output array, to which the result may be written (may be used to reuse an array, or write to
            a memory mapped array)
        :param selection: selection to apply
        :return:


@docsubst<EOL><INDENT>def to_items(self, column_names=None, selection=None, strings=True, virtual=False):<DEDENT>

items = []<EOL>for name in column_names or self.get_column_names(strings=strings, virtual=virtual):<EOL><INDENT>items.append((name, self.evaluate(name, selection=selection)))<EOL><DEDENT>return items<EOL>

Return a list of [(column_name, ndarray), ...)] pairs where the ndarray corresponds to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: list of (name, ndarray) pairs


@docsubst<EOL><INDENT>def to_dict(self, column_names=None, selection=None, strings=True, virtual=False):<DEDENT>

return dict(self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual))<EOL>

Return a dict containing the ndarray corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: dict


@docsubst<EOL><INDENT>def to_copy(self, column_names=None, selection=None, strings=True, virtual=False, selections=True):<DEDENT>

if column_names:<EOL><INDENT>column_names = _ensure_strings_from_expressions(column_names)<EOL><DEDENT>df = vaex.from_items(*self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=False))<EOL>if virtual:<EOL><INDENT>for name, value in self.virtual_columns.items():<EOL><INDENT>df.add_virtual_column(name, value)<EOL><DEDENT><DEDENT>if selections:<EOL><INDENT>for key, value in self.selection_histories.items():<EOL><INDENT>if key != FILTER_SELECTION_NAME:<EOL><INDENT>df.selection_histories[key] = list(value)<EOL><DEDENT><DEDENT>for key, value in self.selection_history_indices.items():<EOL><INDENT>if key != FILTER_SELECTION_NAME:<EOL><INDENT>df.selection_history_indices[key] = value<EOL><DEDENT><DEDENT><DEDENT>df.functions.update(self.functions)<EOL>df.copy_metadata(self)<EOL>return df<EOL>

Return a copy of the DataFrame, if selection is None, it does not copy the data, it just has a reference

        :param column_names: list of column names, to copy, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param selections: copy selections to a new DataFrame
        :return: dict


@docsubst<EOL><INDENT>def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=False, index_name=None):<DEDENT>

import pandas as pd<EOL>data = self.to_dict(column_names=column_names, selection=selection, strings=strings, virtual=virtual)<EOL>if index_name is not None:<EOL><INDENT>if index_name in data:<EOL><INDENT>index = data.pop(index_name)<EOL><DEDENT>else:<EOL><INDENT>index = self.evaluate(index_name, selection=selection)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>index = None<EOL><DEDENT>df = pd.DataFrame(data=data, index=index)<EOL>if index is not None:<EOL><INDENT>df.index.name = index_name<EOL><DEDENT>return df<EOL>

Return a pandas DataFrame containing the ndarray corresponding to the evaluated data

         If index is given, that column is used for the index of the dataframe.

         Example

         >>> df_pandas = df.to_pandas_df(["x", "y", "z"])
         >>> df_copy = vaex.from_pandas(df_pandas)

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index_column: if this column is given it is used for the index of the DataFrame
        :return: pandas.DataFrame object


@docsubst<EOL><INDENT>def to_arrow_table(self, column_names=None, selection=None, strings=True, virtual=False):<DEDENT>

from vaex_arrow.convert import arrow_table_from_vaex_df<EOL>return arrow_table_from_vaex_df(self, column_names, selection, strings, virtual)<EOL>

Returns an arrow Table object containing the arrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: pyarrow.Table object


@docsubst<EOL><INDENT>def to_astropy_table(self, column_names=None, selection=None, strings=True, virtual=False, index=None):<DEDENT>

from astropy.table import Table, Column, MaskedColumn<EOL>meta = dict()<EOL>meta["<STR_LIT:name>"] = self.name<EOL>meta["<STR_LIT:description>"] = self.description<EOL>table = Table(meta=meta)<EOL>for name, data in self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual):<EOL><INDENT>if self.dtype(name) == str_type:  <EOL><INDENT>data = np.array(data).astype('<STR_LIT>')<EOL><DEDENT>meta = dict()<EOL>if name in self.ucds:<EOL><INDENT>meta["<STR_LIT>"] = self.ucds[name]<EOL><DEDENT>if np.ma.isMaskedArray(data):<EOL><INDENT>cls = MaskedColumn<EOL><DEDENT>else:<EOL><INDENT>cls = Column<EOL><DEDENT>table[name] = cls(data, unit=self.unit(name), description=self.descriptions.get(name), meta=meta)<EOL><DEDENT>return table<EOL>

Returns a astropy table object containing the ndarrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index: if this column is given it is used for the index of the DataFrame
        :return: astropy.table.Table object


<EOL>vars = set(self.get_column_names()) | set(self.variables.keys())<EOL>funcs = set(expression_namespace.keys())<EOL>return vaex.expresso.validate_expression(expression, vars, funcs)<EOL>

Validate an expression (may throw Exceptions)


def select(self, boolean_expression, mode="<STR_LIT:replace>", name="<STR_LIT:default>"):

raise NotImplementedError<EOL>

Select rows based on the boolean_expression, if there was a previous selection, the mode is taken into account.

        if boolean_expression is None, remove the selection, has_selection() will returns false

        Note that per DataFrame, only one selection is possible.

        :param str boolean_expression: boolean expression, such as 'x < 0', '(x < 0) || (y > -10)' or None to remove the selection
        :param str mode: boolean operation to perform with the previous selection, "replace", "and", "or", "xor", "subtract"
        :return: None


if isinstance(f_or_array, (np.ndarray, Column)):<EOL><INDENT>data = ar = f_or_array<EOL>if self._length_original is None:<EOL><INDENT>self._length_unfiltered = _len(data)<EOL>self._length_original = _len(data)<EOL>self._index_end = self._length_unfiltered<EOL><DEDENT>if _len(ar) != self.length_original():<EOL><INDENT>if self.filtered:<EOL><INDENT>if len(self) == len(ar):<EOL><INDENT>raise ValueError("<STR_LIT>" % (len(ar), len(self), self.length_unfiltered()))<EOL><DEDENT><DEDENT>raise ValueError("<STR_LIT>" % (len(ar), self.length_original()))<EOL><DEDENT>self.columns[name] = f_or_array<EOL>if name not in self.column_names:<EOL><INDENT>self.column_names.append(name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self._save_assign_expression(name, Expression(self, name))<EOL>


def rename_column(self, name, new_name, unique=False, store_in_state=True):

new_name = vaex.utils.find_valid_name(new_name, used=[] if not unique else list(self))<EOL>data = self.columns.get(name)<EOL>if data is not None:<EOL><INDENT>del self.columns[name]<EOL>self.column_names[self.column_names.index(name)] = new_name<EOL>self.columns[new_name] = data<EOL><DEDENT>else:<EOL><INDENT>expression = self.virtual_columns[name]<EOL>del self.virtual_columns[name]<EOL>self.virtual_columns[new_name] = expression<EOL><DEDENT>if store_in_state:<EOL><INDENT>self._renamed_columns.append((name, new_name))<EOL><DEDENT>for d in [self.ucds, self.units, self.descriptions]:<EOL><INDENT>if name in d:<EOL><INDENT>d[new_name] = d[name]<EOL>del d[name]<EOL><DEDENT><DEDENT>return new_name<EOL>

Renames a column, not this is only the in memory name, this will not be reflected on disk


@_hidden<EOL><INDENT>def add_column_healpix(self, name="<STR_LIT>", longitude="<STR_LIT>", latitude="<STR_LIT>", degrees=True, healpix_order=<NUM_LIT:12>, nest=True):<DEDENT>

import healpy as hp<EOL>if degrees:<EOL><INDENT>scale = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>scale = "<STR_LIT>"<EOL><DEDENT>phi = self.evaluate("<STR_LIT>" % (longitude, scale))<EOL>theta = self.evaluate("<STR_LIT>" % (latitude, scale))<EOL>hp_index = hp.ang2pix(hp.order2nside(healpix_order), theta, phi, nest=nest)<EOL>self.add_column("<STR_LIT>", hp_index)<EOL>

Add a healpix (in memory) column based on a longitude and latitude

        :param name: Name of column
        :param longitude: longitude expression
        :param latitude: latitude expression  (astronomical convenction latitude=90 is north pole)
        :param degrees: If lon/lat are in degrees (default) or radians.
        :param healpix_order: healpix order, >= 0
        :param nest: Nested healpix (default) or ring.


@_hidden<EOL><INDENT>def add_virtual_columns_matrix3d(self, x, y, z, xnew, ynew, znew, matrix, matrix_name='<STR_LIT>', matrix_is_expression=False, translation=[<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>], propagate_uncertainties=False):<DEDENT>

m = matrix<EOL>x, y, z = self._expr(x, y, z)<EOL>self[xnew] = m[<NUM_LIT:0>][<NUM_LIT:0>] * x + m[<NUM_LIT:0>][<NUM_LIT:1>] * y + m[<NUM_LIT:0>][<NUM_LIT:2>] * z + translation[<NUM_LIT:0>]<EOL>self[ynew] = m[<NUM_LIT:1>][<NUM_LIT:0>] * x + m[<NUM_LIT:1>][<NUM_LIT:1>] * y + m[<NUM_LIT:1>][<NUM_LIT:2>] * z + translation[<NUM_LIT:1>]<EOL>self[znew] = m[<NUM_LIT:2>][<NUM_LIT:0>] * x + m[<NUM_LIT:2>][<NUM_LIT:1>] * y + m[<NUM_LIT:2>][<NUM_LIT:2>] * z + translation[<NUM_LIT:2>]<EOL>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[xnew], self[ynew], self[znew]], [x, y, z])<EOL><DEDENT>

:param str x: name of x column
:param str y:
:param str z:
:param str xnew: name of transformed x column
:param str ynew:
:param str znew:
:param list[list] matrix: 2d array or list, with [row,column] order
:param str matrix_name:
:return:


names = _ensure_strings_from_expressions(columns)<EOL>virtual_columns = self._expr(*columns, always_list=True)<EOL>if depending_variables is None:<EOL><INDENT>depending_variables = set()<EOL>for expression in virtual_columns:<EOL><INDENT>depending_variables |= expression.variables()<EOL><DEDENT>depending_variables = list(sorted(list(depending_variables)))<EOL><DEDENT>fs = [self[self.virtual_columns[name]] for name in names]<EOL>jacobian = self._jacobian(fs, depending_variables)<EOL>m = len(fs)<EOL>n = len(depending_variables)<EOL>cov_matrix = self._covariance_matrix_guess(depending_variables, full=cov_matrix == "<STR_LIT>", as_expression=True)<EOL>cov_matrix_out = [[self['<STR_LIT:0>'] for __ in range(m)] for __ in range(m)]<EOL>for i in range(m):<EOL><INDENT>for j in range(m):<EOL><INDENT>for k in range(n):<EOL><INDENT>for l in range(n):<EOL><INDENT>if jacobian[i][k].expression == '<STR_LIT:0>' or jacobian[j][l].expression == '<STR_LIT:0>' or cov_matrix[k][l].expression == '<STR_LIT:0>':<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>cov_matrix_out[i][j] = cov_matrix_out[i][j] + jacobian[i][k] * cov_matrix[k][l] * jacobian[j][l]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>for i in range(m):<EOL><INDENT>for j in range(i + <NUM_LIT:1>):<EOL><INDENT>sigma = cov_matrix_out[i][j]<EOL>sigma = self._expr(vaex.expresso.simplify(_ensure_string_from_expression(sigma)))<EOL>if i != j:<EOL><INDENT>self.add_virtual_column(covariance_format.format(names[i], names[j]), sigma)<EOL><DEDENT>else:<EOL><INDENT>self.add_virtual_column(uncertainty_format.format(names[i]), np.sqrt(sigma))<EOL><DEDENT><DEDENT><DEDENT>

Propagates uncertainties (full covariance matrix) for a set of virtual columns.

        Covariance matrix of the depending variables is guessed by finding columns prefixed by "e"
        or `"e_"` or postfixed by "_error", "_uncertainty", "e" and `"_e"`.
        Off diagonals (covariance or correlation) by postfixes with "_correlation" or "_corr" for
        correlation or "_covariance" or "_cov" for covariances.
        (Note that x_y_cov = x_e * y_e * x_y_correlation.)

        Example

        >>> df = vaex.from_scalars(x=1, y=2, e_x=0.1, e_y=0.2)
        >>> df["u"] = df.x + df.y
        >>> df["v"] = np.log10(df.x)
        >>> df.propagate_uncertainties([df.u, df.v])
        >>> df.u_uncertainty, df.v_uncertainty

        :param columns: list of columns for which to calculate the covariance matrix.
        :param depending_variables: If not given, it is found out automatically, otherwise a list of columns which have uncertainties.
        :param cov_matrix: List of list with expressions giving the covariance matrix, in the same order as depending_variables. If 'full' or 'auto',
            the covariance matrix for the depending_variables will be guessed, where 'full' gives an error if an entry was not found.


x = self[x]<EOL>y = self[y]<EOL>if radians:<EOL><INDENT>to_degrees = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>to_degrees = "<STR_LIT>"<EOL><DEDENT>r = np.sqrt(x**<NUM_LIT:2> + y**<NUM_LIT:2>)<EOL>self[radius_out] = r<EOL>phi = np.arctan2(y, x)<EOL>if not radians:<EOL><INDENT>phi = phi * <NUM_LIT>/np.pi<EOL><DEDENT>self[azimuth_out] = phi<EOL>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[radius_out], self[azimuth_out]])<EOL><DEDENT>

Convert cartesian to polar coordinates

        :param x: expression for x
        :param y: expression for y
        :param radius_out: name for the virtual column for the radius
        :param azimuth_out: name for the virtual column for the azimuth angle
        :param propagate_uncertainties: {propagate_uncertainties}
        :param radians: if True, azimuth is in radians, defaults to degrees
        :return:


@_hidden<EOL><INDENT>def add_virtual_columns_cartesian_velocities_to_spherical(self, x="<STR_LIT:x>", y="<STR_LIT:y>", z="<STR_LIT:z>", vx="<STR_LIT>", vy="<STR_LIT>", vz="<STR_LIT>", vr="<STR_LIT>", vlong="<STR_LIT>", vlat="<STR_LIT>", distance=None):<DEDENT>

<EOL>if distance is None:<EOL><INDENT>distance = "<STR_LIT>".format(**locals())<EOL><DEDENT>self.add_virtual_column(vr, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(vlong, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(vlat, "<STR_LIT>".format(**locals()))<EOL>

Concert velocities from a cartesian to a spherical coordinate system

        TODO: errors

        :param x: name of x column (input)
        :param y:         y
        :param z:         z
        :param vx:       vx
        :param vy:       vy
        :param vz:       vz
        :param vr: name of the column for the radial velocity in the r direction (output)
        :param vlong: name of the column for the velocity component in the longitude direction  (output)
        :param vlat: name of the column for the velocity component in the latitude direction, positive points to the north pole (output)
        :param distance: Expression for distance, if not given defaults to sqrt(x**2+y**2+z**2), but if this column already exists, passing this expression may lead to a better performance
        :return:


@_hidden<EOL><INDENT>def add_virtual_columns_cartesian_velocities_to_polar(self, x="<STR_LIT:x>", y="<STR_LIT:y>", vx="<STR_LIT>", radius_polar=None, vy="<STR_LIT>", vr_out="<STR_LIT>", vazimuth_out="<STR_LIT>",<EOL>propagate_uncertainties=False,):<DEDENT>

x = self._expr(x)<EOL>y = self._expr(y)<EOL>vx = self._expr(vx)<EOL>vy = self._expr(vy)<EOL>if radius_polar is None:<EOL><INDENT>radius_polar = np.sqrt(x**<NUM_LIT:2> + y**<NUM_LIT:2>)<EOL><DEDENT>radius_polar = self._expr(radius_polar)<EOL>self[vr_out]       = (x*vx + y*vy) / radius_polar<EOL>self[vazimuth_out] = (x*vy - y*vx) / radius_polar<EOL>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[vr_out], self[vazimuth_out]])<EOL><DEDENT>

Convert cartesian to polar velocities.

        :param x:
        :param y:
        :param vx:
        :param radius_polar: Optional expression for the radius, may lead to a better performance when given.
        :param vy:
        :param vr_out:
        :param vazimuth_out:
        :param propagate_uncertainties: {propagate_uncertainties}
        :return:


@_hidden<EOL><INDENT>def add_virtual_columns_polar_velocities_to_cartesian(self, x='<STR_LIT:x>', y='<STR_LIT:y>', azimuth=None, vr='<STR_LIT>', vazimuth='<STR_LIT>', vx_out='<STR_LIT>', vy_out='<STR_LIT>', propagate_uncertainties=False):<DEDENT>

x = self._expr(x)<EOL>y = self._expr(y)<EOL>vr = self._expr(vr)<EOL>vazimuth = self._expr(vazimuth)<EOL>if azimuth is not None:<EOL><INDENT>azimuth = self._expr(azimuth)<EOL>azimuth = np.deg2rad(azimuth)<EOL><DEDENT>else:<EOL><INDENT>azimuth = np.arctan2(y, x)<EOL><DEDENT>azimuth = self._expr(azimuth)<EOL>self[vx_out] = vr * np.cos(azimuth) - vazimuth * np.sin(azimuth)<EOL>self[vy_out] = vr * np.sin(azimuth) + vazimuth * np.cos(azimuth)<EOL>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[vx_out], self[vy_out]])<EOL><DEDENT>

Convert cylindrical polar velocities to Cartesian.

        :param x:
        :param y:
        :param azimuth: Optional expression for the azimuth in degrees , may lead to a better performance when given.
        :param vr:
        :param vazimuth:
        :param vx_out:
        :param vy_out:
        :param propagate_uncertainties: {propagate_uncertainties}


@_hidden<EOL><INDENT>def add_virtual_columns_rotation(self, x, y, xnew, ynew, angle_degrees, propagate_uncertainties=False):<DEDENT>

x = _ensure_string_from_expression(x)<EOL>y = _ensure_string_from_expression(y)<EOL>theta = np.radians(angle_degrees)<EOL>matrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])<EOL>m = matrix_name = x + "<STR_LIT:_>" + y + "<STR_LIT>"<EOL>for i in range(<NUM_LIT:2>):<EOL><INDENT>for j in range(<NUM_LIT:2>):<EOL><INDENT>self.set_variable(matrix_name + "<STR_LIT>" % (i, j), matrix[i, j].item())<EOL><DEDENT><DEDENT>self[xnew] = self._expr("<STR_LIT>".format(**locals()))<EOL>self[ynew] = self._expr("<STR_LIT>".format(**locals()))<EOL>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[xnew], self[ynew]])<EOL><DEDENT>

Rotation in 2d.

        :param str x: Name/expression of x column
        :param str y: idem for y
        :param str xnew: name of transformed x column
        :param str ynew:
        :param float angle_degrees: rotation in degrees, anti clockwise
        :return:


@docsubst<EOL><INDENT>@_hidden<EOL>def add_virtual_columns_spherical_to_cartesian(self, alpha, delta, distance, xname="<STR_LIT:x>", yname="<STR_LIT:y>", zname="<STR_LIT:z>",<EOL>propagate_uncertainties=False,<EOL>center=[<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>], center_name="<STR_LIT>", radians=False):<DEDENT>

alpha = self._expr(alpha)<EOL>delta = self._expr(delta)<EOL>distance = self._expr(distance)<EOL>if not radians:<EOL><INDENT>alpha = alpha * self._expr('<STR_LIT>')/<NUM_LIT><EOL>delta = delta * self._expr('<STR_LIT>')/<NUM_LIT><EOL><DEDENT>if center[<NUM_LIT:0>]:<EOL><INDENT>self[xname] = np.cos(alpha) * np.cos(delta) * distance + center[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>self[xname] = np.cos(alpha) * np.cos(delta) * distance<EOL><DEDENT>if center[<NUM_LIT:1>]:<EOL><INDENT>self[yname] = np.sin(alpha) * np.cos(delta) * distance + center[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>self[yname] = np.sin(alpha) * np.cos(delta) * distance<EOL><DEDENT>if center[<NUM_LIT:2>]:<EOL><INDENT>self[zname] =                 np.sin(delta) * distance + center[<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>self[zname] =                 np.sin(delta) * distance<EOL><DEDENT>if propagate_uncertainties:<EOL><INDENT>self.propagate_uncertainties([self[xname], self[yname], self[zname]])<EOL><DEDENT>

Convert spherical to cartesian coordinates.

        :param alpha:
        :param delta: polar angle, ranging from the -90 (south pole) to 90 (north pole)
        :param distance: radial distance, determines the units of x, y and z
        :param xname:
        :param yname:
        :param zname:
        :param propagate_uncertainties: {propagate_uncertainties}
        :param center:
        :param center_name:
        :param radians:
        :return:


@_hidden<EOL><INDENT>def add_virtual_columns_cartesian_to_spherical(self, x="<STR_LIT:x>", y="<STR_LIT:y>", z="<STR_LIT:z>", alpha="<STR_LIT:l>", delta="<STR_LIT:b>", distance="<STR_LIT>", radians=False, center=None, center_name="<STR_LIT>"):<DEDENT>

transform = "<STR_LIT>" if radians else "<STR_LIT>"<EOL>if center is not None:<EOL><INDENT>self.add_variable(center_name, center)<EOL><DEDENT>if center is not None and center[<NUM_LIT:0>] != <NUM_LIT:0>:<EOL><INDENT>x = "<STR_LIT>".format(**locals())<EOL><DEDENT>if center is not None and center[<NUM_LIT:1>] != <NUM_LIT:0>:<EOL><INDENT>y = "<STR_LIT>".format(**locals())<EOL><DEDENT>if center is not None and center[<NUM_LIT:2>] != <NUM_LIT:0>:<EOL><INDENT>z = "<STR_LIT>".format(**locals())<EOL><DEDENT>self.add_virtual_column(distance, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(alpha, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(delta, "<STR_LIT>".format(**locals()))<EOL>

Convert cartesian to spherical coordinates.

        :param x:
        :param y:
        :param z:
        :param alpha:
        :param delta: name for polar angle, ranges from -90 to 90 (or -pi to pi when radians is True).
        :param distance:
        :param radians:
        :param center:
        :param center_name:
        :return:


@_hidden<EOL><INDENT>def add_virtual_columns_aitoff(self, alpha, delta, x, y, radians=True):<DEDENT>

transform = "<STR_LIT>" if radians else "<STR_LIT>"<EOL>aitoff_alpha = "<STR_LIT>" % (alpha, delta)<EOL>aitoff_alpha = re.sub("<STR_LIT>", "<STR_LIT:_>", aitoff_alpha)<EOL>self.add_virtual_column(aitoff_alpha, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(x, "<STR_LIT>".format(**locals()))<EOL>self.add_virtual_column(y, "<STR_LIT>".format(**locals()))<EOL>

Add aitoff (https://en.wikipedia.org/wiki/Aitoff_projection) projection

        :param alpha: azimuth angle
        :param delta: polar angle
        :param x: output name for x coordinate
        :param y: output name for y coordinate
        :param radians: input and output in radians (True), or degrees (False)
        :return:


def add_virtual_column(self, name, expression, unique=False):

type = "<STR_LIT>" if name in self.virtual_columns else "<STR_LIT>"<EOL>expression = _ensure_string_from_expression(expression)<EOL>if name in self.get_column_names(virtual=False):<EOL><INDENT>renamed = '<STR_LIT>' +vaex.utils.find_valid_name(name, used=self.get_column_names())<EOL>expression = self._rename(name, renamed, expression)[<NUM_LIT:0>].expression<EOL><DEDENT>name = vaex.utils.find_valid_name(name, used=[] if not unique else self.get_column_names())<EOL>self.virtual_columns[name] = expression<EOL>self.column_names.append(name)<EOL>self._save_assign_expression(name)<EOL>self.signal_column_changed.emit(self, name, "<STR_LIT>")<EOL>

Add a virtual column to the DataFrame.

        Example:

        >>> df.add_virtual_column("r", "sqrt(x**2 + y**2 + z**2)")
        >>> df.select("r < 10")

        :param: str name: name of virtual column
        :param: expression: expression for the column
        :param str unique: if name is already used, make it unique by adding a postfix, e.g. _1, or _2


del self.virtual_columns[name]<EOL>self.signal_column_changed.emit(self, name, "<STR_LIT>")<EOL>

Deletes a virtual column from a DataFrame.


def add_variable(self, name, expression, overwrite=True, unique=True):

if unique or overwrite or name not in self.variables:<EOL><INDENT>existing_names = self.get_column_names(virtual=False) + list(self.variables.keys())<EOL>name = vaex.utils.find_valid_name(name, used=[] if not unique else existing_names)<EOL>self.variables[name] = expression<EOL>self.signal_variable_changed.emit(self, name, "<STR_LIT>")<EOL>if unique:<EOL><INDENT>return name<EOL><DEDENT><DEDENT>

Add a variable to to a DataFrame.

        A variable may refer to other variables, and virtual columns and expression may refer to variables.

        Example

        >>> df.add_variable('center', 0)
        >>> df.add_virtual_column('x_prime', 'x-center')
        >>> df.select('x_prime < 0')

        :param: str name: name of virtual varible
        :param: expression: expression for the variable


del self.variables[name]<EOL>self.signal_variable_changed.emit(self, name, "<STR_LIT>")<EOL>


return self[:min(n, len(self))]<EOL>

Return a shallow copy a DataFrame with the first n rows.


N = len(self)<EOL>return self[max(<NUM_LIT:0>, N - n):min(len(self), N)]<EOL>

Return a shallow copy a DataFrame with the last n rows.


from IPython import display<EOL>display.display(display.HTML(self._head_and_tail_table(n)))<EOL>

Display the first and last n elements of a DataFrame.


def describe(self, strings=True, virtual=True, selection=None):

import pandas as pd<EOL>N = len(self)<EOL>columns = {}<EOL>for feature in self.get_column_names(strings=strings, virtual=virtual)[:]:<EOL><INDENT>dtype = str(self.dtype(feature)) if self.dtype(feature) != str else '<STR_LIT:str>'<EOL>if self.dtype(feature) == str_type or self.dtype(feature).kind in ['<STR_LIT:S>', '<STR_LIT>', '<STR_LIT:O>']:<EOL><INDENT>count = self.count(feature, selection=selection, delay=True)<EOL>self.execute()<EOL>count = count.get()<EOL>columns[feature] = ((dtype, count, N-count, '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>count = self.count(feature, selection=selection, delay=True)<EOL>mean = self.mean(feature, selection=selection, delay=True)<EOL>std = self.std(feature, selection=selection, delay=True)<EOL>minmax = self.minmax(feature, selection=selection, delay=True)<EOL>self.execute()<EOL>count, mean, std, minmax = count.get(), mean.get(), std.get(), minmax.get()<EOL>count = int(count)<EOL>columns[feature] = ((dtype, count, N-count, mean, std, minmax[<NUM_LIT:0>], minmax[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>return pd.DataFrame(data=columns, index=['<STR_LIT>', '<STR_LIT:count>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>

Give a description of the DataFrame.

        >>> import vaex
        >>> df = vaex.example()[['x', 'y', 'z']]
        >>> df.describe()
                         x          y          z
        dtype      float64    float64    float64
        count       330000     330000     330000
        missing          0          0          0
        mean    -0.0671315 -0.0535899  0.0169582
        std        7.31746    7.78605    5.05521
        min       -128.294   -71.5524   -44.3342
        max        271.366    146.466    50.7185
        >>> df.describe(selection=df.x > 0)
                           x         y          z
        dtype        float64   float64    float64
        count         164060    164060     164060
        missing       165940    165940     165940
        mean         5.13572 -0.486786 -0.0868073
        std          5.18701   7.61621    5.02831
        min      1.51635e-05  -71.5524   -44.3342
        max          271.366   78.0724    40.2191

        :param bool strings: Describe string columns or not
        :param bool virtual: Describe virtual columns or not
        :param selection: Optional selection to use.
        :return: Pandas dataframe


def cat(self, i1, i2, format='<STR_LIT:html>'):

from IPython import display<EOL>if format == '<STR_LIT:html>':<EOL><INDENT>output = self._as_html_table(i1, i2)<EOL>display.display(display.HTML(output))<EOL><DEDENT>else:<EOL><INDENT>output = self._as_table(i1, i2, format=format)<EOL>print(output)<EOL><DEDENT>

Display the DataFrame from row i1 till i2

        For format, see https://pypi.org/project/tabulate/

        :param int i1: Start row
        :param int i2: End row.
        :param str format: Format to use, e.g. 'html', 'plain', 'latex'


self._output_css()<EOL>return self._head_and_tail_table()<EOL>


return <NUM_LIT:0><EOL>


return self._current_row is not None<EOL>

Returns True/False is there currently is a picked row.


return self._current_row<EOL>

Individual rows can be 'picked', this is the index (integer) of the current row, or None there is nothing picked.


if (value is not None) and ((value < <NUM_LIT:0>) or (value >= len(self))):<EOL><INDENT>raise IndexError("<STR_LIT>" % (value, len(self)))<EOL><DEDENT>self._current_row = value<EOL>self.signal_pick.emit(self, value)<EOL>

Set the current row, and emit the signal signal_pick.


return len(self.column_names)<EOL>

Returns the number of columns (including virtual columns).


def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):

def column_filter(name):<EOL><INDENT>'''<STR_LIT>'''<EOL>if regex and not re.match(regex, name):<EOL><INDENT>return False<EOL><DEDENT>if not virtual and name in self.virtual_columns:<EOL><INDENT>return False<EOL><DEDENT>if not strings and (self.dtype(name) == str_type or self.dtype(name).type == np.string_):<EOL><INDENT>return False<EOL><DEDENT>if not hidden and name.startswith('<STR_LIT>'):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL><DEDENT>return [name for name in self.column_names if column_filter(name)]<EOL>

Return a list of column names

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']

        :param virtual: If False, skip virtual columns
        :param hidden: If False, skip hidden columns
        :param strings: If False, skip string columns
        :param regex: Only return column names matching the (optional) regular expression
        :rtype: list of str

        Example:
        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']


if not self.filtered:<EOL><INDENT>return self._length_unfiltered<EOL><DEDENT>else:<EOL><INDENT>return int(self.count())<EOL><DEDENT>

Returns the number of rows in the DataFrame (filtering applied).


raise NotImplementedError<EOL>

Returns the number of rows that are selected.


return self._length_original<EOL>

the full length of the DataFrame, independent what active_fraction is, or filtering. This is the real length of the underlying ndarrays.


return self._length_unfiltered<EOL>

The length of the arrays that should be considered (respecting active range), but without filtering.


return self._active_fraction<EOL>

Value in the range (0, 1], to work only with a subset of rows.


if value != self._active_fraction:<EOL><INDENT>self._active_fraction = value<EOL>self.select(None)<EOL>self.set_current_row(None)<EOL>self._length_unfiltered = int(round(self._length_original * self._active_fraction))<EOL>self._index_start = <NUM_LIT:0><EOL>self._index_end = self._length_unfiltered<EOL>self.signal_active_fraction_changed.emit(self, value)<EOL><DEDENT>

Sets the active_fraction, set picked row to None, and remove selection.

        TODO: we may be able to keep the selection, if we keep the expression, and also the picked row


logger.debug("<STR_LIT>", (i1, i2))<EOL>self._active_fraction = (i2 - i1) / float(self.length_original())<EOL>self._index_start = i1<EOL>self._index_end = i2<EOL>self.select(None)<EOL>self.set_current_row(None)<EOL>self._length_unfiltered = i2 - i1<EOL>self.signal_active_fraction_changed.emit(self, self._active_fraction)<EOL>


df = self if inplace else self.copy()<EOL>for name in df:<EOL><INDENT>column = df.columns.get(name)<EOL>if column is not None:<EOL><INDENT>if self._index_start == <NUM_LIT:0> and len(column) == self._index_end:<EOL><INDENT>pass  <EOL><DEDENT>else:<EOL><INDENT>if isinstance(column, np.ndarray):  <EOL><INDENT>df.columns[name] = column[self._index_start:self._index_end]<EOL><DEDENT>else:<EOL><INDENT>df.columns[name] = column.trim(self._index_start, self._index_end)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>df._length_original = self.length_unfiltered()<EOL>df._length_unfiltered = df._length_original<EOL>df._index_start = <NUM_LIT:0><EOL>df._index_end = df._length_original<EOL>df._active_fraction = <NUM_LIT:1><EOL>return df<EOL>

Return a DataFrame, where all columns are 'trimmed' by the active range.

        For the returned DataFrame, df.get_active_range() returns (0, df.length_original()).

        {note_copy}

        :param inplace: {inplace}
        :rtype: DataFrame


df = self.copy()<EOL>direct_indices_map = {}<EOL>indices = np.array(indices)<EOL>for name in df:<EOL><INDENT>column = df.columns.get(name)<EOL>if column is not None:<EOL><INDENT>if isinstance(column, ColumnIndexed):<EOL><INDENT>if id(column.indices) not in direct_indices_map:<EOL><INDENT>direct_indices = column.indices[indices]<EOL>direct_indices_map[id(column.indices)] = direct_indices<EOL><DEDENT>else:<EOL><INDENT>direct_indices = direct_indices_map[id(column.indices)]<EOL><DEDENT>df.columns[name] = ColumnIndexed(column.df, direct_indices, column.name)<EOL><DEDENT>else:<EOL><INDENT>df.columns[name] = ColumnIndexed(self, indices, name)<EOL><DEDENT><DEDENT><DEDENT>df._length_original = len(indices)<EOL>df._length_unfiltered = df._length_original<EOL>df.set_selection(None, name=FILTER_SELECTION_NAME)<EOL>return df<EOL>

Returns a DataFrame containing only rows indexed by indices

        {note_copy}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df.take([0,2])
         #  s      x
         0  a      1
         1  c      3

        :param indices: sequence (list or numpy array) with row numbers
        :return: DataFrame which is a shallow copy of the original data.
        :rtype: DataFrame


trimmed = self.trim()<EOL>if trimmed.filtered:<EOL><INDENT>indices = trimmed._filtered_range_to_unfiltered_indices(<NUM_LIT:0>, len(trimmed))<EOL>return trimmed.take(indices)<EOL><DEDENT>else:<EOL><INDENT>return trimmed<EOL><DEDENT>

Return a DataFrame containing only the filtered rows.

        {note_copy}

        The resulting DataFrame may be more efficient to work with when the original DataFrame is
        heavily filtered (contains just a small number of rows).

        If no filtering is applied, it returns a trimmed view.
        For the returned df, len(df) == df.length_original() == df.length_unfiltered()

        :rtype: DataFrame


@docsubst<EOL><INDENT>def sample(self, n=None, frac=None, replace=False, weights=None, random_state=None):<DEDENT>

self = self.extract()<EOL>if type(random_state) == int or random_state is None:<EOL><INDENT>random_state = np.random.RandomState(seed=random_state)<EOL><DEDENT>if n is None and frac is None:<EOL><INDENT>n = <NUM_LIT:1><EOL><DEDENT>elif frac is not None:<EOL><INDENT>n = int(round(frac * len(self)))<EOL><DEDENT>weights_values = None<EOL>if weights is not None:<EOL><INDENT>weights_values = self.evaluate(weights)<EOL>weights_values = weights_values / self.sum(weights)<EOL><DEDENT>indices = random_state.choice(len(self), n, replace=replace, p=weights_values)<EOL>return self.take(indices)<EOL>

Returns a DataFrame with a random set of rows

        {note_copy}

        Provide either n or frac.

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df
          #  s      x
          0  a      1
          1  b      2
          2  c      3
          3  d      4
        >>> df.sample(n=2, random_state=42) # 2 random rows, fixed seed
          #  s      x
          0  b      2
          1  d      4
        >>> df.sample(frac=1, random_state=42) # 'shuffling'
          #  s      x
          0  c      3
          1  a      1
          2  d      4
          3  b      2
        >>> df.sample(frac=1, replace=True, random_state=42) # useful for bootstrap (may contain repeated samples)
          #  s      x
          0  d      4
          1  a      1
          2  a      1
          3  d      4

        :param int n: number of samples to take (default 1 if frac is None)
        :param float frac: fractional number of takes to take
        :param bool replace: If true, a row may be drawn multiple times
        :param str or expression weights: (unnormalized) probability that a row can be drawn
        :param int or RandomState: seed or RandomState for reproducability, when None a random seed it chosen
        :return: {return_shallow_copy}
        :rtype: DataFrame


@docsubst<EOL><INDENT>@vaex.utils.gen_to_list<EOL>def split_random(self, frac, random_state=None):<DEDENT>

self = self.extract()<EOL>if type(random_state) == int or random_state is None:<EOL><INDENT>random_state = np.random.RandomState(seed=random_state)<EOL><DEDENT>indices = random_state.choice(len(self), len(self), replace=False)<EOL>return self.take(indices).split(frac)<EOL>

Returns a list containing random portions of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex, import numpy as np
        >>> np.random.seed(111)
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split_random(frac=0.3, random_state=42):
        ...     print(dfs.x.values)
        ...
        [8 1 5]
        [0 7 2 9 4 3 6]
        >>> for split in df.split_random(frac=[0.2, 0.3, 0.5], random_state=42):
        ...     print(dfs.x.values)
        [8 1]
        [5 0 7]
        [2 9 4 3 6]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :param int random_state: (default, None) Random number seed for reproducibility.
        :return: A list of DataFrames.
        :rtype: list


self = self.extract()<EOL>if _issequence(frac):<EOL><INDENT>total = sum(frac)<EOL>frac = [k / total for k in frac]<EOL><DEDENT>else:<EOL><INDENT>assert frac <= <NUM_LIT:1>, "<STR_LIT>"<EOL>frac = [frac, <NUM_LIT:1> - frac]<EOL><DEDENT>offsets = np.round(np.cumsum(frac) * len(self)).astype(np.int64)<EOL>start = <NUM_LIT:0><EOL>for offset in offsets:<EOL><INDENT>yield self[start:offset]<EOL>start = offset<EOL><DEDENT>

Returns a list containing ordered subsets of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split(frac=0.3):
        ...     print(dfs.x.values)
        ...
        [0 1 3]
        [3 4 5 6 7 8 9]
        >>> for split in df.split(frac=[0.2, 0.3, 0.5]):
        ...     print(dfs.x.values)
        [0 1]
        [2 3 4]
        [5 6 7 8 9]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :return: A list of DataFrames.
        :rtype: list


@docsubst<EOL><INDENT>def sort(self, by, ascending=True, kind='<STR_LIT>'):<DEDENT>

self = self.trim()<EOL>values = self.evaluate(by, filtered=False)<EOL>indices = np.argsort(values, kind=kind)<EOL>if not ascending:<EOL><INDENT>indices = indices[::-<NUM_LIT:1>].copy()  <EOL><DEDENT>return self.take(indices)<EOL>

Return a sorted DataFrame, sorted by the expression 'by'

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df['y'] = (df.x-1.8)**2
        >>> df
          #  s      x     y
          0  a      1  0.64
          1  b      2  0.04
          2  c      3  1.44
          3  d      4  4.84
        >>> df.sort('y', ascending=False)  # Note: passing '(x-1.8)**2' gives the same result
          #  s      x     y
          0  d      4  4.84
          1  c      3  1.44
          2  a      1  0.64
          3  b      2  0.04

        :param str or expression by: expression to sort by
        :param bool ascending: ascending (default, True) or descending (False)
        :param str kind: kind of algorithm to use (passed to numpy.argsort)


@docsubst<EOL><INDENT>def fillna(self, value, fill_nan=True, fill_masked=True, column_names=None, prefix='<STR_LIT>', inplace=False):<DEDENT>

df = self.trim(inplace=inplace)<EOL>column_names = column_names or list(self)<EOL>for name in column_names:<EOL><INDENT>column = df.columns.get(name)<EOL>if column is not None:<EOL><INDENT>new_name = df.rename_column(name, prefix + name)<EOL>expr = df[new_name]<EOL>df[name] = df.func.fillna(expr, value, fill_nan=fill_nan, fill_masked=fill_masked)<EOL><DEDENT>else:<EOL><INDENT>df[name] = df.func.fillna(df[name], value, fill_nan=fill_nan, fill_masked=fill_masked)<EOL><DEDENT><DEDENT>return df<EOL>

Return a DataFrame, where missing values/NaN are filled with 'value'.

        The original columns will be renamed, and by default they will be hidden columns. No data is lost.

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex
        >>> import numpy as np
        >>> x = np.array([3, 1, np.nan, 10, np.nan])
        >>> df = vaex.from_arrays(x=x)
        >>> df_filled = df.fillna(value=-1, column_names=['x'])
        >>> df_filled
          #    x
          0    3
          1    1
          2   -1
          3   10
          4   -1

        :param float value: The value to use for filling nan or masked values.
        :param bool fill_na: If True, fill np.nan values with `value`.
        :param bool fill_masked: If True, fill masked values with `values`.
        :param list column_names: List of column names in which to fill missing values.
        :param str prefix: The prefix to give the original columns.
        :param inplace: {inplace}


def materialize(self, virtual_column, inplace=False):

df = self.trim(inplace=inplace)<EOL>virtual_column = _ensure_string_from_expression(virtual_column)<EOL>if virtual_column not in df.virtual_columns:<EOL><INDENT>raise KeyError('<STR_LIT>' % virtual_column)<EOL><DEDENT>ar = df.evaluate(virtual_column, filtered=False)<EOL>del df[virtual_column]<EOL>df.add_column(virtual_column, ar)<EOL>return df<EOL>

Returns a new DataFrame where the virtual column is turned into an in memory numpy array.

        Example:

        >>> x = np.arange(1,4)
        >>> y = np.arange(2,5)
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5 # 'r' is a virtual column (computed on the fly)
        >>> df = df.materialize('r')  # now 'r' is a 'real' column (i.e. a numpy array)

        :param inplace: {inplace}


name = _normalize_selection_name(name)<EOL>selection_history = self.selection_histories[name]<EOL>index = self.selection_history_indices[name]<EOL>if index == -<NUM_LIT:1>:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return selection_history[index]<EOL><DEDENT>

Get the current selection object (mostly for internal use atm).


def selection_undo(self, name="<STR_LIT:default>", executor=None):

logger.debug("<STR_LIT>")<EOL>executor = executor or self.executor<EOL>assert self.selection_can_undo(name=name)<EOL>selection_history = self.selection_histories[name]<EOL>index = self.selection_history_indices[name]<EOL>self.selection_history_indices[name] -= <NUM_LIT:1><EOL>self.signal_selection_changed.emit(self)<EOL>logger.debug("<STR_LIT>", selection_history, self.selection_history_indices[name])<EOL>


def selection_redo(self, name="<STR_LIT:default>", executor=None):

logger.debug("<STR_LIT>")<EOL>executor = executor or self.executor<EOL>assert self.selection_can_redo(name=name)<EOL>selection_history = self.selection_histories[name]<EOL>index = self.selection_history_indices[name]<EOL>next = selection_history[index + <NUM_LIT:1>]<EOL>self.selection_history_indices[name] += <NUM_LIT:1><EOL>self.signal_selection_changed.emit(self)<EOL>logger.debug("<STR_LIT>", selection_history, index)<EOL>


return self.selection_history_indices[name] > -<NUM_LIT:1><EOL>


return (self.selection_history_indices[name] + <NUM_LIT:1>) < len(self.selection_histories[name])<EOL>


def select(self, boolean_expression, mode="<STR_LIT:replace>", name="<STR_LIT:default>", executor=None):

boolean_expression = _ensure_string_from_expression(boolean_expression)<EOL>if boolean_expression is None and not self.has_selection(name=name):<EOL><INDENT>pass  <EOL>self.signal_selection_changed.emit(self)  <EOL><DEDENT>else:<EOL><INDENT>def create(current):<EOL><INDENT>return selections.SelectionExpression(boolean_expression, current, mode) if boolean_expression else None<EOL><DEDENT>self._selection(create, name)<EOL><DEDENT>

Perform a selection, defined by the boolean expression, and combined with the previous selection using the given mode.

        Selections are recorded in a history tree, per name, undo/redo can be done for them separately.

        :param str boolean_expression: Any valid column expression, with comparison operators
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name: history tree or selection 'slot' to use
        :param executor:
        :return:


def select_non_missing(self, drop_nan=True, drop_masked=True, column_names=None, mode="<STR_LIT:replace>", name="<STR_LIT:default>"):

column_names = column_names or self.get_column_names(virtual=False)<EOL>def create(current):<EOL><INDENT>return selections.SelectionDropNa(drop_nan, drop_masked, column_names, current, mode)<EOL><DEDENT>self._selection(create, name)<EOL>

Create a selection that selects rows having non missing values for all columns in column_names.

        The name reflect Panda's, no rows are really dropped, but a mask is kept to keep track of the selection

        :param drop_nan: drop rows when there is a NaN in any of the columns (will only affect float values)
        :param drop_masked: drop rows when there is a masked value in any of the columns
        :param column_names: The columns to consider, default: all (real, non-virtual) columns
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name: history tree or selection 'slot' to use
        :return:


def dropna(self, drop_nan=True, drop_masked=True, column_names=None):

copy = self.copy()<EOL>copy.select_non_missing(drop_nan=drop_nan, drop_masked=drop_masked, column_names=column_names,<EOL>name=FILTER_SELECTION_NAME, mode='<STR_LIT>')<EOL>return copy<EOL>

Create a shallow copy of a DataFrame, with filtering set using select_non_missing.

        :param drop_nan: drop rows when there is a NaN in any of the columns (will only affect float values)
        :param drop_masked: drop rows when there is a masked value in any of the columns
        :param column_names: The columns to consider, default: all (real, non-virtual) columns
        :rtype: DataFrame


logger.debug("<STR_LIT>")<EOL>self.select(None, name=name)<EOL>


def select_rectangle(self, x, y, limits, mode="<STR_LIT:replace>", name="<STR_LIT:default>"):

self.select_box([x, y], limits, mode=mode, name=name)<EOL>

Select a 2d rectangular box in the space given by x and y, bounds by limits.

        Example:

        >>> df.select_box('x', 'y', [(0, 10), (0, 1)])

        :param x: expression for the x space
        :param y: expression fo the y space
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:


def select_box(self, spaces, limits, mode="<STR_LIT:replace>", name="<STR_LIT:default>"):

sorted_limits = [(min(l), max(l)) for l in limits]<EOL>expressions = ["<STR_LIT>" % (expression, lmin, expression, lmax) for<EOL>(expression, (lmin, lmax)) in zip(spaces, sorted_limits)]<EOL>self.select("<STR_LIT:&>".join(expressions), mode=mode, name=name)<EOL>

Select a n-dimensional rectangular box bounded by limits.

        The following examples are equivalent:

        >>> df.select_box(['x', 'y'], [(0, 10), (0, 1)])
        >>> df.select_rectangle('x', 'y', [(0, 10), (0, 1)])

        :param spaces: list of expressions
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:
        :param name:
        :return:


def select_circle(self, x, y, xc, yc, r, mode="<STR_LIT:replace>", name="<STR_LIT:default>", inclusive=True):

<EOL>if inclusive:<EOL><INDENT>expr = (self[x] - xc)**<NUM_LIT:2> + (self[y] - yc)**<NUM_LIT:2> <= r**<NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>expr = (self[x] - xc)**<NUM_LIT:2> + (self[y] - yc)**<NUM_LIT:2> < r**<NUM_LIT:2><EOL><DEDENT>self.select(boolean_expression=expr, mode=mode, name=name)<EOL>

Select a circular region centred on xc, yc, with a radius of r.

Example:

>>> df.select_circle('x','y',2,3,1)

:param x: expression for the x space
:param y: expression for the y space
:param xc: location of the centre of the circle in x
:param yc: location of the centre of the circle in y
:param r: the radius of the circle
:param name: name of the selection
:param mode:
:return:


def select_ellipse(self, x, y, xc, yc, width, height, angle=<NUM_LIT:0>, mode="<STR_LIT:replace>", name="<STR_LIT:default>", radians=False, inclusive=True):

<EOL>if radians:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>alpha = np.deg2rad(angle)<EOL><DEDENT>xr = width / <NUM_LIT:2><EOL>yr = height / <NUM_LIT:2><EOL>r = max(xr, yr)<EOL>a = xr / r<EOL>b = yr / r<EOL>expr = "<STR_LIT>".format(**locals())<EOL>if inclusive:<EOL><INDENT>expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**<NUM_LIT:2> / a**<NUM_LIT:2> + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**<NUM_LIT:2> / b**<NUM_LIT:2> <= r**<NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**<NUM_LIT:2> / a**<NUM_LIT:2> + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**<NUM_LIT:2> / b**<NUM_LIT:2> < r**<NUM_LIT:2><EOL><DEDENT>self.select(boolean_expression=expr, mode=mode, name=name)<EOL>

Select an elliptical region centred on xc, yc, with a certain width, height
and angle.

Example:

>>> df.select_ellipse('x','y', 2, -1, 5,1, 30, name='my_ellipse')

:param x: expression for the x space
:param y: expression for the y space
:param xc: location of the centre of the ellipse in x
:param yc: location of the centre of the ellipse in y
:param width: the width of the ellipse (diameter)
:param height: the width of the ellipse (diameter)
:param angle: (degrees) orientation of the ellipse, counter-clockwise
              measured from the y axis
:param name: name of the selection
:param mode:
:return:


def select_lasso(self, expression_x, expression_y, xsequence, ysequence, mode="<STR_LIT:replace>", name="<STR_LIT:default>", executor=None):

def create(current):<EOL><INDENT>return selections.SelectionLasso(expression_x, expression_y, xsequence, ysequence, current, mode)<EOL><DEDENT>self._selection(create, name, executor=executor)<EOL>

For performance reasons, a lasso selection is handled differently.

        :param str expression_x: Name/expression for the x coordinate
        :param str expression_y: Name/expression for the y coordinate
        :param xsequence: list of x numbers defining the lasso, together with y
        :param ysequence:
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name:
        :param executor:
        :return:


def select_inverse(self, name="<STR_LIT:default>", executor=None):

def create(current):<EOL><INDENT>return selections.SelectionInvert(current)<EOL><DEDENT>self._selection(create, name, executor=executor)<EOL>

Invert the selection, i.e. what is selected will not be, and vice versa

        :param str name:
        :param executor:
        :return:


def set_selection(self, selection, name="<STR_LIT:default>", executor=None):

def create(current):<EOL><INDENT>return selection<EOL><DEDENT>self._selection(create, name, executor=executor, execute_fully=True)<EOL>

Sets the selection object

        :param selection: Selection object
        :param name: selection 'slot'
        :param executor:
        :return:


def _selection(self, create_selection, name, executor=None, execute_fully=False):

selection_history = self.selection_histories[name]<EOL>previous_index = self.selection_history_indices[name]<EOL>current = selection_history[previous_index] if selection_history else None<EOL>selection = create_selection(current)<EOL>executor = executor or self.executor<EOL>selection_history.append(selection)<EOL>self.selection_history_indices[name] += <NUM_LIT:1><EOL>del selection_history[self.selection_history_indices[name]:-<NUM_LIT:1>]<EOL>if <NUM_LIT:0>:<EOL><INDENT>if self.is_local():<EOL><INDENT>if selection:<EOL><INDENT>result = vaex.promise.Promise.fulfilled(None)<EOL>self.signal_selection_changed.emit(self)<EOL><DEDENT>else:<EOL><INDENT>result = vaex.promise.Promise.fulfilled(None)<EOL>self.signal_selection_changed.emit(self)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.signal_selection_changed.emit(self)<EOL>result = vaex.promise.Promise.fulfilled(None)<EOL><DEDENT><DEDENT>self.signal_selection_changed.emit(self)<EOL>result = vaex.promise.Promise.fulfilled(None)<EOL>logger.debug("<STR_LIT>", selection_history, self.selection_history_indices[name])<EOL>return result<EOL>

select_lasso and select almost share the same code


return self.get_selection(name) is not None<EOL>

Returns True if there is a selection with the given name.