if secure:<EOL><INDENT>url_base = GRAVATAR_SECURE_URL<EOL><DEDENT>else:<EOL><INDENT>url_base = GRAVATAR_URL<EOL><DEDENT>email_hash = calculate_gravatar_hash(email)<EOL>url = '<STR_LIT>'.format(base=url_base, hash=email_hash)<EOL>return url<EOL>

Builds a url to a gravatar profile from an email address.

:param email: The email to fetch the gravatar for
:param secure: If True use https, otherwise plain http


v = sys.version[<NUM_LIT:0>]<EOL>if v == '<STR_LIT:3>':<EOL><INDENT>return input(prompt)<EOL><DEDENT>else:<EOL><INDENT>return raw_input(prompt)<EOL><DEDENT>


def choice_input(options=[], prompt='<STR_LIT>', <EOL>showopts=True, qopt=False):

choice = None<EOL>if showopts:<EOL><INDENT>prompt = prompt + '<STR_LIT:U+0020>' + str(options)<EOL><DEDENT>if qopt:<EOL><INDENT>prompt = prompt + '<STR_LIT>'<EOL><DEDENT>while not choice:<EOL><INDENT>try:<EOL><INDENT>choice = string_input(prompt + '<STR_LIT:U+0020>')<EOL><DEDENT>except SyntaxError:<EOL><INDENT>if options == []:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if choice:<EOL><INDENT>if choice in options:<EOL><INDENT>return choice<EOL><DEDENT>elif qopt == True and choice == '<STR_LIT:q>':<EOL><INDENT>choice = None<EOL>is_sure = string_input('<STR_LIT>')<EOL>if is_sure in ('<STR_LIT:Y>', '<STR_LIT:y>', '<STR_LIT:yes>'):<EOL><INDENT>exit('<STR_LIT>')<EOL><DEDENT><DEDENT>elif options == []:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' + str(options) +<EOL>'<STR_LIT>')<EOL>if options:<EOL><INDENT>choice = None<EOL><DEDENT><DEDENT><DEDENT>elif options == []:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' + str(options) + <EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>

Get input from a list of choices (q to quit)


def long_input(prompt='<STR_LIT>' +'<STR_LIT>' +'<STR_LIT>',<EOL>maxlines = None, maxlength = None):

lines = []<EOL>print(prompt)<EOL>lnum = <NUM_LIT:1><EOL>try:<EOL><INDENT>while True:<EOL><INDENT>if maxlines:<EOL><INDENT>if lnum > maxlines:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>if maxlength:<EOL><INDENT>lines.append(string_input('<STR_LIT>')[:maxlength])<EOL><DEDENT>else:<EOL><INDENT>lines.append(string_input('<STR_LIT>'))<EOL><DEDENT>lnum += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>if maxlength:<EOL><INDENT>lines.append(string_input('<STR_LIT>')[:maxlength])<EOL><DEDENT>else:<EOL><INDENT>lines.append(string_input('<STR_LIT>'))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>except EOFError:<EOL><INDENT>pass<EOL><DEDENT>finally:<EOL><INDENT>return '<STR_LIT:\n>'.join(lines)<EOL><DEDENT>


def list_input(prompt='<STR_LIT>' +'<STR_LIT>' +'<STR_LIT>',<EOL>maxitems=None, maxlength=None):

lines = []<EOL>print(prompt)<EOL>inum = <NUM_LIT:1><EOL>try:<EOL><INDENT>while True:<EOL><INDENT>if maxitems:<EOL><INDENT>if inum > maxitems:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>if maxlength:<EOL><INDENT>lines.append(string_input('<STR_LIT>')[:maxlength])<EOL><DEDENT>else:<EOL><INDENT>lines.append(string_input('<STR_LIT>'))<EOL><DEDENT>inum += <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>if maxlength:<EOL><INDENT>lines.append(string_input('<STR_LIT>')[:maxlength])<EOL><DEDENT>else:<EOL><INDENT>lines.append(string_input('<STR_LIT>'))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>except EOFError:<EOL><INDENT>pass<EOL><DEDENT>finally:<EOL><INDENT>return lines<EOL><DEDENT>


fileok = False<EOL>while not fileok:<EOL><INDENT>filename = string_input('<STR_LIT>')<EOL>if extension:<EOL><INDENT>if not filename.endswith(extension):<EOL><INDENT>if extension.startswith('<STR_LIT:.>'):<EOL><INDENT>filename = filename + extension<EOL><DEDENT>else:<EOL><INDENT>filename = filename + '<STR_LIT:.>' + extension<EOL><DEDENT><DEDENT><DEDENT>if os.path.isfile(filename):<EOL><INDENT>choice = choice_input(prompt=filename +'<STR_LIT>',<EOL>options=['<STR_LIT:y>', '<STR_LIT:n>'])<EOL>if choice == '<STR_LIT:y>':<EOL><INDENT>try:<EOL><INDENT>nowtime = time.time()<EOL>with open(filename, '<STR_LIT:a>') as f:<EOL><INDENT>os.utime(filename, (nowtime, nowtime))<EOL><DEDENT>fileok = True<EOL><DEDENT>except IOError:<EOL><INDENT>print('<STR_LIT>' + filename +'<STR_LIT>')<EOL><DEDENT>except PermissionError:<EOL><INDENT>print('<STR_LIT>' + filename +'<STR_LIT>')<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>print(filename + '<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>choice = choice_input(<EOL>prompt=filename + '<STR_LIT>',<EOL>options=['<STR_LIT:y>', '<STR_LIT:n>'])<EOL>if choice == '<STR_LIT:y>':<EOL><INDENT>try:<EOL><INDENT>nowtime = time.time()<EOL>with open(filename, '<STR_LIT:w>') as f:<EOL><INDENT>os.utime(filename, (nowtime, nowtime))<EOL><DEDENT>fileok = True<EOL><DEDENT>except IOError:<EOL><INDENT>print('<STR_LIT>' + filename +'<STR_LIT>')<EOL><DEDENT>except PermissionError:<EOL><INDENT>print('<STR_LIT>' + filename +'<STR_LIT>')<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>print(filename + '<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return filename<EOL>


raise NotImplementedError<EOL>

Returns the full code, encoded in the barcode.

        :returns: Full human readable code.
        :rtype: String


output = self.render(options)<EOL>_filename = self.writer.save(filename, output)<EOL>return _filename<EOL>

Renders the barcode and saves it in `filename`.

        :parameters:
            filename : String
                Filename to save the barcode in (without filename
                extension).
            options : Dict
                The same as in `self.render`.

        :returns: The full filename with extension.
        :rtype: String


output = self.render(options)<EOL>if hasattr(output, '<STR_LIT>'):<EOL><INDENT>output.save(fp, format=self.writer.format)<EOL><DEDENT>else:<EOL><INDENT>fp.write(output)<EOL><DEDENT>

Renders the barcode and writes it to the file like object
        `fp`.

        :parameters:
            fp : File like object
                Object to write the raw data in.
            options : Dict
                The same as in `self.render`.


options = Barcode.default_writer_options.copy()<EOL>options.update(writer_options or {})<EOL>if options['<STR_LIT>']:<EOL><INDENT>options['<STR_LIT:text>'] = self.get_fullcode()<EOL><DEDENT>self.writer.set_options(options)<EOL>code = self.build()<EOL>raw = Barcode.raw = self.writer.render(code)<EOL>return raw<EOL>

Renders the barcode using `self.writer`.

        :parameters:
            writer_options : Dict
                Options for `self.writer`, see writer docs for details.

        :returns: Output of the writers render method.


def calculate_size(self, modules_per_line, number_of_lines, dpi=<NUM_LIT>):

width = <NUM_LIT:2> * self.quiet_zone + modules_per_line * self.module_width<EOL>height = <NUM_LIT> + self.module_height * number_of_lines<EOL>if self.font_size and self.text:<EOL><INDENT>height += pt2mm(self.font_size) / <NUM_LIT:2> + self.text_distance<EOL><DEDENT>return int(mm2px(width, dpi)), int(mm2px(height, dpi))<EOL>

Calculates the size of the barcode in pixel.

        :parameters:
            modules_per_line : Integer
                Number of modules in one line.
            number_of_lines : Integer
                Number of lines of the barcode.
            dpi : Integer
                DPI to calculate.

        :returns: Width and height of the barcode in pixel.
        :rtype: Tuple


raise NotImplementedError<EOL>

Saves the rendered output to `filename`.

        :parameters:
            filename : String
                Filename without extension.
            output : String
                The rendered output.

        :returns: The full filename with extension.
        :rtype: String


def register_callback(self, action, callback):

self._callbacks[action] = callback<EOL>

Register one of the three callbacks if not given at instance
        creation.

        :parameters:
            action : String
                One of 'initialize', 'paint_module', 'paint_text', 'finish'.
            callback : Function
                The callback function for the given action.


for key, val in options.items():<EOL><INDENT>key = key.lstrip('<STR_LIT:_>')<EOL>if hasattr(self, key):<EOL><INDENT>setattr(self, key, val)<EOL><DEDENT><DEDENT>

Sets the given options as instance attributes (only
        if they are known).

        :parameters:
            options : Dict
                All known instance attributes and more if the childclass
                has defined them before this call.

        :rtype: None


if self._callbacks['<STR_LIT>'] is not None:<EOL><INDENT>self._callbacks['<STR_LIT>'](code)<EOL><DEDENT>ypos = <NUM_LIT:1.0><EOL>for line in code:<EOL><INDENT>xpos = self.quiet_zone<EOL>for mod in line:<EOL><INDENT>if mod == '<STR_LIT:0>':<EOL><INDENT>color = self.background<EOL><DEDENT>else:<EOL><INDENT>color = self.foreground<EOL><DEDENT>self._callbacks['<STR_LIT>'](xpos, ypos, self.module_width,<EOL>color)<EOL>xpos += self.module_width<EOL><DEDENT>self._callbacks['<STR_LIT>'](xpos, ypos, self.quiet_zone,<EOL>self.background)<EOL>ypos += self.module_height<EOL><DEDENT>if self.text and self._callbacks['<STR_LIT>'] is not None:<EOL><INDENT>ypos += self.text_distance<EOL>if self.center_text:<EOL><INDENT>xpos = xpos / <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>xpos = self.quiet_zone + <NUM_LIT><EOL><DEDENT>self._callbacks['<STR_LIT>'](xpos, ypos)<EOL><DEDENT>return self._callbacks['<STR_LIT>']()<EOL>

Renders the barcode to whatever the inheriting writer provides,
        using the registered callbacks.

        :parameters:
            code : List
                List of strings matching the writer spec
                (only contain 0 or 1).


def sum_(x, y):<EOL><INDENT>return int(x) + int(y)<EOL><DEDENT>evensum = reduce(sum_, self.ean[::<NUM_LIT:2>])<EOL>oddsum = reduce(sum_, self.ean[<NUM_LIT:1>::<NUM_LIT:2>])<EOL>return (<NUM_LIT:10> - ((evensum + oddsum * <NUM_LIT:3>) % <NUM_LIT:10>)) % <NUM_LIT:10><EOL>

Calculates the checksum for EAN13-Code.

        :returns: The checksum for `self.ean`.
        :rtype: Integer


code = _ean.EDGE[:]<EOL>pattern = _ean.LEFT_PATTERN[int(self.ean[<NUM_LIT:0>])]<EOL>for i, number in enumerate(self.ean[<NUM_LIT:1>:<NUM_LIT:7>]):<EOL><INDENT>code += _ean.CODES[pattern[i]][int(number)]<EOL><DEDENT>code += _ean.MIDDLE<EOL>for number in self.ean[<NUM_LIT:7>:]:<EOL><INDENT>code += _ean.CODES['<STR_LIT:C>'][int(number)]<EOL><DEDENT>code += _ean.EDGE<EOL>return [code]<EOL>

Builds the barcode pattern from `self.ean`.

        :returns: The pattern as string
        :rtype: String


code = self.build()<EOL>for i, line in enumerate(code):<EOL><INDENT>code[i] = line.replace('<STR_LIT:1>', '<STR_LIT:|>').replace('<STR_LIT:0>', '<STR_LIT:U+0020>')<EOL><DEDENT>return '<STR_LIT:\n>'.join(code)<EOL>

Returns an ascii representation of the barcode.

        :rtype: String


def sum_(x, y):<EOL><INDENT>return int(x) + int(y)<EOL><DEDENT>evensum = reduce(sum_, self.ean[::<NUM_LIT:2>])<EOL>oddsum = reduce(sum_, self.ean[<NUM_LIT:1>::<NUM_LIT:2>])<EOL>return (<NUM_LIT:10> - ((evensum * <NUM_LIT:3> + oddsum) % <NUM_LIT:10>)) % <NUM_LIT:10><EOL>

Calculates the checksum for EAN8-Code.

        :returns: The checksum for `self.ean`.
        :rtype: Integer


code = _ean.EDGE[:]<EOL>for number in self.ean[:<NUM_LIT:4>]:<EOL><INDENT>code += _ean.CODES['<STR_LIT:A>'][int(number)]<EOL><DEDENT>code += _ean.MIDDLE<EOL>for number in self.ean[<NUM_LIT:4>:]:<EOL><INDENT>code += _ean.CODES['<STR_LIT:C>'][int(number)]<EOL><DEDENT>code += _ean.EDGE<EOL>return [code]<EOL>


def sum_(x, y):<EOL><INDENT>return int(x) + int(y)<EOL><DEDENT>evensum = reduce(sum_, self.ean[::<NUM_LIT:2>])<EOL>oddsum = reduce(sum_, self.ean[<NUM_LIT:1>::<NUM_LIT:2>])<EOL>return (<NUM_LIT:10> - (((evensum * <NUM_LIT:3>) + oddsum) % <NUM_LIT:10>)) % <NUM_LIT:10><EOL>


if state in (SASLState.FAILURE, SASLState.SUCCESS,<EOL>SASLState.CHALLENGE):<EOL><INDENT>return state<EOL><DEDENT>if state in ("<STR_LIT>", "<STR_LIT:success>", "<STR_LIT>"):<EOL><INDENT>return SASLState(state)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>", state)<EOL><DEDENT>

Comptaibility layer for old :class:`SASLInterface`
implementations.

Accepts the follwing set of :class:`SASLState` or strings and
maps the strings to :class:`SASLState` elements as follows:

  ``"challenge"``
    :member:`SASLState.CHALLENGE`

   ``"failue"``
     :member:`SASLState.FAILURE`

   ``"success"``
     :member:`SASLState.SUCCESS`


@abc.abstractmethod<EOL><INDENT>@asyncio.coroutine<EOL>def initiate(self, mechanism, payload=None):<DEDENT>

Send a SASL initiation request for the given `mechanism`. Depending on
the `mechanism`, an initial `payload` *may* be given. The `payload` is
then a :class:`bytes` object which needs to be passed as initial
payload during the initiation request.

Wait for a reply by the peer and return the reply as a next-state tuple
in the format documented at :class:`SASLInterface`.


Send a response to a challenge. The `payload` is a :class:`bytes`
object which is to be sent as response.

Wait for a reply by the peer and return the reply as a next-state tuple
in the format documented at :class:`SASLInterface`.


Abort the authentication. The result is either the failure tuple
(``(SASLState.FAILURE, None)``) or a :class:`SASLFailure` exception if
the response from the peer did not indicate abortion (e.g. another
error was returned by the peer or the peer indicated success).


@asyncio.coroutine<EOL><INDENT>def initiate(self, mechanism, payload=None):<DEDENT>

if self._state != SASLState.INITIAL:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>next_state, payload = yield from self.interface.initiate(<EOL>mechanism,<EOL>payload=payload)<EOL><DEDENT>except SASLFailure:<EOL><INDENT>self._state = SASLState.FAILURE<EOL>raise<EOL><DEDENT>next_state = SASLState.from_reply(next_state)<EOL>self._state = next_state<EOL>return next_state, payload<EOL>

Initiate the SASL handshake and advertise the use of the given
`mechanism`. If `payload` is not :data:`None`, it will be base64
encoded and sent as initial client response along with the ``<auth />``
element.

Return the next state of the state machine as tuple (see
:class:`SASLStateMachine` for details).


if self._state == SASLState.SUCCESS_SIMULATE_CHALLENGE:<EOL><INDENT>if payload != b"<STR_LIT>":<EOL><INDENT>self._state = SASLState.FAILURE<EOL>raise SASLFailure(<EOL>None,<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>self._state = SASLState.SUCCESS<EOL>return SASLState.SUCCESS, None<EOL><DEDENT>if self._state != SASLState.CHALLENGE:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>next_state, payload = yield from self.interface.respond(payload)<EOL><DEDENT>except SASLFailure:<EOL><INDENT>self._state = SASLState.FAILURE<EOL>raise<EOL><DEDENT>next_state = SASLState.from_reply(next_state)<EOL>if next_state == SASLState.SUCCESS and payload is not None:<EOL><INDENT>self._state = SASLState.SUCCESS_SIMULATE_CHALLENGE<EOL>return SASLState.CHALLENGE, payload<EOL><DEDENT>self._state = next_state<EOL>return next_state, payload<EOL>

Send a response to the previously received challenge, with the given
`payload`. The payload is encoded using base64 and transmitted to the
server.

Return the next state of the state machine as tuple (see
:class:`SASLStateMachine` for details).


if self._state == SASLState.INITIAL:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if self._state == SASLState.SUCCESS_SIMULATE_CHALLENGE:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>return (yield from self.interface.abort())<EOL><DEDENT>finally:<EOL><INDENT>self._state = SASLState.FAILURE<EOL><DEDENT>

Abort an initiated SASL authentication process. The expected result
state is ``failure``.


Determine whether this class can perform any SASL mechanism in the set
of strings ``mechanisms``.

If the class cannot perform any of the SASL mechanisms in
``mechanisms``, it must return :data:`None`.

Otherwise, it must return a non-:data:`None` value. Applications must
not assign any meaning to any value (except that :data:`None` is a sure
indicator that the class cannot perform any of the listed mechanisms)
and must not alter any value returned by this function. Note that even
:data:`False` indicates success!

The return value must be passed as second argument to
:meth:`authenticate`. :meth:`authenticate` must not be called with a
:data:`None` value.


@asyncio.coroutine<EOL><INDENT>@abc.abstractmethod<EOL>def authenticate(self, sm, token):<DEDENT>

Execute the mechanism identified by `token` (the non-:data:`None` value
which has been returned by :meth:`any_supported` before) using the
given :class:`SASLStateMachine` `sm`.

If authentication fails, an appropriate exception is raised
(:class:`AuthenticationFailure`). If the authentication fails for a
reason unrelated to credentials, :class:`SASLFailure` is raised.


raise NotImplementedError<EOL>

Return the name of the channel-binding mechanism.
:rtype: :class:`bytes`


raise NotImplementedError<EOL>

Return the channel binding data.
:returns: the channel binding data
:rtype: :class:`bytes`


for c in chars:<EOL><INDENT>if any(in_table(c) for in_table in tables):<EOL><INDENT>return c<EOL><DEDENT><DEDENT>return None<EOL>

Perform a check against the table predicates in `tables`. `tables` must be
a reusable iterable containing characteristic functions of character sets,
that is, functions which return :data:`True` if the character is in the
table.

The function returns the first character occuring in any of the tables or
:data:`None` if no character matches.


chars[:] = list(unicodedata.normalize("<STR_LIT>", "<STR_LIT>".join(chars)))<EOL>

Perform the stringprep normalization. Operates in-place on a list of
unicode characters provided in `chars`.


<EOL>if not chars:<EOL><INDENT>return<EOL><DEDENT>has_RandALCat = any(is_RandALCat(c) for c in chars)<EOL>if not has_RandALCat:<EOL><INDENT>return<EOL><DEDENT>has_LCat = any(is_LCat(c) for c in chars)<EOL>if has_LCat:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if not is_RandALCat(chars[<NUM_LIT:0>]) or not is_RandALCat(chars[-<NUM_LIT:1>]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Check proper bidirectionality as per stringprep. Operates on a list of
unicode characters provided in `chars`.


violator = check_against_tables(chars, bad_tables)<EOL>if violator is not None:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(ord(violator)))<EOL><DEDENT>

Check against prohibited output, by checking whether any of the characters
from `chars` are in any of the `bad_tables`.

Operates in-place on a list of code points from `chars`.


bad_tables = (<EOL>stringprep.in_table_a1,)<EOL>violator = check_against_tables(chars, bad_tables)<EOL>if violator is not None:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(ord(violator)))<EOL><DEDENT>

Check that `chars` does not contain any unassigned code points as per
the given list of `bad_tables`.

Operates on a list of unicode code points provided in `chars`.


i = <NUM_LIT:0><EOL>while i < len(chars):<EOL><INDENT>c = chars[i]<EOL>if stringprep.in_table_c12(c):<EOL><INDENT>chars[i] = "<STR_LIT>"<EOL><DEDENT>elif stringprep.in_table_b1(c):<EOL><INDENT>del chars[i]<EOL>continue<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>

Perform the stringprep mapping step of SASLprep. Operates in-place on a
list of unicode characters provided in `chars`.


Process the given `string` using the SASLprep profile. In the error cases
defined in `RFC 3454`_ (stringprep), a :class:`ValueError` is raised.


Implement the ``trace`` profile specified in :rfc:`4505`.


assert len(a) == len(b)<EOL>return bytes(map(operator.xor, a, b))<EOL>

Calculate the byte wise exclusive of of two :class:`bytes` objects
of the same length.


def __init__(self, geometry_string, options={}, *args, **kwargs):

<EOL>self.geometry_string = geometry_string<EOL>self.options = options<EOL>super(HyperlinkedSorlImageField, self).__init__(*args, **kwargs)<EOL>

Create an instance of the HyperlinkedSorlImageField image serializer.

Args:
    geometry_string (str): The size of your cropped image.
    options (Optional[dict]): A dict of sorl options.
    *args: (Optional) Default serializers.ImageField arguments.
    **kwargs: (Optional) Default serializers.ImageField keyword
    arguments.

For a description of sorl geometry strings and additional sorl options,
please see https://sorl-thumbnail.readthedocs.org/en/latest/examples.html?highlight=geometry#low-level-api-examples


if not value:<EOL><INDENT>return None<EOL><DEDENT>image = get_thumbnail(value, self.geometry_string, **self.options)<EOL>try:<EOL><INDENT>request = self.context.get('<STR_LIT>', None)<EOL>return request.build_absolute_uri(image.url)<EOL><DEDENT>except:<EOL><INDENT>try:<EOL><INDENT>return super(HyperlinkedSorlImageField, self).to_representation(image)<EOL><DEDENT>except AttributeError:  <EOL><INDENT>return super(HyperlinkedSorlImageField, self).to_native(image.url)  <EOL><DEDENT><DEDENT>

Perform the actual serialization.

Args:
    value: the image to transform
Returns:
    a url pointing at a scaled and cached image


<EOL>if self.within_logging_statement():<EOL><INDENT>if self.within_logging_argument() and self.is_format_call(node):<EOL><INDENT>self.violations.append((node, STRING_FORMAT_VIOLATION))<EOL>super(LoggingVisitor, self).generic_visit(node)<EOL>return<EOL><DEDENT><DEDENT>logging_level = self.detect_logging_level(node)<EOL>if logging_level and self.current_logging_level is None:<EOL><INDENT>self.current_logging_level = logging_level<EOL><DEDENT>if logging_level is None:<EOL><INDENT>super(LoggingVisitor, self).generic_visit(node)<EOL>return<EOL><DEDENT>self.current_logging_call = node<EOL>if logging_level == "<STR_LIT>":<EOL><INDENT>self.violations.append((node, WARN_VIOLATION))<EOL><DEDENT>self.check_exc_info(node)<EOL>for index, child in enumerate(iter_child_nodes(node)):<EOL><INDENT>if index == <NUM_LIT:1>:<EOL><INDENT>self.current_logging_argument = child<EOL><DEDENT>if index >= <NUM_LIT:1>:<EOL><INDENT>self.check_exception_arg(child)<EOL><DEDENT>if index > <NUM_LIT:1> and isinstance(child, keyword) and child.arg == "<STR_LIT>":<EOL><INDENT>self.current_extra_keyword = child<EOL><DEDENT>super(LoggingVisitor, self).visit(child)<EOL>self.current_logging_argument = None<EOL>self.current_extra_keyword = None<EOL><DEDENT>self.current_logging_call = None<EOL>self.current_logging_level = None<EOL>

Visit a function call.

We expect every logging statement and string format to be a function call.


if self.within_logging_statement() and self.within_logging_argument():<EOL><INDENT>if isinstance(node.op, Mod):<EOL><INDENT>self.violations.append((node, PERCENT_FORMAT_VIOLATION))<EOL><DEDENT>if isinstance(node.op, Add):<EOL><INDENT>self.violations.append((node, STRING_CONCAT_VIOLATION))<EOL><DEDENT><DEDENT>super(LoggingVisitor, self).generic_visit(node)<EOL>

Process binary operations while processing the first logging argument.


if self.should_check_whitelist(node):<EOL><INDENT>for key in node.keys:<EOL><INDENT>if key.s in self.whitelist or key.s.startswith("<STR_LIT>"):<EOL><INDENT>continue<EOL><DEDENT>self.violations.append((self.current_logging_call, WHITELIST_VIOLATION.format(key.s)))<EOL><DEDENT><DEDENT>if self.should_check_extra_exception(node):<EOL><INDENT>for value in node.values:<EOL><INDENT>self.check_exception_arg(value)<EOL><DEDENT><DEDENT>super(LoggingVisitor, self).generic_visit(node)<EOL>


if version_info >= (<NUM_LIT:3>, <NUM_LIT:6>):<EOL><INDENT>if self.within_logging_statement():<EOL><INDENT>if any(isinstance(i, FormattedValue) for i in node.values):<EOL><INDENT>if self.within_logging_argument():<EOL><INDENT>self.violations.append((node, FSTRING_VIOLATION))<EOL>super(LoggingVisitor, self).generic_visit(node)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


if self.should_check_whitelist(node):<EOL><INDENT>if node.arg not in self.whitelist and not node.arg.startswith("<STR_LIT>"):<EOL><INDENT>self.violations.append((self.current_logging_call, WHITELIST_VIOLATION.format(node.arg)))<EOL><DEDENT><DEDENT>if self.should_check_extra_exception(node):<EOL><INDENT>self.check_exception_arg(node.value)<EOL><DEDENT>super(LoggingVisitor, self).generic_visit(node)<EOL>


name = self.get_except_handler_name(node)<EOL>if not name:<EOL><INDENT>super(LoggingVisitor, self).generic_visit(node)<EOL>return<EOL><DEDENT>self.current_except_names.append(name)<EOL>super(LoggingVisitor, self).generic_visit(node)<EOL>self.current_except_names.pop()<EOL>


try:<EOL><INDENT>if self.get_id_attr(node.func.value) == "<STR_LIT>":<EOL><INDENT>return None<EOL><DEDENT>if node.func.attr in LOGGING_LEVELS:<EOL><INDENT>return node.func.attr<EOL><DEDENT><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>return None<EOL>

Heuristic to decide whether an AST Call is a logging call.


try:<EOL><INDENT>return node.func.attr == "<STR_LIT>"<EOL><DEDENT>except AttributeError:<EOL><INDENT>return False<EOL><DEDENT>


name = node.name<EOL>if not name:<EOL><INDENT>return None<EOL><DEDENT>if version_info < (<NUM_LIT:3>,):<EOL><INDENT>return name.id<EOL><DEDENT>return name<EOL>

Helper to get the exception name from an ExceptHandler node in both py2 and py3.


if not hasattr(value, "<STR_LIT:id>") and hasattr(value, "<STR_LIT:value>"):<EOL><INDENT>value = value.value<EOL><DEDENT>return value.id<EOL>

Check if value has id attribute and return it.

        :param value: The value to get id from.
        :return: The value.id.


return isinstance(node, Name) and node.id in self.current_except_names<EOL>

Checks if the node is a bare exception name from an except block.


return (<EOL>isinstance(node, Call)<EOL>and isinstance(node.func, Name)<EOL>and node.func.id in ('<STR_LIT:str>', '<STR_LIT>')<EOL>and node.args<EOL>and self.is_bare_exception(node.args[<NUM_LIT:0>])<EOL>)<EOL>

Checks if the node is the expression str(e) or unicode(e), where e is an exception name from an except block


if self.current_logging_level not in ('<STR_LIT:error>', '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>for kw in node.keywords:<EOL><INDENT>if kw.arg == '<STR_LIT>':<EOL><INDENT>if self.current_logging_level == '<STR_LIT:error>':<EOL><INDENT>violation = ERROR_EXC_INFO_VIOLATION<EOL><DEDENT>else:<EOL><INDENT>violation = REDUNDANT_EXC_INFO_VIOLATION<EOL><DEDENT>self.violations.append((node, violation))<EOL><DEDENT><DEDENT>

Reports a violation if exc_info keyword is used with logging.error or logging.exception.


return ["<STR_LIT>"]<EOL>

Example whitelist entry point used for testing.


def sort_enum_for_model(cls, name=None, symbol_name=_symbol_name):

enum, _ = _sort_enum_for_model(cls, name, symbol_name)<EOL>return enum<EOL>

Create Graphene Enum for sorting a SQLAlchemy class query

    Parameters
    - cls : Sqlalchemy model class
        Model used to create the sort enumerator
    - name : str, optional, default None
        Name to use for the enumerator. If not provided it will be set to `cls.__name__ + 'SortEnum'`
    - symbol_name : function, optional, default `_symbol_name`
        Function which takes the column name and a boolean indicating if the sort direction is ascending,
        and returns the symbol name for the current column and sort direction.
        The default function will create, for a column named 'foo', the symbols 'foo_asc' and 'foo_desc'

    Returns
    - Enum
        The Graphene enumerator


enum, default = _sort_enum_for_model(cls)<EOL>if not has_default:<EOL><INDENT>default = None<EOL><DEDENT>return Argument(List(enum), default_value=default)<EOL>

Returns a Graphene argument for the sort field that accepts a list of sorting directions for a model.
    If `has_default` is True (the default) it will sort the result by the primary key(s)


def _chunk(self, size="<STR_LIT>", axis=None, padding=None):

if self.split == len(self.shape) and padding is None:<EOL><INDENT>self._rdd = self._rdd.map(lambda kv: (kv[<NUM_LIT:0>]+(<NUM_LIT:0>,), array(kv[<NUM_LIT:1>], ndmin=<NUM_LIT:1>)))<EOL>self._shape = self._shape + (<NUM_LIT:1>,)<EOL>self._plan = (<NUM_LIT:1>,)<EOL>self._padding = array([<NUM_LIT:0>])<EOL>return self<EOL><DEDENT>rdd = self._rdd<EOL>self._plan, self._padding = self.getplan(size, axis, padding)<EOL>if any([x + y > z for x, y, z in zip(self.plan, self.padding, self.vshape)]):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>% (tuple(self.plan), tuple(self.padding), tuple(self.vshape)))<EOL><DEDENT>if any([x > y for x, y in zip(self.padding, self.plan)]):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>% (tuple(self.padding), tuple(self.plan)))<EOL><DEDENT>slices = self.getslices(self.plan, self.padding, self.vshape)<EOL>labels = list(product(*[list(enumerate(s)) for s in slices]))<EOL>scheme = [list(zip(*s)) for s in labels]<EOL>def _chunk(record):<EOL><INDENT>k, v = record[<NUM_LIT:0>], record[<NUM_LIT:1>]<EOL>for (chk, slc) in scheme:<EOL><INDENT>if type(k) is int:<EOL><INDENT>k = (k,)<EOL><DEDENT>yield k + chk, v[slc]<EOL><DEDENT><DEDENT>rdd = rdd.flatMap(_chunk)<EOL>return self._constructor(rdd, shape=self.shape, split=self.split,<EOL>dtype=self.dtype, plan=self.plan, padding=self.padding, ordered=self._ordered)<EOL>

Split values of distributed array into chunks.

Transforms an underlying pair RDD of (key, value) into
records of the form: (key, chunk id), (chunked value).
Here, chunk id is a tuple identifying the chunk and
chunked value is a subset of the data from each original value,
that has been divided along the specified dimensions.

Parameters
----------
size : str or tuple or int
    If str, the average size (in KB) of the chunks in all value dimensions.
    If int or tuple, an explicit specification of the number chunks in
    each value dimension.

axis : tuple, optional, default=None
    One or more axes to estimate chunks for, if provided any
    other axes will use one chunk.

padding: tuple or int, default = None
    Number of elements per dimension that will overlap with the adjacent chunk.
    If a tuple, specifies padding along each chunked dimension; if a int, same
    padding will be applied to all chunked dimensions.


plan, padding, vshape, split = self.plan, self.padding, self.vshape, self.split<EOL>nchunks = self.getnumber(plan, vshape)<EOL>full_shape = concatenate((nchunks, plan))<EOL>n = len(vshape)<EOL>perm = concatenate(list(zip(range(n), range(n, <NUM_LIT:2>*n))))<EOL>if self.uniform:<EOL><INDENT>def _unchunk(it):<EOL><INDENT>ordered = sorted(it, key=lambda kv: kv[<NUM_LIT:0>][split:])<EOL>keys, values = zip(*ordered)<EOL>yield keys[<NUM_LIT:0>][:split], asarray(values).reshape(full_shape).transpose(perm).reshape(vshape)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>def _unchunk(it):<EOL><INDENT>ordered = sorted(it, key=lambda kv: kv[<NUM_LIT:0>][split:])<EOL>keys, values = zip(*ordered)<EOL>k_chks = [k[split:] for k in keys]<EOL>arr = empty(nchunks, dtype='<STR_LIT:object>')<EOL>for (i, d) in zip(k_chks, values):<EOL><INDENT>arr[i] = d<EOL><DEDENT>yield keys[<NUM_LIT:0>][:split], allstack(arr.tolist())<EOL><DEDENT><DEDENT>if self.padded:<EOL><INDENT>removepad = self.removepad<EOL>rdd = self._rdd.map(lambda kv: (kv[<NUM_LIT:0>], removepad(kv[<NUM_LIT:0>][split:], kv[<NUM_LIT:1>], nchunks, padding, axes=range(n))))<EOL><DEDENT>else:<EOL><INDENT>rdd = self._rdd<EOL><DEDENT>if array_equal(self.plan, self.vshape):<EOL><INDENT>rdd = rdd.map(lambda kv: (kv[<NUM_LIT:0>][:split], kv[<NUM_LIT:1>]))<EOL>ordered = self._ordered<EOL><DEDENT>else:<EOL><INDENT>ranges = self.kshape<EOL>npartitions = int(prod(ranges))<EOL>if len(self.kshape) == <NUM_LIT:0>:<EOL><INDENT>partitioner = lambda k: <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>partitioner = lambda k: ravel_multi_index(k[:split], ranges)<EOL><DEDENT>rdd = rdd.partitionBy(numPartitions=npartitions, partitionFunc=partitioner).mapPartitions(_unchunk)<EOL>ordered = True<EOL><DEDENT>if array_equal(self.vshape, [<NUM_LIT:1>]):<EOL><INDENT>rdd = rdd.mapValues(lambda v: squeeze(v))<EOL>newshape = self.shape[:-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>newshape = self.shape<EOL><DEDENT>return BoltArraySpark(rdd, shape=newshape, split=self._split,<EOL>dtype=self.dtype, ordered=ordered)<EOL>

Convert a chunked array back into a full array with (key,value) pairs
where key is a tuple of indices, and value is an ndarray.


def keys_to_values(self, axes, size=None):

if len(axes) == <NUM_LIT:0>:<EOL><INDENT>return self<EOL><DEDENT>kmask = self.kmask(axes)<EOL>if size is None:<EOL><INDENT>size = self.kshape[kmask]<EOL><DEDENT>newplan = r_[size, self.plan]<EOL>newsplit = self._split - len(axes)<EOL>newshape = tuple(r_[self.kshape[~kmask], self.kshape[kmask], self.vshape].astype(int).tolist())<EOL>newpadding = r_[zeros(len(axes), dtype=int), self.padding]<EOL>result = self._constructor(None, shape=newshape, split=newsplit,<EOL>dtype=self.dtype, plan=newplan, padding=newpadding, ordered=True)<EOL>split = self.split<EOL>def _relabel(record):<EOL><INDENT>k, data = record<EOL>keys, chks = asarray(k[:split], '<STR_LIT:int>'), k[split:]<EOL>movingkeys, stationarykeys = keys[kmask], keys[~kmask]<EOL>newchks = [int(m) for m in movingkeys/size]  <EOL>labels = mod(movingkeys, size)<EOL>return tuple(stationarykeys) + tuple(newchks) + tuple(chks) + tuple(labels), data<EOL><DEDENT>rdd = self._rdd.map(_relabel)<EOL>nchunks = result.getnumber(result.plan, result.vshape)<EOL>npartitions = int(prod(result.kshape) * prod(nchunks))<EOL>ranges = tuple(result.kshape) + tuple(nchunks)<EOL>n = len(axes)<EOL>if n == <NUM_LIT:0>:<EOL><INDENT>s = slice(None)<EOL><DEDENT>else:<EOL><INDENT>s = slice(-n)<EOL><DEDENT>partitioner = lambda k: ravel_multi_index(k[s], ranges)<EOL>rdd = rdd.partitionBy(numPartitions=npartitions, partitionFunc=partitioner)<EOL>uniform = result.uniform<EOL>def _rebuild(it):<EOL><INDENT>ordered = sorted(it, key=lambda kv: kv[<NUM_LIT:0>][n:])<EOL>keys, data = zip(*ordered)<EOL>k = keys[<NUM_LIT:0>][s]<EOL>labels = asarray([x[-n:] for x in keys])<EOL>if uniform:<EOL><INDENT>labelshape = tuple(size)<EOL><DEDENT>else:<EOL><INDENT>labelshape = tuple(amax(labels, axis=<NUM_LIT:0>) - amin(labels, axis=<NUM_LIT:0>) + <NUM_LIT:1>)<EOL><DEDENT>valshape = data[<NUM_LIT:0>].shape<EOL>fullshape = labelshape + valshape<EOL>yield k, asarray(data).reshape(fullshape)<EOL><DEDENT>result._rdd = rdd.mapPartitions(_rebuild)<EOL>if array_equal(self.vshape, [<NUM_LIT:1>]):<EOL><INDENT>result._rdd = result._rdd.mapValues(lambda v: squeeze(v))<EOL>result._shape = result.shape[:-<NUM_LIT:1>]<EOL>result._plan = result.plan[:-<NUM_LIT:1>]<EOL><DEDENT>return result<EOL>

Move indices in the keys into the values.

Padding on these new value-dimensions is not currently supported and is set to 0.

Parameters
----------
axes : tuple
    Axes from keys to move to values.

size : tuple, optional, default=None
    Size of chunks for the values along the new dimensions.
    If None, then no chunking for all axes (number of chunks = 1)

Returns
-------
ChunkedArray


def map(self, func, value_shape=None, dtype=None):

if value_shape is None or dtype is None:<EOL><INDENT>try:<EOL><INDENT>mapped = func(random.randn(*self.plan).astype(self.dtype))<EOL><DEDENT>except Exception:<EOL><INDENT>first = self._rdd.first()<EOL>if first:<EOL><INDENT>mapped = func(first[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>if value_shape is None:<EOL><INDENT>value_shape = mapped.shape<EOL><DEDENT>if dtype is None:<EOL><INDENT>dtype = mapped.dtype<EOL><DEDENT><DEDENT>chunked_dims = where(self.plan != self.vshape)[<NUM_LIT:0>]<EOL>unchunked_dims = where(self.plan == self.vshape)[<NUM_LIT:0>]<EOL>if len(value_shape) != len(self.plan):<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>if any([value_shape[i] != self.plan[i] for i in chunked_dims]):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>def check_and_apply(v):<EOL><INDENT>new = func(v)<EOL>if len(unchunked_dims) > <NUM_LIT:0>:<EOL><INDENT>if any([new.shape[i] != value_shape[i] for i in unchunked_dims]):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>if len(chunked_dims) > <NUM_LIT:0>:<EOL><INDENT>if any([v.shape[i] != new.shape[i] for i in chunked_dims]):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>return new<EOL><DEDENT>rdd = self._rdd.mapValues(check_and_apply)<EOL>vshape = [value_shape[i] if i in unchunked_dims else self.vshape[i] for i in range(len(self.vshape))]<EOL>newshape = r_[self.kshape, vshape].astype(int).tolist()<EOL>return self._constructor(rdd, shape=tuple(newshape), dtype=dtype,<EOL>plan=asarray(value_shape)).__finalize__(self)<EOL>

Apply an array -> array function on each subarray.

The function can change the shape of the subarray, but only along
dimensions that are not chunked.

Parameters
----------
func : function
    Function of a single subarray to apply

value_shape:
    Known shape of chunking plan after the map

dtype: numpy.dtype, optional, default=None
    Known dtype of values resulting from operation

Returns
-------
ChunkedArray


def process_record(val):<EOL><INDENT>newval = empty(<NUM_LIT:1>, dtype="<STR_LIT:object>")<EOL>newval[<NUM_LIT:0>]  = func(val)<EOL>return newval<EOL><DEDENT>rdd = self._rdd.mapValues(process_record)<EOL>nchunks = self.getnumber(self.plan, self.vshape)<EOL>newshape = tuple([int(s) for s in r_[self.kshape, nchunks]])<EOL>newsplit = len(self.shape)<EOL>return BoltArraySpark(rdd, shape=newshape, split=newsplit, ordered=self._ordered, dtype="<STR_LIT:object>")<EOL>

Apply a generic array -> object to each subarray

The resulting object is a BoltArraySpark of dtype object where the
blocked dimensions are replaced with indices indication block ID.


def getplan(self, size="<STR_LIT>", axes=None, padding=None):

from numpy import dtype as gettype<EOL>plan = self.vshape<EOL>if axes is None:<EOL><INDENT>if isinstance(size, str):<EOL><INDENT>axes = arange(len(self.vshape))<EOL><DEDENT>else:<EOL><INDENT>axes = arange(len(size))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>axes = asarray(axes, '<STR_LIT:int>')<EOL><DEDENT>pad = array(len(self.vshape)*[<NUM_LIT:0>, ])<EOL>if padding is not None:<EOL><INDENT>pad[axes] = padding<EOL><DEDENT>if isinstance(size, tuple):<EOL><INDENT>plan[axes] = size<EOL><DEDENT>elif isinstance(size, str):<EOL><INDENT>size = <NUM_LIT> * float(size)<EOL>elsize = gettype(self.dtype).itemsize<EOL>nelements = prod(self.vshape)<EOL>dims = self.vshape[self.vmask(axes)]<EOL>if size <= elsize:<EOL><INDENT>s = ones(len(axes))<EOL><DEDENT>else:<EOL><INDENT>remsize = <NUM_LIT:1.0> * nelements * elsize<EOL>s = []<EOL>for (i, d) in enumerate(dims):<EOL><INDENT>minsize = remsize/d<EOL>if minsize >= size:<EOL><INDENT>s.append(<NUM_LIT:1>)<EOL>remsize = minsize<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>s.append(min(d, floor(size/minsize)))<EOL>s[i+<NUM_LIT:1>:] = plan[i+<NUM_LIT:1>:]<EOL>break<EOL><DEDENT><DEDENT><DEDENT>plan[axes] = s<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return plan, pad<EOL>

Identify a plan for chunking values along each dimension.

Generates an ndarray with the size (in number of elements) of chunks
in each dimension. If provided, will estimate chunks for only a
subset of axes, leaving all others to the full size of the axis.

Parameters
----------
size : string or tuple
     If str, the average size (in KB) of the chunks in all value dimensions.
     If int/tuple, an explicit specification of the number chunks in
     each moving value dimension.

axes : tuple, optional, default=None
      One or more axes to estimate chunks for, if provided any
      other axes will use one chunk.

padding : tuple or int, option, default=None
    Size over overlapping padding between chunks in each dimension.
    If tuple, specifies padding along each chunked dimension; if int,
    all dimensions use same padding; if None, no padding


@staticmethod<EOL><INDENT>def removepad(idx, value, number, padding, axes=None):<DEDENT>

if axes is None:<EOL><INDENT>axes = range(len(number))<EOL><DEDENT>mask = len(number)*[False, ]<EOL>for i in range(len(mask)):<EOL><INDENT>if i in axes and padding[i] != <NUM_LIT:0>:<EOL><INDENT>mask[i] = True<EOL><DEDENT><DEDENT>starts = [<NUM_LIT:0> if (i == <NUM_LIT:0> or not m) else p for (i, m, p) in zip(idx, mask, padding)]<EOL>stops = [None if (i == n-<NUM_LIT:1> or not m) else -p for (i, m, p, n) in zip(idx, mask, padding, number)]<EOL>slices = [slice(i1, i2) for (i1, i2) in zip(starts, stops)]<EOL>return value[slices]<EOL>

Remove the padding from chunks.

Given a chunk and its corresponding index, use the plan and padding to remove any
padding from the chunk along with specified axes.

Parameters
----------
idx: tuple or array-like
    The chunk index, indicating which chunk this is.

value: ndarray
    The chunk that goes along with the index.

number: ndarray or array-like
    The number of chunks along each dimension.

padding: ndarray or array-like
    The padding scheme.

axes: tuple, optional, default = None
    The axes (in the values) along which to remove padding.


nchunks = []<EOL>for size, d in zip(plan, shape):<EOL><INDENT>nchunks.append(int(ceil(<NUM_LIT:1.0> * d/size)))<EOL><DEDENT>return nchunks<EOL>

Obtain number of chunks for the given dimensions and chunk sizes.

Given a plan for the number of chunks along each dimension,
calculate the number of chunks that this will lead to.

Parameters
----------
plan: tuple or array-like
    Size of chunks (in number of elements) along each dimensions.
    Length must be equal to the number of dimensions.

shape : tuple
     Shape of array to be chunked.


@staticmethod<EOL><INDENT>def getslices(plan, padding, shape):<DEDENT>

slices = []<EOL>for size, pad, d in zip(plan, padding, shape):<EOL><INDENT>nchunks = int(floor(d/size))<EOL>remainder = d % size<EOL>start = <NUM_LIT:0><EOL>dimslices = []<EOL>for idx in range(nchunks):<EOL><INDENT>end = start + size<EOL>if idx == <NUM_LIT:0>:<EOL><INDENT>left = start<EOL><DEDENT>else:<EOL><INDENT>left = start - pad<EOL><DEDENT>if idx == nchunks:<EOL><INDENT>right = end<EOL><DEDENT>else:<EOL><INDENT>right = end + pad<EOL><DEDENT>dimslices.append(slice(left, right, <NUM_LIT:1>))<EOL>start = end<EOL><DEDENT>if remainder:<EOL><INDENT>dimslices.append(slice(end - pad, d, <NUM_LIT:1>))<EOL><DEDENT>slices.append(dimslices)<EOL><DEDENT>return slices<EOL>

Obtain slices for the given dimensions, padding, and chunks.

Given a plan for the number of chunks along each dimension and the amount of padding,
calculate a list of slices required to generate those chunks.

Parameters
----------
plan: tuple or array-like
    Size of chunks (in number of elements) along each dimensions.
    Length must be equal to the number of dimensions.

padding: tuple or array-like
    Size of overlap (in number of elements) between chunks along each dimension.
    Length must be equal to the number of dimensions.

shape: tuple
     Dimensions of axes to be chunked.


inds = asarray(inds, '<STR_LIT:int>')<EOL>mask = zeros(n, dtype=bool)<EOL>mask[inds] = True<EOL>return mask<EOL>

Obtain a binary mask by setting a subset of entries to true.

Parameters
----------
inds : array-like
    Which indices to set as true.

n : int
    The length of the target mask.


return self._rdd<EOL>

Return the RDD wrapped by the ChunkedArray.

Returns
-------
RDD


@staticmethod<EOL><INDENT>def array(a, context=None, axis=(<NUM_LIT:0>,), dtype=None, npartitions=None):<DEDENT>

if dtype is None:<EOL><INDENT>arry = asarray(a)<EOL>dtype = arry.dtype<EOL><DEDENT>else:<EOL><INDENT>arry = asarray(a, dtype)<EOL><DEDENT>shape = arry.shape<EOL>ndim = len(shape)<EOL>axes = ConstructSpark._format_axes(axis, arry.shape)<EOL>key_axes, value_axes = get_kv_axes(arry.shape, axes)<EOL>permutation = key_axes + value_axes<EOL>arry = arry.transpose(*permutation)<EOL>split = len(axes)<EOL>if split < <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>" % split)<EOL><DEDENT>if split > len(shape):<EOL><INDENT>raise ValueError("<STR_LIT>" % (ndim, split))<EOL><DEDENT>key_shape = shape[:split]<EOL>val_shape = shape[split:]<EOL>keys = zip(*unravel_index(arange(<NUM_LIT:0>, int(prod(key_shape))), key_shape))<EOL>vals = arry.reshape((prod(key_shape),) + val_shape)<EOL>rdd = context.parallelize(zip(keys, vals), npartitions)<EOL>return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)<EOL>

Create a spark bolt array from a local array.

Parameters
----------
a : array-like
    An array, any object exposing the array interface, an
    object whose __array__ method returns an array, or any
    (nested) sequence.

context : SparkContext
    A context running Spark. (see pyspark)

axis : tuple, optional, default=(0,)
    Which axes to distribute the array along. The resulting
    distributed object will use keys to represent these axes,
    with the remaining axes represented by values.

dtype : data-type, optional, default=None
    The desired data-type for the array. If None, will
    be determined from the data. (see numpy)

npartitions : int
    Number of partitions for parallization.

Returns
-------
BoltArraySpark


@staticmethod<EOL><INDENT>def ones(shape, context=None, axis=(<NUM_LIT:0>,), dtype=float64, npartitions=None):<DEDENT>

from numpy import ones<EOL>return ConstructSpark._wrap(ones, shape, context, axis, dtype, npartitions)<EOL>

Create a spark bolt array of ones.

Parameters
----------
shape : tuple
    The desired shape of the array.

context : SparkContext
    A context running Spark. (see pyspark)

axis : tuple, optional, default=(0,)
    Which axes to distribute the array along. The resulting
    distributed object will use keys to represent these axes,
    with the remaining axes represented by values.

dtype : data-type, optional, default=float64
    The desired data-type for the array. If None, will
    be determined from the data. (see numpy)

npartitions : int
    Number of partitions for parallization.

Returns
-------
BoltArraySpark


@staticmethod<EOL><INDENT>def zeros(shape, context=None, axis=(<NUM_LIT:0>,), dtype=float64, npartitions=None):<DEDENT>

from numpy import zeros<EOL>return ConstructSpark._wrap(zeros, shape, context, axis, dtype, npartitions)<EOL>

Create a spark bolt array of zeros.

Parameters
----------
shape : tuple
    The desired shape of the array.

context : SparkContext
    A context running Spark. (see pyspark)

axis : tuple, optional, default=(0,)
    Which axes to distribute the array along. The resulting
    distributed object will use keys to represent these axes,
    with the remaining axes represented by values.

dtype : data-type, optional, default=float64
    The desired data-type for the array. If None, will
    be determined from the data. (see numpy)

npartitions : int
    Number of partitions for parallization.

Returns
-------
BoltArraySpark


if not isinstance(arrays, tuple):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not len(arrays) == <NUM_LIT:2>:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>first, second = arrays<EOL>if isinstance(first, BoltArraySpark):<EOL><INDENT>return first.concatenate(second, axis)<EOL><DEDENT>elif isinstance(second, BoltArraySpark):<EOL><INDENT>first = ConstructSpark.array(first, second._rdd.context)<EOL>return first.concatenate(second, axis)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Join two bolt arrays together, at least one of which is in spark.

Parameters
----------
arrays : tuple
    A pair of arrays. At least one must be a spark array,
    the other can be a local bolt array, a local numpy array,
    or an array-like.

axis : int, optional, default=0
    The axis along which the arrays will be joined.

Returns
-------
BoltArraySpark


try:<EOL><INDENT>from pyspark import SparkContext<EOL><DEDENT>except ImportError:<EOL><INDENT>return False<EOL><DEDENT>cond1 = any([isinstance(arg, SparkContext) for arg in args])<EOL>cond2 = isinstance(kwargs.get('<STR_LIT>', None), SparkContext)<EOL>cond3 = any([isinstance(arg, BoltArraySpark) for arg in args])<EOL>cond4 = any([any([isinstance(sub, BoltArraySpark) for sub in arg])<EOL>if isinstance(arg, (tuple, list)) else False for arg in args])<EOL>return cond1 or cond2 or cond3 or cond4<EOL>

Check that arguments are consistent with spark array construction.

Conditions are:
(1) a positional argument is a SparkContext
(2) keyword arg 'context' is a SparkContext
(3) an argument is a BoltArraySpark, or
(4) an argument is a nested list containing a BoltArraySpark


if isinstance(axes, int):<EOL><INDENT>axes = (axes,)<EOL><DEDENT>elif isinstance(axes, list) or hasattr(axes, '<STR_LIT>'):<EOL><INDENT>axes = tuple(axes)<EOL><DEDENT>if not isinstance(axes, tuple):<EOL><INDENT>raise ValueError("<STR_LIT>" % str(axes))<EOL><DEDENT>if min(axes) < <NUM_LIT:0> or max(axes) > len(shape) - <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>" % (str(axes), str(shape)))<EOL><DEDENT>return axes<EOL>


@staticmethod<EOL><INDENT>def _wrap(func, shape, context=None, axis=(<NUM_LIT:0>,), dtype=None, npartitions=None):<DEDENT>

if isinstance(shape, int):<EOL><INDENT>shape = (shape,)<EOL><DEDENT>key_shape, value_shape = get_kv_shape(shape, ConstructSpark._format_axes(axis, shape))<EOL>split = len(key_shape)<EOL>rdd = context.parallelize(list(product(*[arange(x) for x in key_shape])), npartitions)<EOL>rdd = rdd.map(lambda x: (x, func(value_shape, dtype, order='<STR_LIT:C>')))<EOL>return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)<EOL>

Wrap an existing numpy constructor in a parallelized construction


new = argpack(shape)<EOL>old = self.shape<EOL>isreshapeable(new, old)<EOL>if new == old:<EOL><INDENT>return self._barray<EOL><DEDENT>def f(k):<EOL><INDENT>return unravel_index(ravel_multi_index(k, old), new)<EOL><DEDENT>newrdd = self._barray._rdd.map(lambda kv: (f(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>]))<EOL>newsplit = len(new)<EOL>newshape = new + self._barray.values.shape<EOL>return BoltArraySpark(newrdd, shape=newshape, split=newsplit).__finalize__(self._barray)<EOL>

Reshape just the keys of a BoltArraySpark, returning a
new BoltArraySpark.

Parameters
----------
shape : tuple
      New proposed axes.


new = argpack(axes)<EOL>old = range(self.ndim)<EOL>istransposeable(new, old)<EOL>if new == old:<EOL><INDENT>return self._barray<EOL><DEDENT>def f(k):<EOL><INDENT>return tuple(k[i] for i in new)<EOL><DEDENT>newrdd = self._barray._rdd.map(lambda kv: (f(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>]))<EOL>newshape = tuple(self.shape[i] for i in new) + self._barray.values.shape<EOL>return BoltArraySpark(newrdd, shape=newshape, ordered=False).__finalize__(self._barray)<EOL>

Transpose just the keys of a BoltArraySpark, returning a
new BoltArraySpark.

Parameters
----------
axes : tuple
     New proposed axes.


new = argpack(shape)<EOL>old = self.shape<EOL>isreshapeable(new, old)<EOL>if new == old:<EOL><INDENT>return self._barray<EOL><DEDENT>def f(v):<EOL><INDENT>return v.reshape(new)<EOL><DEDENT>newrdd = self._barray._rdd.mapValues(f)<EOL>newshape = self._barray.keys.shape + new<EOL>return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)<EOL>

Reshape just the values of a BoltArraySpark, returning a
new BoltArraySpark.

Parameters
----------
shape : tuple
      New proposed axes.


new = argpack(axes)<EOL>old = range(self.ndim)<EOL>istransposeable(new, old)<EOL>if new == old:<EOL><INDENT>return self._barray<EOL><DEDENT>def f(v):<EOL><INDENT>return v.transpose(new)<EOL><DEDENT>newrdd = self._barray._rdd.mapValues(f)<EOL>newshape = self._barray.keys.shape + tuple(self.shape[i] for i in new)<EOL>return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)<EOL>

Transpose just the values of a BoltArraySpark, returning a
new BoltArraySpark.

Parameters
----------
axes : tuple
     New proposed axes.


starts = [<NUM_LIT:0>]<EOL>if rdd.getNumPartitions() > <NUM_LIT:1>:<EOL><INDENT>nums = rdd.mapPartitions(lambda it: [sum(<NUM_LIT:1> for _ in it)]).collect()<EOL>count = sum(nums)<EOL>for i in range(len(nums) - <NUM_LIT:1>):<EOL><INDENT>starts.append(starts[-<NUM_LIT:1>] + nums[i])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>count = rdd.count()<EOL><DEDENT>def func(k, it):<EOL><INDENT>for i, v in enumerate(it, starts[k]):<EOL><INDENT>yield v, i<EOL><DEDENT><DEDENT>return count, rdd.mapPartitionsWithIndex(func)<EOL>

Alternate version of Spark's zipWithIndex that eagerly returns count.


def tostacks(partition):<EOL><INDENT>keys = []<EOL>arrs = []<EOL>for key, arr in partition:<EOL><INDENT>keys.append(key)<EOL>arrs.append(arr)<EOL>if size and <NUM_LIT:0> <= size <= len(keys):<EOL><INDENT>yield (keys, asarray(arrs))<EOL>keys, arrs = [], []<EOL><DEDENT><DEDENT>if keys:<EOL><INDENT>yield (keys, asarray(arrs))<EOL><DEDENT><DEDENT>rdd = self._rdd.mapPartitions(tostacks)<EOL>return self._constructor(rdd).__finalize__(self)<EOL>

Make an intermediate RDD where all records are combined into a
list of keys and larger ndarray along a new 0th dimension.


from bolt.spark.array import BoltArraySpark<EOL>if self._rekeyed:<EOL><INDENT>rdd = self._rdd<EOL><DEDENT>else:<EOL><INDENT>rdd = self._rdd.flatMap(lambda kv: zip(kv[<NUM_LIT:0>], list(kv[<NUM_LIT:1>])))<EOL><DEDENT>return BoltArraySpark(rdd, shape=self.shape, split=self.split)<EOL>

Unstack array and return a new BoltArraySpark via flatMap().


vshape = self.shape[self.split:]<EOL>x = self._rdd.values().first()<EOL>if x.shape == vshape:<EOL><INDENT>a, b = asarray([x]), asarray([x, x])<EOL><DEDENT>else:<EOL><INDENT>a, b = x, concatenate((x, x))<EOL><DEDENT>try:<EOL><INDENT>atest = func(a)<EOL>btest = func(b)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % e)<EOL><DEDENT>if not (isinstance(atest, ndarray) and isinstance(btest, ndarray)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif atest.shape == btest.shape:<EOL><INDENT>if self._rekeyed is True:<EOL><INDENT>rdd = self._rdd.map(lambda kv: (kv[<NUM_LIT:0>], func(kv[<NUM_LIT:1>])))<EOL>shape = (self.shape[<NUM_LIT:0>],) + atest.shape<EOL><DEDENT>else:<EOL><INDENT>count, rdd = zip_with_index(self._rdd.values())<EOL>rdd = rdd.map(lambda kv: ((kv[<NUM_LIT:1>],), func(kv[<NUM_LIT:0>])))<EOL>shape = (count,) + atest.shape<EOL><DEDENT>split = <NUM_LIT:1><EOL>rekeyed = True<EOL><DEDENT>elif atest.shape[<NUM_LIT:0>] == a.shape[<NUM_LIT:0>] and btest.shape[<NUM_LIT:0>] == b.shape[<NUM_LIT:0>]:<EOL><INDENT>shape = self.shape[<NUM_LIT:0>:self.split] + atest.shape[<NUM_LIT:1>:]<EOL>split = self.split<EOL>rdd = self._rdd.map(lambda kv: (kv[<NUM_LIT:0>], func(kv[<NUM_LIT:1>])))<EOL>rekeyed = self._rekeyed<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return self._constructor(rdd, rekeyed=rekeyed, shape=shape, split=split).__finalize__(self)<EOL>

Apply a function on each subarray.

Parameters
----------
func : function
     This is applied to each value in the intermediate RDD.

Returns
-------
StackedArray


return self._rdd<EOL>

Return the RDD wrapped by the StackedArray.

Returns
-------
RDD


rdd = self._rdd.repartition(npartitions)<EOL>return self._constructor(rdd, ordered=False).__finalize__(self)<EOL>

Repartitions the underlying RDD

Parameters
----------
npartitions : int
    Number of partitions to repartion the underlying RDD to


stk = StackedArray(self._rdd, shape=self.shape, split=self.split)<EOL>return stk.stack(size)<EOL>

Aggregates records of a distributed array.

Stacking should improve the performance of vectorized operations,
but the resulting StackedArray object only exposes a restricted set
of operations (e.g. map, reduce). The unstack method can be used
to restore the full bolt array.

Parameters
----------
size : int, optional, default=None
    The maximum size for each stack (number of original records),
    will aggregate groups of records per partition up to this size,
    if None will aggregate all records on each partition.

Returns
-------
StackedArray


<EOL>inshape(self.shape, axis)<EOL>tokeys = [(a - self.split) for a in axis if a >= self.split]<EOL>tovalues = [a for a in range(self.split) if a not in axis]<EOL>if tokeys or tovalues:<EOL><INDENT>return self.swap(tovalues, tokeys)<EOL><DEDENT>else:<EOL><INDENT>return self<EOL><DEDENT>

Align spark bolt array so that axes for iteration are in the keys.

This operation is applied before most functional operators.
It ensures that the specified axes are valid, and swaps
key/value axes so that functional operators can be applied
over the correct records.

Parameters
----------
axis: tuple[int]
    One or more axes that wil be iterated over by a functional operator

Returns
-------
BoltArraySpark


from bolt.local.array import BoltArrayLocal<EOL>rdd = self._rdd if self._ordered else self._rdd.sortByKey()<EOL>return BoltArrayLocal(rdd.values().first())<EOL>


def map(self, func, axis=(<NUM_LIT:0>,), value_shape=None, dtype=None, with_keys=False):

axis = tupleize(axis)<EOL>swapped = self._align(axis)<EOL>if with_keys:<EOL><INDENT>test_func = lambda x: func(((<NUM_LIT:0>,), x))<EOL><DEDENT>else:<EOL><INDENT>test_func = func<EOL><DEDENT>if value_shape is None or dtype is None:<EOL><INDENT>try:<EOL><INDENT>mapped = test_func(random.randn(*swapped.values.shape).astype(self.dtype))<EOL><DEDENT>except Exception:<EOL><INDENT>first = swapped._rdd.first()<EOL>if first:<EOL><INDENT>mapped = test_func(first[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>if value_shape is None:<EOL><INDENT>value_shape = mapped.shape<EOL><DEDENT>if dtype is None:<EOL><INDENT>dtype = mapped.dtype<EOL><DEDENT><DEDENT>shape = tuple([swapped._shape[ax] for ax in range(len(axis))]) + tupleize(value_shape)<EOL>if with_keys:<EOL><INDENT>rdd = swapped._rdd.map(lambda kv: (kv[<NUM_LIT:0>], func(kv)))<EOL><DEDENT>else:<EOL><INDENT>rdd = swapped._rdd.mapValues(func)<EOL><DEDENT>def check(v):<EOL><INDENT>if len(v.shape) > <NUM_LIT:0> and v.shape != tupleize(value_shape):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>return v<EOL><DEDENT>rdd = rdd.mapValues(lambda v: check(v))<EOL>return self._constructor(rdd, shape=shape, dtype=dtype, split=swapped.split).__finalize__(swapped)<EOL>

Apply a function across an axis.

Array will be aligned so that the desired set of axes
are in the keys, which may incur a swap.

Parameters
----------
func : function
    Function of a single array to apply. If with_keys=True,
    function should be of a (tuple, array) pair.

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to apply function along.

value_shape : tuple, optional, default=None
    Known shape of values resulting from operation

dtype: numpy.dtype, optional, default=None
    Known dtype of values resulting from operation

with_keys : bool, optional, default=False
    Include keys as an argument to the function

Returns
-------
BoltArraySpark


def filter(self, func, axis=(<NUM_LIT:0>,), sort=False):

axis = tupleize(axis)<EOL>swapped = self._align(axis)<EOL>def f(record):<EOL><INDENT>return func(record[<NUM_LIT:1>])<EOL><DEDENT>rdd = swapped._rdd.filter(f)<EOL>if sort:<EOL><INDENT>rdd = rdd.sortByKey().values()<EOL><DEDENT>else:<EOL><INDENT>rdd = rdd.values()<EOL><DEDENT>count, zipped = zip_with_index(rdd)<EOL>if not count:<EOL><INDENT>count = zipped.count()<EOL><DEDENT>reindexed = zipped.map(lambda kv: (tupleize(kv[<NUM_LIT:1>]), kv[<NUM_LIT:0>]))<EOL>remaining = list(swapped.shape[len(axis):])<EOL>if count != <NUM_LIT:0>:<EOL><INDENT>shape = tuple([count] + remaining)<EOL><DEDENT>else:<EOL><INDENT>shape = (<NUM_LIT:0>,)<EOL><DEDENT>return self._constructor(reindexed, shape=shape, split=<NUM_LIT:1>).__finalize__(swapped)<EOL>

Filter array along an axis.

Applies a function which should evaluate to boolean,
along a single axis or multiple axes. Array will be
aligned so that the desired set of axes are in the keys,
which may incur a swap.

Parameters
----------
func : function
    Function to apply, should return boolean

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to filter along.

sort: bool, optional, default=False
    Whether or not to sort by key before reindexing

Returns
-------
BoltArraySpark


def reduce(self, func, axis=(<NUM_LIT:0>,), keepdims=False):

from bolt.local.array import BoltArrayLocal<EOL>from numpy import ndarray<EOL>axis = tupleize(axis)<EOL>swapped = self._align(axis)<EOL>arr = swapped._rdd.values().treeReduce(func, depth=<NUM_LIT:3>)<EOL>if keepdims:<EOL><INDENT>for i in axis:<EOL><INDENT>arr = expand_dims(arr, axis=i)<EOL><DEDENT><DEDENT>if not isinstance(arr, ndarray):<EOL><INDENT>return arr<EOL><DEDENT>elif arr.shape == (<NUM_LIT:1>,):<EOL><INDENT>return arr[<NUM_LIT:0>]<EOL><DEDENT>return BoltArrayLocal(arr)<EOL>

Reduce an array along an axis.

Applies a commutative/associative function of two
arguments cumulatively to all arrays along an axis.
Array will be aligned so that the desired set of axes
are in the keys, which may incur a swap.

Parameters
----------
func : function
    Function of two arrays that returns a single array

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to reduce along.

Returns
-------
BoltArraySpark


def _stat(self, axis=None, func=None, name=None, keepdims=False):

if axis is None:<EOL><INDENT>axis = list(range(len(self.shape)))<EOL><DEDENT>axis = tupleize(axis)<EOL>if func and not name:<EOL><INDENT>return self.reduce(func, axis, keepdims)<EOL><DEDENT>if name and not func:<EOL><INDENT>from bolt.local.array import BoltArrayLocal<EOL>swapped = self._align(axis)<EOL>def reducer(left, right):<EOL><INDENT>return left.combine(right)<EOL><DEDENT>counter = swapped._rdd.values().mapPartitions(lambda i: [StatCounter(values=i, stats=name)]).treeReduce(reducer, depth=<NUM_LIT:3>)<EOL>arr = getattr(counter, name)<EOL>if keepdims:<EOL><INDENT>for i in axis:<EOL><INDENT>arr = expand_dims(arr, axis=i)<EOL><DEDENT><DEDENT>return BoltArrayLocal(arr).toscalar()<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

Compute a statistic over an axis.

Can provide either a function (for use in a reduce)
or a name (for use by a stat counter).

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

func : function, optional, default=None
    Function for reduce, see BoltArraySpark.reduce

name : str
    A named statistic, see StatCounter

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def mean(self, axis=None, keepdims=False):

return self._stat(axis, name='<STR_LIT>', keepdims=keepdims)<EOL>

Return the mean of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def var(self, axis=None, keepdims=False):

Return the variance of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def std(self, axis=None, keepdims=False):

Return the standard deviation of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def sum(self, axis=None, keepdims=False):

from operator import add<EOL>return self._stat(axis, func=add, keepdims=keepdims)<EOL>

Return the sum of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def max(self, axis=None, keepdims=False):

from numpy import maximum<EOL>return self._stat(axis, func=maximum, keepdims=keepdims)<EOL>

Return the maximum of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def min(self, axis=None, keepdims=False):

from numpy import minimum<EOL>return self._stat(axis, func=minimum, keepdims=keepdims)<EOL>

Return the minimum of the array over the given axis.

Parameters
----------
axis : tuple or int, optional, default=None
    Axis to compute statistic over, if None
    will compute over all axes

keepdims : boolean, optional, default=False
    Keep axis remaining after operation with size 1.


def concatenate(self, arry, axis=<NUM_LIT:0>):

if isinstance(arry, ndarray):<EOL><INDENT>from bolt.spark.construct import ConstructSpark<EOL>arry = ConstructSpark.array(arry, self._rdd.context, axis=range(<NUM_LIT:0>, self.split))<EOL><DEDENT>else:<EOL><INDENT>if not isinstance(arry, BoltArraySpark):<EOL><INDENT>raise ValueError("<STR_LIT>" % type(arry))<EOL><DEDENT><DEDENT>if not all([x == y if not i == axis else True<EOL>for i, (x, y) in enumerate(zip(self.shape, arry.shape))]):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if not self.split == arry.split:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>if axis < self.split:<EOL><INDENT>shape = self.keys.shape<EOL>def key_func(key):<EOL><INDENT>key = list(key)<EOL>key[axis] += shape[axis]<EOL>return tuple(key)<EOL><DEDENT>rdd = self._rdd.union(arry._rdd.map(lambda kv: (key_func(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>])))<EOL><DEDENT>else:<EOL><INDENT>from numpy import concatenate as npconcatenate<EOL>shift = axis - self.split<EOL>rdd = self._rdd.join(arry._rdd).map(lambda kv: (kv[<NUM_LIT:0>], npconcatenate(kv[<NUM_LIT:1>], axis=shift)))<EOL><DEDENT>shape = tuple([x + y if i == axis else x<EOL>for i, (x, y) in enumerate(zip(self.shape, arry.shape))])<EOL>return self._constructor(rdd, shape=shape, ordered=False).__finalize__(self)<EOL>

Join this array with another array.

Paramters
---------
arry : ndarray, BoltArrayLocal, or BoltArraySpark
    Another array to concatenate with

axis : int, optional, default=0
    The axis along which arrays will be joined.

Returns
-------
BoltArraySpark


key_slices = index[<NUM_LIT:0>:self.split]<EOL>value_slices = index[self.split:]<EOL>def key_check(key):<EOL><INDENT>def inrange(k, s):<EOL><INDENT>if s.step > <NUM_LIT:0>:<EOL><INDENT>return s.start <= k < s.stop<EOL><DEDENT>else:<EOL><INDENT>return s.stop < k <= s.start<EOL><DEDENT><DEDENT>def check(k, s):<EOL><INDENT>return inrange(k, s) and mod(k - s.start, s.step) == <NUM_LIT:0><EOL><DEDENT>out = [check(k, s) for k, s in zip(key, key_slices)]<EOL>return all(out)<EOL><DEDENT>def key_func(key):<EOL><INDENT>return tuple([(k - s.start)/s.step for k, s in zip(key, key_slices)])<EOL><DEDENT>filtered = self._rdd.filter(lambda kv: key_check(kv[<NUM_LIT:0>]))<EOL>if self._split == self.ndim:<EOL><INDENT>rdd = filtered.map(lambda kv: (key_func(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>]))<EOL><DEDENT>else:<EOL><INDENT>value_slices = [s if s.stop != -<NUM_LIT:1> else slice(s.start, None, s.step) for s in value_slices]<EOL>rdd = filtered.map(lambda kv: (key_func(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>][value_slices]))<EOL><DEDENT>shape = tuple([int(ceil((s.stop - s.start) / float(s.step))) for s in index])<EOL>split = self.split<EOL>return rdd, shape, split<EOL>


index = [asarray(i) for i in index]<EOL>shape = index[<NUM_LIT:0>].shape<EOL>if not all([i.shape == shape for i in index]):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>" + ("<STR_LIT>" * self.ndim)<EOL>% tuple([i.shape for i in index]))<EOL><DEDENT>index = tuple([listify(i, d) for (i, d) in zip(index, self.shape)])<EOL>key_tuples = list(zip(*index[<NUM_LIT:0>:self.split]))<EOL>value_tuples = list(zip(*index[self.split:]))<EOL>d = {}<EOL>for k, g in groupby(zip(value_tuples, key_tuples), lambda x: x[<NUM_LIT:1>]):<EOL><INDENT>d[k] = map(lambda x: x[<NUM_LIT:0>], list(g))<EOL><DEDENT>def key_check(key):<EOL><INDENT>return key in key_tuples<EOL><DEDENT>def key_func(key):<EOL><INDENT>return unravel_index(key, shape)<EOL><DEDENT>filtered = self._rdd.filter(lambda kv: key_check(kv[<NUM_LIT:0>]))<EOL>if len(value_tuples) > <NUM_LIT:0>:<EOL><INDENT>flattened = filtered.flatMap(lambda kv: [(kv[<NUM_LIT:0>], kv[<NUM_LIT:1>][i]) for i in d[kv[<NUM_LIT:0>]]])<EOL><DEDENT>else:<EOL><INDENT>flattened = filtered<EOL><DEDENT>indexed = flattened.zipWithIndex()<EOL>rdd = indexed.map(lambda kkv: (key_func(kkv[<NUM_LIT:1>]), kkv[<NUM_LIT:0>][<NUM_LIT:1>]))<EOL>split = len(shape)<EOL>return rdd, shape, split<EOL>

Advanced indexing (for sets, lists, or ndarrays).


<EOL>loc = where([isinstance(i, (tuple, list, ndarray)) for i in index])[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>idx = list(index[loc])<EOL>if isinstance(idx[<NUM_LIT:0>], (tuple, list, ndarray)):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if loc < self.split:<EOL><INDENT>def newkey(key):<EOL><INDENT>newkey = list(key)<EOL>newkey[loc] = idx.index(key[loc])<EOL>return tuple(newkey)<EOL><DEDENT>rdd = self._rdd.filter(lambda kv: kv[<NUM_LIT:0>][loc] in idx).map(lambda kv: (newkey(kv[<NUM_LIT:0>]), kv[<NUM_LIT:1>]))<EOL><DEDENT>else:<EOL><INDENT>slices = [slice(<NUM_LIT:0>, None, None) for _ in self.values.shape]<EOL>slices[loc - self.split] = idx<EOL>rdd = self._rdd.map(lambda kv: (kv[<NUM_LIT:0>], kv[<NUM_LIT:1>][slices]))<EOL><DEDENT>newshape = list(self.shape)<EOL>newshape[loc] = len(idx)<EOL>barray = self._constructor(rdd, shape=tuple(newshape)).__finalize__(self)<EOL>new_index = index[:]<EOL>new_index[loc] = slice(<NUM_LIT:0>, None, None)<EOL>barray = barray[tuple(new_index)]<EOL>return barray._rdd, barray.shape, barray.split<EOL>

Mixed indexing (combines basic and advanced indexes)

Assumes that only a single advanced index is used, due to the complicated
behavior needed to be compatible with NumPy otherwise.


if isinstance(index, tuple):<EOL><INDENT>index = list(index)<EOL><DEDENT>else:<EOL><INDENT>index = [index]<EOL><DEDENT>int_locs = where([isinstance(i, int) for i in index])[<NUM_LIT:0>]<EOL>if len(index) > self.ndim:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not all([isinstance(i, (slice, int, list, tuple, ndarray)) for i in index]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if len(index) < self.ndim:<EOL><INDENT>index += tuple([slice(<NUM_LIT:0>, None, None) for _ in range(self.ndim - len(index))])<EOL><DEDENT>for n, idx in enumerate(index):<EOL><INDENT>size = self.shape[n]<EOL>if isinstance(idx, (slice, int)):<EOL><INDENT>slc = slicify(idx, size)<EOL>if slc.step > <NUM_LIT:0>:<EOL><INDENT>minval, maxval = slc.start, slc.stop<EOL><DEDENT>else:<EOL><INDENT>minval, maxval = slc.stop, slc.start<EOL><DEDENT>if minval > size-<NUM_LIT:1> or maxval < <NUM_LIT:1> or minval >= maxval:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(idx, n, size))<EOL><DEDENT>index[n] = slc<EOL><DEDENT>else:<EOL><INDENT>adjusted = array(idx)<EOL>inds = where(adjusted<<NUM_LIT:0>)<EOL>adjusted[inds] += size<EOL>if adjusted.min() < <NUM_LIT:0> or adjusted.max() > size-<NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(idx, n, size))<EOL><DEDENT>index[n] = adjusted<EOL><DEDENT><DEDENT>if all([isinstance(i, slice) for i in index]):<EOL><INDENT>rdd, shape, split = self._getbasic(index)<EOL><DEDENT>elif all([isinstance(i, (tuple, list, ndarray)) for i in index]):<EOL><INDENT>rdd, shape, split = self._getadvanced(index)<EOL><DEDENT>elif sum([isinstance(i, (tuple, list, ndarray)) for i in index]) == <NUM_LIT:1>:<EOL><INDENT>rdd, shape, split = self._getmixed(index)<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if self._ordered is False or any([isinstance(s, slice) and s.step<<NUM_LIT:0> for s in index[:self.split]]):<EOL><INDENT>ordered = False<EOL><DEDENT>else:<EOL><INDENT>ordered = True<EOL><DEDENT>result = self._constructor(rdd, shape=shape, split=split, ordered=ordered).__finalize__(self)<EOL>if len(int_locs) == self.ndim:<EOL><INDENT>return result.squeeze().toarray()[()]<EOL><DEDENT>else:<EOL><INDENT>return result.squeeze(tuple(int_locs))<EOL><DEDENT>

Get an item from the array through indexing.

Supports basic indexing with slices and ints, or advanced
indexing with lists or ndarrays of integers.
Mixing basic and advanced indexing across axes is currently supported
only for a single advanced index amidst multiple basic indices.

Parameters
----------
index : tuple of slices, ints, list, tuple, or ndarrays
    One or more index specifications

Returns
-------
BoltSparkArray


def chunk(self, size="<STR_LIT>", axis=None, padding=None):

if type(size) is not str:<EOL><INDENT>size = tupleize((size))<EOL><DEDENT>axis = tupleize((axis))<EOL>padding = tupleize((padding))<EOL>from bolt.spark.chunk import ChunkedArray<EOL>chnk = ChunkedArray(rdd=self._rdd, shape=self._shape, split=self._split, dtype=self._dtype)<EOL>return chnk._chunk(size, axis, padding)<EOL>

Chunks records of a distributed array.

Chunking breaks arrays into subarrays, using an specified
size of chunks along each value dimension. Can alternatively
specify an average chunk byte size (in kilobytes) and the size of
chunks (as ints) will be computed automatically.

Parameters
----------
size : tuple, int, or str, optional, default = "150"
    A string giving the size in kilobytes, or a tuple with the size
    of chunks along each dimension.

axis : int or tuple, optional, default = None
    One or more axis to chunk array along, if None
    will use all axes,

padding: tuple or int, default = None
    Number of elements per dimension that will overlap with the adjacent chunk.
    If a tuple, specifies padding along each chunked dimension; if a int, same
    padding will be applied to all chunked dimensions.

Returns
-------
ChunkedArray


def swap(self, kaxes, vaxes, size="<STR_LIT>"):

kaxes = asarray(tupleize(kaxes), '<STR_LIT:int>')<EOL>vaxes = asarray(tupleize(vaxes), '<STR_LIT:int>')<EOL>if type(size) is not str:<EOL><INDENT>size = tupleize(size)<EOL><DEDENT>if len(kaxes) == self.keys.ndim and len(vaxes) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if len(kaxes) == <NUM_LIT:0> and len(vaxes) == <NUM_LIT:0>:<EOL><INDENT>return self<EOL><DEDENT>from bolt.spark.chunk import ChunkedArray<EOL>chunks = self.chunk(size)<EOL>swapped = chunks.keys_to_values(kaxes).values_to_keys([v+len(kaxes) for v in vaxes])<EOL>barray = swapped.unchunk()<EOL>return barray<EOL>

Swap axes from keys to values.

This is the core operation underlying shape manipulation
on the Spark bolt array. It exchanges an arbitrary set of axes
between the keys and the valeus. If either is None, will only
move axes in one direction (from keys to values, or values to keys).
Keys moved to values will be placed immediately after the split;
values moved to keys will be placed immediately before the split.

Parameters
----------
kaxes : tuple
    Axes from keys to move to values

vaxes : tuple
    Axes from values to move to keys

size : tuple or int, optional, default = "150"
    Can either provide a string giving the size in kilobytes,
    or a tuple with the number of chunks along each
    value dimension being moved

Returns
-------
BoltArraySpark


if len(axes) == <NUM_LIT:0>:<EOL><INDENT>p = arange(self.ndim-<NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>p = asarray(argpack(axes))<EOL><DEDENT>istransposeable(p, range(self.ndim))<EOL>split = self.split<EOL>new_keys, new_values = p[:split], p[split:]<EOL>swapping_keys = sort(new_values[new_values < split])<EOL>swapping_values = sort(new_keys[new_keys >= split])<EOL>stationary_keys = sort(new_keys[new_keys < split])<EOL>stationary_values = sort(new_values[new_values >= split])<EOL>p_swap = r_[stationary_keys, swapping_values, swapping_keys, stationary_values]<EOL>p_swap_inv = argsort(p_swap)<EOL>p_x = p_swap_inv[p]<EOL>p_keys, p_values = p_x[:split], p_x[split:]-split<EOL>arr = self.swap(swapping_keys, swapping_values-split)<EOL>arr = arr.keys.transpose(tuple(p_keys.tolist()))<EOL>arr = arr.values.transpose(tuple(p_values.tolist()))<EOL>return arr<EOL>

Return an array with the axes transposed.

This operation will incur a swap unless the
desiured permutation can be obtained
only by transpoing the keys or the values.

Parameters
----------
axes : None, tuple of ints, or n ints
    If None, will reverse axis order.


return self.transpose()<EOL>

Transpose by reversing the order of the axes.


p = list(range(self.ndim))<EOL>p[axis1] = axis2<EOL>p[axis2] = axis1<EOL>return self.transpose(p)<EOL>

Return the array with two axes interchanged.

Parameters
----------
axis1 : int
    The first axis to swap

axis2 : int
    The second axis to swap


new = argpack(shape)<EOL>isreshapeable(new, self.shape)<EOL>if new == self.shape:<EOL><INDENT>return self<EOL><DEDENT>i = self._reshapebasic(new)<EOL>if i == -<NUM_LIT:1>:<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>new_key_shape, new_value_shape = new[:i], new[i:]<EOL>return self.keys.reshape(new_key_shape).values.reshape(new_value_shape)<EOL><DEDENT>

Return an array with the same data but a new shape.

Currently only supports reshaping that independently
reshapes the keys, or the values, or both.

Parameters
----------
shape :  tuple of ints, or n ints
    New shape


new = tupleize(shape)<EOL>old_key_size = prod(self.keys.shape)<EOL>old_value_size = prod(self.values.shape)<EOL>for i in range(len(new)):<EOL><INDENT>new_key_size = prod(new[:i])<EOL>new_value_size = prod(new[i:])<EOL>if new_key_size == old_key_size and new_value_size == old_value_size:<EOL><INDENT>return i<EOL><DEDENT><DEDENT>return -<NUM_LIT:1><EOL>

Check if the requested reshape can be broken into independant reshapes
on the keys and values. If it can, returns the index in the new shape
separating keys from values, otherwise returns -1


if not any([d == <NUM_LIT:1> for d in self.shape]):<EOL><INDENT>return self<EOL><DEDENT>if axis is None:<EOL><INDENT>drop = where(asarray(self.shape) == <NUM_LIT:1>)[<NUM_LIT:0>]<EOL><DEDENT>elif isinstance(axis, int):<EOL><INDENT>drop = asarray((axis,))<EOL><DEDENT>elif isinstance(axis, tuple):<EOL><INDENT>drop = asarray(axis)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if any([self.shape[i] > <NUM_LIT:1> for i in drop]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if any(asarray(drop) < self.split):<EOL><INDENT>kmask = set([d for d in drop if d < self.split])<EOL>kfunc = lambda k: tuple([kk for ii, kk in enumerate(k) if ii not in kmask])<EOL><DEDENT>else:<EOL><INDENT>kfunc = lambda k: k<EOL><DEDENT>if any(asarray(drop) >= self.split):<EOL><INDENT>vmask = tuple([d - self.split for d in drop if d >= self.split])<EOL>vfunc = lambda v: v.squeeze(vmask)<EOL><DEDENT>else:<EOL><INDENT>vfunc = lambda v: v<EOL><DEDENT>rdd = self._rdd.map(lambda kv: (kfunc(kv[<NUM_LIT:0>]), vfunc(kv[<NUM_LIT:1>])))<EOL>shape = tuple([ss for ii, ss in enumerate(self.shape) if ii not in drop])<EOL>split = len([d for d in range(self.keys.ndim) if d not in drop])<EOL>return self._constructor(rdd, shape=shape, split=split).__finalize__(self)<EOL>

Remove one or more single-dimensional axes from the array.

Parameters
----------
axis : tuple or int
    One or more singleton axes to remove.


def astype(self, dtype, casting='<STR_LIT>'):

rdd = self._rdd.mapValues(lambda v: v.astype(dtype, '<STR_LIT>', casting))<EOL>return self._constructor(rdd, dtype=dtype).__finalize__(self)<EOL>

Cast the array to a specified type.

Parameters
----------
dtype : str or dtype
    Typecode or data-type to cast the array to (see numpy)


rdd = self._rdd.mapValues(lambda v: v.clip(min=min, max=max))<EOL>return self._constructor(rdd).__finalize__(self)<EOL>

Clip values above and below.

Parameters
----------
min : scalar or array-like
    Minimum value. If array, will be broadcasted

max : scalar or array-like
    Maximum value. If array, will be broadcasted.


return self._shape<EOL>


return prod(self._shape)<EOL>


return len(self._shape)<EOL>


return self._split<EOL>

Axis at which the array is split into keys/values.


return self._dtype<EOL>


from bolt.spark.shapes import Keys<EOL>return Keys(self)<EOL>


from bolt.local.array import BoltArrayLocal<EOL>return BoltArrayLocal(self.toarray())<EOL>

Returns a local bolt array by first collecting as an array.


rdd = self._rdd if self._ordered else self._rdd.sortByKey()<EOL>x = rdd.values().collect()<EOL>return asarray(x).reshape(self.shape)<EOL>

Returns the contents as a local array.

Will likely cause memory problems for large objects.


return self._rdd<EOL>

Return the underlying RDD of the bolt array.


for x in self._rdd.take(<NUM_LIT:10>):<EOL><INDENT>print(x)<EOL><DEDENT>

Show a pretty-printed representation of this BoltArrayLocal.


raise NotImplementedError<EOL>


raise NotImplementedError<EOL>

Return the sum of the array elements over the given axis.


raise NotImplementedError<EOL>

Return the mean of the array elements over the given axis.


raise NotImplementedError<EOL>

Return the variance of the array elements over the given axis.


raise NotImplementedError<EOL>

Return the standard deviation of the array elements over the given axis.


raise NotImplementedError<EOL>

Return the minimum of the array elements over the given axis or axes.


raise NotImplementedError<EOL>

Return the maximum of the array elements over the given axis or axes.


raise NotImplementedError<EOL>

Return an array with the axes transposed.


raise NotImplementedError<EOL>

Transpose by reversing the order of the axes.


raise NotImplementedError<EOL>

Return an array with the same data but a new shape.


raise NotImplementedError<EOL>

Remove one or more single-dimensional axes from the array.


raise NotImplementedError<EOL>

Return an array with two axes interchanged.


raise NotImplementedError<EOL>

Apply a function across one or more axes.


raise NotImplementedError<EOL>

Reduce an array across one or more axes.


raise NotImplementedError<EOL>

Filter an array across one or more axes.


@staticmethod<EOL><INDENT>def array(a, dtype=None, order='<STR_LIT:C>'):<DEDENT>

return BoltArrayLocal(asarray(a, dtype, order))<EOL>

Create a local bolt array.

Parameters
----------
a : array-like
    An array, any object exposing the array interface, an
    object whose __array__ method returns an array, or any
    (nested) sequence.

dtype : data-type, optional, default=None
    The desired data-type for the array. If None, will
    be determined from the data. (see numpy)

order : {'C', 'F', 'A'}, optional, default='C'
    The order of the array. (see numpy)

Returns
-------
BoltArrayLocal


@staticmethod<EOL><INDENT>def ones(shape, dtype=float64, order='<STR_LIT:C>'):<DEDENT>

from numpy import ones<EOL>return ConstructLocal._wrap(ones, shape, dtype, order)<EOL>

Create a local bolt array of ones.

Parameters
----------
shape : tuple
    Dimensions of the desired array

dtype : data-type, optional, default=float64
    The desired data-type for the array. (see numpy)

order : {'C', 'F', 'A'}, optional, default='C'
    The order of the array. (see numpy)

Returns
-------
BoltArrayLocal


@staticmethod<EOL><INDENT>def zeros(shape, dtype=float64, order='<STR_LIT:C>'):<DEDENT>

from numpy import zeros<EOL>return ConstructLocal._wrap(zeros, shape, dtype, order)<EOL>

Create a local bolt array of zeros.

Parameters
----------
shape : tuple
    Dimensions of the desired array.

dtype : data-type, optional, default=float64
    The desired data-type for the array. (see numpy)

order : {'C', 'F', 'A'}, optional, default='C'
    The order of the array. (see numpy)

Returns
-------
BoltArrayLocal


if not isinstance(arrays, tuple):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>arrays = tuple([asarray(a) for a in arrays])<EOL>from numpy import concatenate<EOL>return BoltArrayLocal(concatenate(arrays, axis))<EOL>

Join a sequence of arrays together.

Parameters
----------
arrays : tuple
    A sequence of array-like e.g. (a1, a2, ...)

axis : int, optional, default=0
    The axis along which the arrays will be joined.

Returns
-------
BoltArrayLocal


<EOL>inshape(self.shape, axes)<EOL>remaining = [dim for dim in range(len(self.shape)) if dim not in axes]<EOL>key_shape = key_shape if key_shape else [self.shape[axis] for axis in axes]<EOL>remaining_shape = [self.shape[axis] for axis in remaining]<EOL>linearized_shape = [prod(key_shape)] + remaining_shape<EOL>transpose_order = axes + remaining<EOL>reshaped = self.transpose(*transpose_order).reshape(*linearized_shape)<EOL>return reshaped<EOL>

Align local bolt array so that axes for iteration are in the keys.

This operation is applied before most functional operators.
It ensures that the specified axes are valid, and might transpose/reshape
the underlying array so that the functional operators can be applied
over the correct records.

Parameters
----------
axes: tuple[int]
    One or more axes that will be iterated over by a functional operator

Returns
-------
BoltArrayLocal


def filter(self, func, axis=(<NUM_LIT:0>,)):

axes = sorted(tupleize(axis))<EOL>reshaped = self._align(axes)<EOL>filtered = asarray(list(filter(func, reshaped)))<EOL>return self._constructor(filtered)<EOL>

Filter array along an axis.

Applies a function which should evaluate to boolean,
along a single axis or multiple axes. Array will be
aligned so that the desired set of axes are in the
keys, which may require a transpose/reshape.

Parameters
----------
func : function
    Function to apply, should return boolean

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to filter along.

Returns
-------
BoltArrayLocal


def map(self, func, axis=(<NUM_LIT:0>,)):

axes = sorted(tupleize(axis))<EOL>key_shape = [self.shape[axis] for axis in axes]<EOL>reshaped = self._align(axes, key_shape=key_shape)<EOL>mapped = asarray(list(map(func, reshaped)))<EOL>elem_shape = mapped[<NUM_LIT:0>].shape<EOL>linearized_shape_inv = key_shape + list(elem_shape)<EOL>reordered = mapped.reshape(*linearized_shape_inv)<EOL>return self._constructor(reordered)<EOL>

Apply a function across an axis.

Array will be aligned so that the desired set of axes
are in the keys, which may require a transpose/reshape.

Parameters
----------
func : function
    Function of a single array to apply

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to apply function along.

Returns
-------
BoltArrayLocal


def reduce(self, func, axis=<NUM_LIT:0>):

axes = sorted(tupleize(axis))<EOL>if isinstance(func, ufunc):<EOL><INDENT>inshape(self.shape, axes)<EOL>reduced = func.reduce(self, axis=tuple(axes))<EOL><DEDENT>else:<EOL><INDENT>reshaped = self._align(axes)<EOL>reduced = reduce(func, reshaped)<EOL><DEDENT>new_array = self._constructor(reduced)<EOL>expected_shape = [self.shape[i] for i in range(len(self.shape)) if i not in axes]<EOL>if new_array.shape != tuple(expected_shape):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return new_array<EOL>

Reduce an array along an axis.

Applies an associative/commutative function of two arguments
cumulatively to all arrays along an axis. Array will be aligned
so that the desired set of axes are in the keys, which may
require a transpose/reshape.

Parameters
----------
func : function
    Function of two arrays that returns a single array

axis : tuple or int, optional, default=(0,)
    Axis or multiple axes to reduce along.

Returns
-------
BoltArrayLocal


return self[<NUM_LIT:0>]<EOL>


if isinstance(arry, ndarray):<EOL><INDENT>from bolt import concatenate<EOL>return concatenate((self, arry), axis)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % type(arry))<EOL><DEDENT>

Join this array with another array.

Paramters
---------
arry : ndarray or BoltArrayLocal
    Another array to concatenate with

axis : int, optional, default=0
    The axis along which arrays will be joined.

Returns
-------
BoltArrayLocal


if self.shape == ():<EOL><INDENT>return self.toarray().reshape(<NUM_LIT:1>)[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return self<EOL><DEDENT>

Returns the single scalar element contained in an array of shape (), if
the array has that shape. Returns self otherwise.


def tospark(self, sc, axis=<NUM_LIT:0>):

from bolt import array<EOL>return array(self.toarray(), sc, axis=axis)<EOL>

Converts a BoltArrayLocal into a BoltArraySpark

Parameters
----------
sc : SparkContext
    The SparkContext which will be used to create the BoltArraySpark

axis : tuple or int, optional, default=0
    The axis (or axes) across which this array will be parallelized

Returns
-------
BoltArraySpark


from bolt import array<EOL>return array(self.toarray(), sc, axis=axis).tordd()<EOL>

Converts a BoltArrayLocal into an RDD

Parameters
----------
sc : SparkContext
    The SparkContext which will be used to create the BoltArraySpark

axis : tuple or int, optional, default=0
    The axis (or axes) across which this array will be parallelized

Returns
-------
RDD[(tuple, ndarray)]


return asarray(self)<EOL>

Returns the underlying ndarray wrapped by this BoltArrayLocal


print(str(self))<EOL>

Show a pretty-printed representation of this BoltArrayLocal


if arg is None:<EOL><INDENT>return None<EOL><DEDENT>if not isinstance(arg, (tuple, list, ndarray, Iterable)):<EOL><INDENT>return tuple((arg,))<EOL><DEDENT>elif isinstance(arg, (list, ndarray)):<EOL><INDENT>return tuple(arg)<EOL><DEDENT>elif isinstance(arg, Iterable) and not isinstance(arg, str):<EOL><INDENT>return tuple(arg)<EOL><DEDENT>else:<EOL><INDENT>return arg<EOL><DEDENT>

Coerce singletons and lists and ndarrays to tuples.

Parameters
----------
arg : tuple, list, ndarray, or singleton
    Item to coerce


if isinstance(args[<NUM_LIT:0>], (tuple, list, ndarray)):<EOL><INDENT>return tupleize(args[<NUM_LIT:0>])<EOL><DEDENT>elif isinstance(args[<NUM_LIT:0>], Iterable) and not isinstance(args[<NUM_LIT:0>], str):<EOL><INDENT>return tupleize(list(args[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>return tuple(args)<EOL><DEDENT>

Coerce a list of arguments to a tuple.

Parameters
----------
args : tuple or nested tuple
    Pack arguments into a tuple, converting ((,...),) or (,) -> (,)


valid = all([(axis < len(shape)) and (axis >= <NUM_LIT:0>) for axis in axes])<EOL>if not valid:<EOL><INDENT>raise ValueError("<STR_LIT>" % str(shape))<EOL><DEDENT>

Checks to see if a list of axes are contained within an array shape.

Parameters
----------
shape : tuple[int]
    the shape of a BoltArray

axes : tuple[int]
    the axes to check against shape


from numpy import allclose<EOL>return (a.shape == b.shape) and allclose(a, b)<EOL>

Test that a and b are close and match in shape.

Parameters
----------
a : ndarray
    First array to check

b : ndarray
    First array to check


return sorted(range(len(seq)), key=seq.__getitem__)<EOL>

Sort a list by a sequence.

Parameters
----------
seq : tuple
    Sequence to sort by


if not all([l.dtype == int for l in lst]):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if npany(asarray(lst) >= dim):<EOL><INDENT>raise ValueError("<STR_LIT>" % dim)<EOL><DEDENT>return lst.flatten()<EOL>

Flatten lists of indices and ensure bounded by a known dim.

Parameters
----------
lst : list
    List of integer indices

dim : tuple
    Bounds for indices


if isinstance(slc, slice):<EOL><INDENT>start = <NUM_LIT:0> if slc.start is None else slc.start<EOL>stop = dim if slc.stop is None else slc.stop<EOL>step = <NUM_LIT:1> if slc.step is None else slc.step<EOL>if start < <NUM_LIT:0>: start += dim<EOL>if stop < <NUM_LIT:0>: stop += dim<EOL>if step > <NUM_LIT:0>:<EOL><INDENT>if start < <NUM_LIT:0>: start = <NUM_LIT:0><EOL>if stop > dim: stop = dim<EOL><DEDENT>else:<EOL><INDENT>if stop < <NUM_LIT:0>: stop = -<NUM_LIT:1><EOL>if start > dim: start = dim-<NUM_LIT:1><EOL><DEDENT>return slice(start, stop, step)<EOL><DEDENT>elif isinstance(slc, int):<EOL><INDENT>if slc < <NUM_LIT:0>:<EOL><INDENT>slc += dim<EOL><DEDENT>return slice(slc, slc+<NUM_LIT:1>, <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % type(slc))<EOL><DEDENT>

Force a slice to have defined start, stop, and step from a known dim.
Start and stop will always be positive. Step may be negative.

There is an exception where a negative step overflows the stop needs to have
the default value set to -1. This is the only case of a negative start/stop
value.

Parameters
----------
slc : slice or int
    The slice to modify, or int to convert to a slice

dim : tuple
    Bound for slice


new, old = tupleize(new), tupleize(old)<EOL>if not len(new) == len(old):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not len(set(new)) == len(set(old)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if any(n < <NUM_LIT:0> for n in new) or max(new) > len(old) - <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Check to see if a proposed tuple of axes is a valid permutation
of an old set of axes. Checks length, axis repetion, and bounds.

Parameters
----------
new : tuple
    tuple of proposed axes

old : tuple
    tuple of old axes


new, old = tupleize(new), tupleize(old)<EOL>if not prod(new) == prod(old):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Check to see if a proposed tuple of axes is a valid reshaping of
the old axes by ensuring that they can be factored.

Parameters
----------
new : tuple
    tuple of proposed axes

old : tuple
    tuple of old axes


if type(vals[<NUM_LIT:0>]) is ndarray:<EOL><INDENT>return concatenate(vals, axis=depth)<EOL><DEDENT>else:<EOL><INDENT>return concatenate([allstack(x, depth+<NUM_LIT:1>) for x in vals], axis=depth)<EOL><DEDENT>

If an ndarray has been split into multiple chunks by splitting it along
each axis at a number of locations, this function rebuilds the
original array from chunks.

Parameters
----------
vals : nested lists of ndarrays
    each level of nesting of the lists representing a dimension of
    the original array.


for d in range(arry.ndim, arry.ndim+extra):<EOL><INDENT>arry = expand_dims(arry, axis=d)<EOL><DEDENT>return arry<EOL>

Expand dimensions by iteratively append empty axes.

Parameters
----------
arry : ndarray
    The original array

extra : int
    The number of empty axes to append


import inspect<EOL>def extract(func):<EOL><INDENT>append = "<STR_LIT>"<EOL>args = inspect.getargspec(func)<EOL>for i, a in enumerate(args.args):<EOL><INDENT>if i < (len(args) - len(args.defaults)):<EOL><INDENT>append += str(a) + "<STR_LIT:U+002CU+0020>"<EOL><DEDENT>else:<EOL><INDENT>default = args.defaults[i-len(args.defaults)]<EOL>if hasattr(default, "<STR_LIT>"):<EOL><INDENT>default = default.__name__<EOL><DEDENT>else:<EOL><INDENT>default = str(default)<EOL><DEDENT>append += str(a) + "<STR_LIT:=>" + default + "<STR_LIT:U+002CU+0020>"<EOL><DEDENT><DEDENT>append = append[:-<NUM_LIT:2>] + "<STR_LIT:)>"<EOL>return append<EOL><DEDENT>doc = f.__doc__ + "<STR_LIT:\n>"<EOL>doc += "<STR_LIT>" + extract(getattr(ConstructLocal, f.__name__)) + "<STR_LIT:\n>"<EOL>doc += "<STR_LIT>" + extract(getattr(ConstructSpark, f.__name__)) + "<STR_LIT:\n>"<EOL>f.__doc__ = doc<EOL>return f<EOL>


if '<STR_LIT>' in kwargs:<EOL><INDENT>mode = kwargs['<STR_LIT>']<EOL>if mode not in constructors:<EOL><INDENT>raise ValueError('<STR_LIT>' % mode)<EOL><DEDENT>del kwargs['<STR_LIT>']<EOL>return constructors[mode]<EOL><DEDENT>else:<EOL><INDENT>for mode, constructor in constructors:<EOL><INDENT>if constructor._argcheck(*args, **kwargs):<EOL><INDENT>return constructor<EOL><DEDENT><DEDENT><DEDENT>return ConstructLocal<EOL>

Use arguments to route constructor.

Applies a series of checks on arguments to identify constructor,
starting with known keyword arguments, and then applying
constructor-specific checks


return lookup(*args, **kwargs).dispatch('<STR_LIT>', *args, **kwargs)<EOL>


from numpy import ones<EOL>import random<EOL>random.seed(<NUM_LIT>)<EOL>func1 = lambda x: x * <NUM_LIT:2><EOL>mapped = b.map(func1)<EOL>res = mapped.toarray()<EOL>assert allclose(res, arr * <NUM_LIT:2>)<EOL>mapped = b.map(func1, axis=<NUM_LIT:0>)<EOL>res = mapped.toarray()<EOL>assert allclose(res, arr * <NUM_LIT:2>)<EOL>mapped = b.map(func1, axis=<NUM_LIT:1>)<EOL>res = mapped.toarray()<EOL>assert allclose(res, (arr * <NUM_LIT:2>).transpose(<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>))<EOL>mapped = b.map(func1, axis=(<NUM_LIT:1>, <NUM_LIT:2>))<EOL>res = mapped.toarray()<EOL>assert allclose(res, (arr * <NUM_LIT:2>).transpose(<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:0>))<EOL>mapped = b.map(func1, axis=(<NUM_LIT:0>, <NUM_LIT:2>))<EOL>res = mapped.toarray()<EOL>assert allclose(res, (arr * <NUM_LIT:2>).transpose(<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>))<EOL>mapped = b.map(func1, axis=<NUM_LIT:2>)<EOL>res = mapped.toarray()<EOL>assert allclose(res, (arr * <NUM_LIT:2>).transpose(<NUM_LIT:2>, <NUM_LIT:0>, <NUM_LIT:1>))<EOL>func2 = lambda x: ones(<NUM_LIT:10>)<EOL>mapped = b.map(func2, axis=<NUM_LIT:0>)<EOL>res = mapped.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:0>], <NUM_LIT:10>)<EOL>mapped = b.map(func2, axis=(<NUM_LIT:0>, <NUM_LIT:1>))<EOL>res = mapped.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:0>], arr.shape[<NUM_LIT:1>], <NUM_LIT:10>)<EOL>with pytest.raises(Exception):<EOL><INDENT>def nonuniform_map(x):<EOL><INDENT>random.seed(x.tostring())<EOL>return random.random()<EOL><DEDENT>func3 = lambda x: ones(<NUM_LIT:10>) if nonuniform_map(x) < <NUM_LIT:0.5> else ones(<NUM_LIT:5>)<EOL>mapped = b.map(func3)<EOL>res = mapped.toarray()<EOL><DEDENT>if b.mode == '<STR_LIT>':<EOL><INDENT>func3 = lambda x: x.astype('<STR_LIT>')<EOL>mapped = b.map(func3, axis=<NUM_LIT:0>)<EOL>assert mapped.dtype == dtype('<STR_LIT>')<EOL>mapped = b.map(func3, axis=<NUM_LIT:0>, dtype=dtype('<STR_LIT>'))<EOL>assert mapped.dtype == dtype('<STR_LIT>')<EOL><DEDENT>

A set of tests for the map operator

Parameters
----------
arr: `ndarray`
    A 2D array used in the construction of `b` (used to check results)
b: `BoltArray`
    The BoltArray to be used for testing


from numpy import ones, sum<EOL>from operator import add<EOL>reduced = b.reduce(add, axis=<NUM_LIT:0>)<EOL>res = reduced.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:1>], arr.shape[<NUM_LIT:2>])<EOL>assert allclose(res, sum(arr, <NUM_LIT:0>))<EOL>reduced = b.reduce(add, axis=(<NUM_LIT:0>, <NUM_LIT:1>))<EOL>res = reduced.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:2>],)<EOL>assert allclose(res, sum(sum(arr, <NUM_LIT:0>), <NUM_LIT:1>))<EOL>reduced = b.reduce(add, axis=<NUM_LIT:1>)<EOL>res = reduced.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:0>], arr.shape[<NUM_LIT:2>])<EOL>assert allclose(res, sum(arr, <NUM_LIT:1>))<EOL>reduced = b.reduce(add, axis=(<NUM_LIT:1>, <NUM_LIT:2>))<EOL>res = reduced.toarray()<EOL>assert res.shape == (arr.shape[<NUM_LIT:0>],)<EOL>assert allclose(res, sum(sum(arr, <NUM_LIT:1>), <NUM_LIT:1>))<EOL>

A set of tests for the reduce operator

Parameters
----------
arr: `ndarray`
    A 3D ndarray used in the construction of `b` (used to check results)
b: `BoltArray`
    The BoltArray to be used for testing