return self._impl.new_cells_from_excel(<EOL>book,<EOL>range_,<EOL>sheet,<EOL>names_row,<EOL>param_cols,<EOL>param_order,<EOL>transpose,<EOL>names_col,<EOL>param_rows,<EOL>)<EOL>

Create multiple cells from an Excel range.

        This method reads values from a range in an Excel file,
        create cells and populate them with the values in the range.
        To use this method, ``openpyxl`` package must be installed.

        The Excel file to read data from is specified by ``book``
        parameters. The ``range_`` can be a range address, such as "$G4:$K10",
        or a named range. In case a range address is given,
        ``sheet`` must also be given.

        By default, cells data are interpreted as being laid out side-by-side.
        ``names_row`` is a row index (starting from 0) to specify the
        row that contains the names of cells and parameters.
        Cells and parameter names must be contained in a single row.
        ``param_cols`` accepts a sequence (such as list or tuple) of
        column indexes (starting from 0) that indicate columns that
        contain cells arguments.

        **2-dimensional cells definitions**

        The optional ``names_col`` and ``param_rows`` parameters are used,
        when data for one cells spans more than one column.
        In such cases, the cells data is 2-dimensional, and
        there must be parameter row(s) across the columns
        that contain arguments of the parameters.
        A sequence of row indexes that indicate parameter rows
        is passed to ``param_rows``.
        The names of those parameters must be contained in the
        same rows as parameter values (arguments), and
        ``names_col`` is to indicate the column position at which
        the parameter names are defined.

        **Horizontal arrangement**

        By default, cells data are interpreted as being placed
        side-by-side, regardless of whether one cells corresponds
        to a single column or multiple columns.
        ``transpose`` parameter is used to alter this orientation,
        and if it is set to ``True``, cells values are
        interpreted as being placed one above the other.
        "row(s)" and "col(s)" in the parameter
        names are interpreted inversely, i.e.
        all indexes passed to "row(s)" parameters are interpreted
        as column indexes,
        and all indexes passed to "col(s)" parameters as row indexes.

        Args:
            book (str): Path to an Excel file.
            range_ (str): Range expression, such as "A1", "$G4:$K10",
                or named range "NamedRange1".
            sheet (str): Sheet name (case ignored).
            names_row (optional): an index number indicating
                what row contains the names of cells and parameters.
                Defaults to the top row (0).
            param_cols (optional): a sequence of index numbers
                indicating parameter columns.
                Defaults to only the leftmost column ([0]).
            names_col (optional): an index number, starting from 0,
                indicating what column contains additional parameters.
            param_rows (optional): a sequence of index numbers, starting from
                0, indicating rows of additional parameters, in case cells are
                defined in two dimensions.
            transpose (optional): Defaults to ``False``.
                If set to ``True``, "row(s)" and "col(s)" in the parameter
                names are interpreted inversely, i.e.
                all indexes passed to "row(s)" parameters are interpreted
                as column indexes,
                and all indexes passed to "col(s)" parameters as row indexes.
            param_order (optional): a sequence to reorder the parameters.
                The elements of the sequence are the indexes of ``param_cols``
                elements, and optionally the index of ``param_rows`` elements
                shifted by the length of ``param_cols``.


if isinstance(item, str):<EOL><INDENT>return item in self._impl.namespace<EOL><DEDENT>elif isinstance(item, Cells):<EOL><INDENT>return item._impl in self._impl.cells.values()<EOL><DEDENT>elif isinstance(item, StaticSpace):<EOL><INDENT>return item._impl in self._impl.spaces.values()<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if item is in the space.

        item can be either a cells or space.

        Args:
            item: a cells or space to check.

        Returns:
            True if item is a direct child of the space, False otherwise.


return list(self.model.spacegraph.predecessors(self))<EOL>

Return an iterator over direct base spaces


parts = name.split("<STR_LIT:.>")<EOL>child = parts.pop(<NUM_LIT:0>)<EOL>if parts:<EOL><INDENT>return self.spaces[child].get_object("<STR_LIT:.>".join(parts))<EOL><DEDENT>else:<EOL><INDENT>return self._namespace_impl[child]<EOL><DEDENT>

Retrieve an object by a dotted name relative to the space.


bases = tuple(<EOL>base.bases[<NUM_LIT:0>] if base.is_dynamic() else base for base in bases_<EOL>)<EOL>if len(bases) == <NUM_LIT:1>:<EOL><INDENT>return bases[<NUM_LIT:0>]<EOL><DEDENT>elif len(bases) > <NUM_LIT:1>:<EOL><INDENT>return self.model.get_dynamic_base(bases)<EOL><DEDENT>else:<EOL><INDENT>RuntimeError("<STR_LIT>")<EOL><DEDENT>

Create or get the base space from a list of spaces

        if a direct base space in `bases` is dynamic, replace it with
        its base.


if name is None:<EOL><INDENT>name = self.spacenamer.get_next(self.namespace)<EOL><DEDENT>if name in self.namespace:<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>if not is_valid_name(name):<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>space = RootDynamicSpaceImpl(<EOL>parent=self,<EOL>name=name,<EOL>formula=formula,<EOL>refs=refs,<EOL>source=source,<EOL>arguments=arguments,<EOL>)<EOL>space.is_derived = False<EOL>self._set_space(space)<EOL>if bases:  <EOL><INDENT>dynbase = self._get_dynamic_base(bases)<EOL>space._dynbase = dynbase<EOL>dynbase._dynamic_subs.append(space)<EOL><DEDENT>return space<EOL>


def get_dynspace(self, args, kwargs=None):

node = get_node(self, *convert_args(args, kwargs))<EOL>key = node[KEY]<EOL>if key in self.param_spaces:<EOL><INDENT>return self.param_spaces[key]<EOL><DEDENT>else:<EOL><INDENT>last_self = self.system.self<EOL>self.system.self = self<EOL>try:<EOL><INDENT>space_args = self.eval_formula(node)<EOL><DEDENT>finally:<EOL><INDENT>self.system.self = last_self<EOL><DEDENT>if space_args is None:<EOL><INDENT>space_args = {"<STR_LIT>": [self]}  <EOL><DEDENT>else:<EOL><INDENT>if "<STR_LIT>" in space_args:<EOL><INDENT>bases = get_impls(space_args["<STR_LIT>"])<EOL>if isinstance(bases, StaticSpaceImpl):<EOL><INDENT>space_args["<STR_LIT>"] = [bases]<EOL><DEDENT>elif bases is None:<EOL><INDENT>space_args["<STR_LIT>"] = [self]  <EOL><DEDENT>else:<EOL><INDENT>space_args["<STR_LIT>"] = bases<EOL><DEDENT><DEDENT>else:<EOL><INDENT>space_args["<STR_LIT>"] = [self]<EOL><DEDENT><DEDENT>space_args["<STR_LIT>"] = node_get_args(node)<EOL>space = self._new_dynspace(**space_args)<EOL>self.param_spaces[key] = space<EOL>space.inherit(clear_value=False)<EOL>return space<EOL><DEDENT>

Create a dynamic root space

        Called from interface methods


super().restore_state(system)<EOL>BaseSpaceContainerImpl.restore_state(self, system)<EOL>for cells in self._cells.values():<EOL><INDENT>cells.restore_state(system)<EOL><DEDENT>

Called after unpickling to restore some attributes manually.


import modelx.io.excel as xl<EOL>cellstable = xl.CellsTable(<EOL>book,<EOL>range_,<EOL>sheet,<EOL>names_row,<EOL>param_cols,<EOL>param_order,<EOL>transpose,<EOL>names_col,<EOL>param_rows,<EOL>)<EOL>if cellstable.param_names:<EOL><INDENT>sig = "<STR_LIT>".join(cellstable.param_names) + "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>sig = "<STR_LIT>"<EOL><DEDENT>blank_func = "<STR_LIT>" + sig + "<STR_LIT>"<EOL>for cellsdata in cellstable.items():<EOL><INDENT>cells = self.new_cells(name=cellsdata.name, formula=blank_func)<EOL>for args, value in cellsdata.items():<EOL><INDENT>cells.set_value(args, value)<EOL><DEDENT><DEDENT>

Create multiple cells from an Excel range.

        Args:
            book (str): Path to an Excel file.
            range_ (str): Range expression, such as "A1", "$G4:$K10",
                or named range "NamedRange1".
            sheet (str): Sheet name (case ignored).
            names_row: Cells names in a sequence, or an integer number, or
              a string expression indicating row (or column depending on
              ```orientation```) to read cells names from.
            param_cols: a sequence of them
                indicating parameter columns (or rows depending on ```
                orientation```)
            param_order: a sequence of integers representing
                the order of params and extra_params.
            transpose: in which direction 'vertical' or 'horizontal'
            names_col: a string or a list of names of the extra params.
            param_rows: integer or string expression, or a sequence of them
                indicating row (or column) to be interpreted as parameters.


if not is_valid_name(name):<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>if name in self.namespace:<EOL><INDENT>if name in self.refs:<EOL><INDENT>if name in self.self_refs:<EOL><INDENT>self.new_ref(name, value)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % name)<EOL><DEDENT><DEDENT>elif name in self.cells:<EOL><INDENT>if self.cells[name].is_scalar():<EOL><INDENT>self.cells[name].set_value((), value)<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError("<STR_LIT>" % name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.new_ref(name, value)<EOL><DEDENT>

Implementation of attribute setting

        ``space.name = value`` by user script
        Called from ``Space.__setattr__``


if name in self.namespace:<EOL><INDENT>if name in self.cells:<EOL><INDENT>self.del_cells(name)<EOL><DEDENT>elif name in self.spaces:<EOL><INDENT>self.del_space(name)<EOL><DEDENT>elif name in self.refs:<EOL><INDENT>self.del_ref(name)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % (name, self.name))<EOL><DEDENT>

Implementation of attribute deletion

        ``del space.name`` by user script
        Called from ``StaticSpace.__delattr__``


if name not in self.spaces:<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>if name in self.static_spaces:<EOL><INDENT>space = self.static_spaces[name]<EOL>if space.is_derived:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>" % repr(space.interface)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>self.static_spaces.del_item(name)<EOL>self.model.spacegraph.remove_node(space)<EOL>self.inherit()<EOL>self.model.spacegraph.update_subspaces(self)<EOL><DEDENT><DEDENT>elif name in self.dynamic_spaces:<EOL><INDENT>self.dynamic_spaces.del_item(name)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>


if name in self.cells:<EOL><INDENT>cells = self.cells[name]<EOL>self.cells.del_item(name)<EOL>self.inherit()<EOL>self.model.spacegraph.update_subspaces(self)<EOL><DEDENT>elif name in self.dynamic_spaces:<EOL><INDENT>cells = self.dynamic_spaces.pop(name)<EOL>self.dynamic_spaces.set_update()<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % name)<EOL><DEDENT>NullImpl(cells)<EOL>

Implementation of cells deletion

        ``del space.name`` where name is a cells, or
        ``del space.cells['name']``


args = [repr(arg) for arg in get_interfaces(self.argvalues)]<EOL>param = "<STR_LIT:U+002CU+0020>".join(args)<EOL>return "<STR_LIT>" % (self.parent.evalrepr, param)<EOL>


<EOL>strio = io.StringIO(source)<EOL>gen = tokenize.generate_tokens(strio.readline)<EOL>tkns = []<EOL>try:<EOL><INDENT>for t in gen:<EOL><INDENT>tkns.append(t)<EOL><DEDENT><DEDENT>except tokenize.TokenError:<EOL><INDENT>pass<EOL><DEDENT>lambda_pos = [(t.type, t.string) for t in tkns].index(<EOL>(tokenize.NAME, "<STR_LIT>")<EOL>)<EOL>tkns = tkns[lambda_pos:]<EOL>lastop_pos = (<EOL>len(tkns) - <NUM_LIT:1> - [t.type for t in tkns[::-<NUM_LIT:1>]].index(tokenize.OP)<EOL>)<EOL>lastop = tkns[lastop_pos]<EOL>fiedlineno = lastop.start[<NUM_LIT:0>]<EOL>fixedline = lastop.line[: lastop.start[<NUM_LIT:1>]] + lastop.line[lastop.end[<NUM_LIT:1>] :]<EOL>tkns = tkns[:lastop_pos]<EOL>fixedlines = "<STR_LIT>"<EOL>last_lineno = <NUM_LIT:0><EOL>for t in tkns:<EOL><INDENT>if last_lineno == t.start[<NUM_LIT:0>]:<EOL><INDENT>continue<EOL><DEDENT>elif t.start[<NUM_LIT:0>] == fiedlineno:<EOL><INDENT>fixedlines += fixedline<EOL>last_lineno = t.start[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>fixedlines += t.line<EOL>last_lineno = t.start[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>return fixedlines<EOL>

Remove the last redundant token from lambda expression

    lambda x: return x)
                      ^
    Return string without irrelevant tokens
    returned from inspect.getsource on lamda expr returns


try:<EOL><INDENT>module_node = compile(<EOL>source, "<STR_LIT>", mode="<STR_LIT>", flags=ast.PyCF_ONLY_AST<EOL>)<EOL><DEDENT>except SyntaxError:<EOL><INDENT>return find_funcdef(fix_lamdaline(source))<EOL><DEDENT>for node in ast.walk(module_node):<EOL><INDENT>if isinstance(node, ast.FunctionDef) or isinstance(node, ast.Lambda):<EOL><INDENT>return node<EOL><DEDENT><DEDENT>raise ValueError("<STR_LIT>")<EOL>

Find the first FuncDef ast object in source


funcdef = find_funcdef(source)<EOL>params = []<EOL>for node in ast.walk(funcdef.args):<EOL><INDENT>if isinstance(node, ast.arg):<EOL><INDENT>if node.arg not in params:<EOL><INDENT>params.append(node.arg)<EOL><DEDENT><DEDENT><DEDENT>return params<EOL>

Extract parameters from a function definition


if source is None:<EOL><INDENT>return None<EOL><DEDENT>source = dedent(source)<EOL>funcdef = find_funcdef(source)<EOL>params = extract_params(source)<EOL>names = []<EOL>if isinstance(funcdef, ast.FunctionDef):<EOL><INDENT>stmts = funcdef.body<EOL><DEDENT>elif isinstance(funcdef, ast.Lambda):<EOL><INDENT>stmts = [funcdef.body]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>for stmt in stmts:<EOL><INDENT>for node in ast.walk(stmt):<EOL><INDENT>if isinstance(node, ast.Name):<EOL><INDENT>if node.id not in names and node.id not in params:<EOL><INDENT>names.append(node.id)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return names<EOL>

Extract names from a function definition

    Looks for a function definition in the source.
    Only the first function definition is examined.

    Returns:
         a list names(identifiers) used in the body of the function
         excluding function parameters.


module_node = ast.parse(dedent(src))<EOL>if len(module_node.body) == <NUM_LIT:1> and isinstance(<EOL>module_node.body[<NUM_LIT:0>], ast.FunctionDef<EOL>):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>


lines = source.splitlines()<EOL>atok = asttokens.ASTTokens(source, parse=True)<EOL>for node in ast.walk(atok.tree):<EOL><INDENT>if isinstance(node, ast.FunctionDef):<EOL><INDENT>break<EOL><DEDENT><DEDENT>if node.decorator_list:<EOL><INDENT>deco_first = node.decorator_list[<NUM_LIT:0>]<EOL>deco_last = node.decorator_list[-<NUM_LIT:1>]<EOL>line_first = atok.tokens[deco_first.first_token.index - <NUM_LIT:1>].start[<NUM_LIT:0>]<EOL>line_last = atok.tokens[deco_last.last_token.index + <NUM_LIT:1>].start[<NUM_LIT:0>]<EOL>lines = lines[:line_first - <NUM_LIT:1>] + lines[line_last:]<EOL><DEDENT>return "<STR_LIT:\n>".join(lines) + "<STR_LIT:\n>"<EOL>

Remove decorators from function definition


def replace_funcname(source: str, name: str):

lines = source.splitlines()<EOL>atok = asttokens.ASTTokens(source, parse=True)<EOL>for node in ast.walk(atok.tree):<EOL><INDENT>if isinstance(node, ast.FunctionDef):<EOL><INDENT>break<EOL><DEDENT><DEDENT>i = node.first_token.index<EOL>for i in range(node.first_token.index, node.last_token.index):<EOL><INDENT>if (atok.tokens[i].type == token.NAME<EOL>and atok.tokens[i].string == "<STR_LIT>"):<EOL><INDENT>break<EOL><DEDENT><DEDENT>lineno, col_begin = atok.tokens[i + <NUM_LIT:1>].start<EOL>lineno_end, col_end = atok.tokens[i + <NUM_LIT:1>].end<EOL>assert lineno == lineno_end<EOL>lines[lineno-<NUM_LIT:1>] = (<EOL>lines[lineno-<NUM_LIT:1>][:col_begin] + name + lines[lineno-<NUM_LIT:1>][col_end:]<EOL>)<EOL>return "<STR_LIT:\n>".join(lines) + "<STR_LIT:\n>"<EOL>


module_node = ast.parse(dedent(src))<EOL>lambdaexp = [node for node in ast.walk(module_node)<EOL>if isinstance(node, ast.Lambda)]<EOL>return bool(lambdaexp)<EOL>

True if only one lambda expression is included


return {"<STR_LIT:source>": self.source, "<STR_LIT>": self.module}<EOL>


if self.module is None:<EOL><INDENT>raise RuntimeError<EOL><DEDENT>elif module is None:<EOL><INDENT>import importlib<EOL>module = ModuleSource(importlib.reload(module))<EOL><DEDENT>elif module.name != self.module:<EOL><INDENT>raise RuntimeError<EOL><DEDENT>if self.name in module.funcs:<EOL><INDENT>func = module.funcs[self.name]<EOL>self.__init__(func=func)<EOL><DEDENT>else:<EOL><INDENT>self.__init__(func=NULL_FORMULA)<EOL><DEDENT>return self<EOL>

Reload the source function from the source module.

        **Internal use only**
        Update the source function of the formula.
        This method is used to updated the underlying formula
        when the source code of the module in which the source function
        is read from is modified.

        If the formula was not created from a module, an error is raised.
        If ``module_`` is not given, the source module of the formula is
        reloaded. If ``module_`` is given and matches the source module,
        then the module_ is used without being reloaded.
        If ``module_`` is given and does not match the source module of
        the formula, an error is raised.

        Args:
            module_: A ``ModuleSource`` object

        Returns:
            self


def clear_descendants(self, source, clear_source=True):

desc = nx.descendants(self, source)<EOL>if clear_source:<EOL><INDENT>desc.add(source)<EOL><DEDENT>self.remove_nodes_from(desc)<EOL>return desc<EOL>

Remove all descendants of(reachable from) `source`.

        Args:
            source: Node descendants
            clear_source(bool): Remove origin too if True.
        Returns:
            set: The removed nodes.


obj_nodes = self.get_nodes_with(obj)<EOL>removed = set()<EOL>for node in obj_nodes:<EOL><INDENT>if self.has_node(node):<EOL><INDENT>removed.update(self.clear_descendants(node))<EOL><DEDENT><DEDENT>return removed<EOL>

Remove all nodes with `obj` and their descendants.


result = set()<EOL>if nx.__version__[<NUM_LIT:0>] == "<STR_LIT:1>":<EOL><INDENT>nodes = self.nodes_iter()<EOL><DEDENT>else:<EOL><INDENT>nodes = self.nodes<EOL><DEDENT>for node in nodes:<EOL><INDENT>if node[OBJ] == obj:<EOL><INDENT>result.add(node)<EOL><DEDENT><DEDENT>return result<EOL>


return DependencyGraph()<EOL>


if nx.__version__[<NUM_LIT:0>] == "<STR_LIT:1>":<EOL><INDENT>return super().add_path(nodes, **attr)<EOL><DEDENT>else:<EOL><INDENT>return nx.add_path(self, nodes, **attr)<EOL><DEDENT>

In replacement for Deprecated add_path method


self._impl.system.rename_model(new_name=name, old_name=self.name)<EOL>


return self._impl.cellgraph<EOL>


return self._impl.global_refs.interfaces<EOL>


if is_valid_name(name):<EOL><INDENT>if name not in self.system.models:<EOL><INDENT>self.name = name<EOL>return True  <EOL><DEDENT>else:  <EOL><INDENT>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>

Rename self. Must be called only by its system.


removed = self.cellgraph.clear_descendants(source, clear_source)<EOL>for node in removed:<EOL><INDENT>del node[OBJ].data[node[KEY]]<EOL><DEDENT>

Clear values and nodes calculated from `source`.


removed = self.cellgraph.clear_obj(obj)<EOL>for node in removed:<EOL><INDENT>del node[OBJ].data[node[KEY]]<EOL><DEDENT>

Clear values and nodes of `obj` and their dependants.


parts = name.split("<STR_LIT:.>")<EOL>space = self.spaces[parts.pop(<NUM_LIT:0>)]<EOL>if parts:<EOL><INDENT>return space.get_object("<STR_LIT:.>".join(parts))<EOL><DEDENT>else:<EOL><INDENT>return space<EOL><DEDENT>

Retrieve an object by a dotted name relative to the model.


Impl.restore_state(self, system)<EOL>BaseSpaceContainerImpl.restore_state(self, system)<EOL>mapping = {}<EOL>for node in self.cellgraph:<EOL><INDENT>if isinstance(node, tuple):<EOL><INDENT>name, key = node<EOL><DEDENT>else:<EOL><INDENT>name, key = node, None<EOL><DEDENT>cells = self.get_object(name)<EOL>mapping[node] = get_node(cells, key, None)<EOL><DEDENT>self.cellgraph = nx.relabel_nodes(self.cellgraph, mapping)<EOL>

Called after unpickling to restore some attributes manually.


def get_dynamic_base(self, bases: tuple):

try:<EOL><INDENT>return self._dynamic_bases_inverse[bases]<EOL><DEDENT>except KeyError:<EOL><INDENT>name = self._dynamic_base_namer.get_next(self._dynamic_bases)<EOL>base = self._new_space(name=name)<EOL>self.spacegraph.add_space(base)<EOL>self._dynamic_bases[name] = base<EOL>self._dynamic_bases_inverse[bases] = base<EOL>base.add_bases(bases)<EOL>return base<EOL><DEDENT>

Create of get a base space for a tuple of bases


return self.predecessors(node)<EOL>


try:<EOL><INDENT>self.add_node("<STR_LIT>")<EOL>for base in bases:<EOL><INDENT>nx.DiGraph.add_edge(self, base, "<STR_LIT>")<EOL><DEDENT>result = self.get_mro("<STR_LIT>")[<NUM_LIT:1>:]<EOL><DEDENT>finally:<EOL><INDENT>self.remove_node("<STR_LIT>")<EOL><DEDENT>return result<EOL>

Check if C3 MRO is possible with given bases


seqs = [self.get_mro(base) for base in self.get_bases(space)] + [<EOL>list(self.get_bases(space))<EOL>]<EOL>res = []<EOL>while True:<EOL><INDENT>non_empty = list(filter(None, seqs))<EOL>if not non_empty:<EOL><INDENT>res.insert(<NUM_LIT:0>, space)<EOL>return res<EOL><DEDENT>for seq in non_empty:  <EOL><INDENT>candidate = seq[<NUM_LIT:0>]<EOL>not_head = [s for s in non_empty if candidate in s[<NUM_LIT:1>:]]<EOL>if not_head:<EOL><INDENT>candidate = None<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not candidate:  <EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>res.append(candidate)<EOL>for seq in non_empty:<EOL><INDENT>if seq[<NUM_LIT:0>] == candidate:<EOL><INDENT>del seq[<NUM_LIT:0>]<EOL><DEDENT><DEDENT><DEDENT>

Calculate the Method Resolution Order of bases using the C3 algorithm.

        Code modified from
        http://code.activestate.com/recipes/577748-calculate-the-mro-of-a-class/

        Args:
            bases: sequence of direct base spaces.

        Returns:
            mro as a list of bases including node itself


def custom_showwarning(<EOL>message, category, filename="<STR_LIT>", lineno=-<NUM_LIT:1>, file=None, line=None<EOL>):

if file is None:<EOL><INDENT>file = sys.stderr<EOL>if file is None:<EOL><INDENT>return<EOL><DEDENT><DEDENT>text = "<STR_LIT>" % (category.__name__, message)<EOL>try:<EOL><INDENT>file.write(text)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>

Hook to override default showwarning.

    https://stackoverflow.com/questions/2187269/python-print-only-the-message-on-warnings


try:<EOL><INDENT>__IPYTHON__<EOL>return True<EOL><DEDENT>except NameError:<EOL><INDENT>return False<EOL><DEDENT>

True if the current shell is an IPython shell.

    Note __IPYTHON__ is not yet set before IPython kernel is initialized.

    https://stackoverflow.com/questions/5376837/how-can-i-do-an-if-run-from-ipython-test-in-python


self.default_showtraceback(<EOL>exc_tuple,<EOL>filename,<EOL>tb_offset,<EOL>exception_only=True,<EOL>running_compiled_code=running_compiled_code,<EOL>)<EOL>

Custom showtraceback for monkey-patching IPython's InteractiveShell

    https://stackoverflow.com/questions/1261668/cannot-override-sys-excepthook


def excepthook(self, except_type, exception, traceback):

if except_type is DeepReferenceError:<EOL><INDENT>print(exception.msg)<EOL><DEDENT>else:<EOL><INDENT>self.default_excepthook(except_type, exception, traceback)<EOL><DEDENT>

Not Used: Custom exception hook to replace sys.excepthook

    This is for CPython's default shell. IPython does not use sys.exepthook.

    https://stackoverflow.com/questions/27674602/hide-traceback-unless-a-debug-flag-is-set


result = "<STR_LIT>"<EOL>for i, value in enumerate(self):<EOL><INDENT>result += "<STR_LIT>".format(i, get_node_repr(value))<EOL><DEDENT>result = result.strip("<STR_LIT:\n>")<EOL>lines = result.split("<STR_LIT:\n>")<EOL>if maxlen and len(lines) > maxlen:<EOL><INDENT>i = int(maxlen / <NUM_LIT:2>)<EOL>lines = lines[:i] + ["<STR_LIT>"] + lines[-(maxlen - i) :]<EOL>result = "<STR_LIT:\n>".join(lines)<EOL><DEDENT>return result<EOL>

if maxlen > 0, the message is shortened to maxlen traces.


if self.is_ipysetup:<EOL><INDENT>return<EOL><DEDENT>from ipykernel.kernelapp import IPKernelApp<EOL>self.shell = IPKernelApp.instance().shell  <EOL>if not self.shell and is_ipython():<EOL><INDENT>self.shell = get_ipython()<EOL><DEDENT>if self.shell:<EOL><INDENT>shell_class = type(self.shell)<EOL>shell_class.default_showtraceback = shell_class.showtraceback<EOL>shell_class.showtraceback = custom_showtraceback<EOL>self.is_ipysetup = True<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>

Monkey patch shell's error handler.

        This method is to monkey-patch the showtraceback method of
        IPython's InteractiveShell to

        __IPYTHON__ is not detected when starting an IPython kernel,
        so this method is called from start_kernel in spyder-modelx.


if not self.is_ipysetup:<EOL><INDENT>return<EOL><DEDENT>shell_class = type(self.shell)<EOL>shell_class.showtraceback = shell_class.default_showtraceback<EOL>del shell_class.default_showtraceback<EOL>self.is_ipysetup = False<EOL>


sys.setrecursionlimit(<NUM_LIT:10>**<NUM_LIT:6>)<EOL>warnings.showwarning = custom_showwarning<EOL>threading.stack_size(<NUM_LIT>)<EOL>

Configure Python settings for modelx

        The error handler is configured later.


orig = self.orig_settings<EOL>sys.setrecursionlimit(orig["<STR_LIT>"])<EOL>if "<STR_LIT>" in orig:<EOL><INDENT>sys.tracebacklimit = orig["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>if hasattr(sys, "<STR_LIT>"):<EOL><INDENT>del sys.tracebacklimit<EOL><DEDENT><DEDENT>if "<STR_LIT>" in orig:<EOL><INDENT>warnings.showwarning = orig["<STR_LIT>"]<EOL><DEDENT>orig.clear()<EOL>threading.stack_size()<EOL>

Restore Python settings to the original states


parts = name.split("<STR_LIT:.>")<EOL>model_name = parts.pop(<NUM_LIT:0>)<EOL>return self.models[model_name].get_object("<STR_LIT:.>".join(parts))<EOL>

Retrieve an object by its absolute name.


found = False<EOL>for arg in args:<EOL><INDENT>if isinstance(arg, Cells):<EOL><INDENT>found = True<EOL>break<EOL><DEDENT><DEDENT>if found:<EOL><INDENT>args = tuple(<EOL>arg.value if isinstance(arg, Cells) else arg for arg in args<EOL>)<EOL><DEDENT>if kwargs is not None:<EOL><INDENT>for key, arg in kwargs.items():<EOL><INDENT>if isinstance(arg, Cells):<EOL><INDENT>kwargs[key] = arg.value<EOL><DEDENT><DEDENT><DEDENT>return args, kwargs<EOL>

If args and kwargs contains Cells, Convert them to their values.


result = []<EOL>for c in cells.values():<EOL><INDENT>params = c.formula.parameters<EOL>for i in range(min(len(result), len(params))):<EOL><INDENT>if params[i] != result[i]:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>for i in range(len(result), len(params)):<EOL><INDENT>result.append(params[i])<EOL><DEDENT><DEDENT>return result<EOL>

Return parameter names if the parameters are shareable among cells.

    Parameters are shareable among multiple cells when all the cells
    have the parameters in the same order if they ever have any.

    For example, if cells are foo(), bar(x), baz(x, y), then
    ('x', 'y') are shareable parameters amounts them, as 'x' and 'y'
    appear in the same order in the parameter list if they ever appear.

    Args:
        cells: An iterator yielding cells.

    Returns:
        None if parameters are not share,
        tuple of shareable parameter names,
        () if cells are all scalars.


return self._impl.find_match(args, kwargs)<EOL>

Returns the best matching args and their value.

        If the cells returns None for the given arguments,
        continue to get a value by passing arguments
        masking the given arguments with Nones.
        The search of non-None value starts from the given arguments
        to the all None arguments in the lexicographical order.
        The masked arguments that returns non-None value
        first is returned with the value.


self._impl.set_value(tuplize_key(self, key), value)<EOL>


return Cells(space=space, name=name, formula=self.formula)<EOL>


return self._impl.clear_value(*args, **kwargs)<EOL>


return self._impl.single_value != <NUM_LIT:0><EOL>

True if self != 0. Called for bool(self).


return self._impl.single_value + other<EOL>


return self.__add__(other)<EOL>


return -self._impl.single_value<EOL>


return +self._impl.single_value<EOL>


return self + -other<EOL>


return -self + other<EOL>


return self._impl.single_value * other<EOL>


return self.__mul__(other)<EOL>


return self._impl.single_value / other<EOL>

self / other: Should promote to float when necessary.


return other / self._impl.single_value<EOL>


return self._impl.single_value ** exponent<EOL>

self ** exponent
        should promote to float or complex when necessary.


return base ** self._impl.single_value<EOL>


raise NotImplementedError<EOL>

Returns the Real distance from 0. Called for abs(self).


if self._impl.is_scalar():<EOL><INDENT>return self._impl.single_value == other<EOL><DEDENT>elif isinstance(other, Cells):<EOL><INDENT>return self is other<EOL><DEDENT>else:<EOL><INDENT>raise TypeError<EOL><DEDENT>


return self._impl.single_value < other<EOL>


return self.__eq__(other) or self.__lt__(other)<EOL>


return self._impl.single_value > other<EOL>


return self.__eq__(other) or self.__gt__(other)<EOL>


return self._impl.to_series(args)<EOL>

Convert the cells itself into a Pandas Series and return it.


return self._impl.to_series(())<EOL>


return self._impl.to_frame(args)<EOL>

Convert the cells itself into a Pandas DataFrame and return it.

        if no `args` are passed, the returned DataFrame contains as many
        values as the cells have.

        if A sequence of arguments to the cells is passed as `args`,
        the returned DataFrame contains values only for the specified `args`.

        Args:
            args: A sequence or iterable of arguments to the cells.

        Returns:
            a DataFrame with a column named after the cells,
            with indexes named after the parameters of the cells.


return self._impl.to_frame(())<EOL>


return self._impl.formula<EOL>


return self._impl.formula.parameters<EOL>


Set formula from a function.
        Deprecated since version 0.0.5. Use formula property instead.


Clear the formula.
        Deprecated since version 0.0.5. Use formula property instead.


return self._impl.single_value<EOL>

Get, set, delete the scalar value.
        The cells must be a scalar cells.


return CellNode(get_node(self._impl, *convert_args(args, kwargs)))<EOL>

Return a :class:`CellNode` object for the given arguments.


return self._impl.predecessors(args, kwargs)<EOL>

Return a list of predecessors of a cell.

        This method returns a list of CellNode objects, whose elements are
        predecessors of (i.e. referenced in the formula
        of) the cell specified by the given arguments.


return self._impl.successors(args, kwargs)<EOL>

Return a list of successors of a cell.

        This method returns a list of CellNode objects, whose elements are
        successors of (i.e. referencing in their formulas)
        the cell specified by the given arguments.


result = super()._baseattrs<EOL>result["<STR_LIT>"] = "<STR_LIT:U+002CU+0020>".join(self.parameters)<EOL>return result<EOL>


return self._impl[OBJ].interface<EOL>


return self._impl[KEY]<EOL>


return self._impl[OBJ].has_cell(self._impl[KEY])<EOL>

Return ``True`` if the cell has a value.


if self.has_value:<EOL><INDENT>return self._impl[OBJ].get_value(self._impl[KEY])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>


return self.cells.preds(*self.args)<EOL>

A list of nodes that this node refers to.


return self.cells.succs(*self.args)<EOL>

A list of nodes that refer to this  node.


result = {<EOL>"<STR_LIT:type>": type(self).__name__,<EOL>"<STR_LIT>": self.cells._baseattrs,<EOL>"<STR_LIT:args>": self.args,<EOL>"<STR_LIT:value>": self.value if self.has_value else None,<EOL>"<STR_LIT>": len(self.preds),<EOL>"<STR_LIT>": len(self.succs),<EOL>"<STR_LIT>": self.cells._impl.repr_parent(),<EOL>"<STR_LIT>": self.cells._get_repr(),<EOL>}<EOL>return result<EOL>


if impls is None:<EOL><INDENT>return None<EOL><DEDENT>elif isinstance(impls, OrderMixin):<EOL><INDENT>result = OrderedDict()<EOL>for name in impls.order:<EOL><INDENT>result[name] = impls[name].interface<EOL><DEDENT>return result<EOL><DEDENT>elif isinstance(impls, Mapping):<EOL><INDENT>return {name: impls[name].interface for name in impls}<EOL><DEDENT>elif isinstance(impls, Sequence):<EOL><INDENT>return [impl.interface for impl in impls]<EOL><DEDENT>else:<EOL><INDENT>return impls.interface<EOL><DEDENT>

Get interfaces from their implementations.


if interfaces is None:<EOL><INDENT>return None<EOL><DEDENT>elif isinstance(interfaces, Mapping):<EOL><INDENT>return {name: interfaces[name]._impl for name in interfaces}<EOL><DEDENT>elif isinstance(interfaces, Sequence):<EOL><INDENT>return [interfaces._impl for interfaces in interfaces]<EOL><DEDENT>else:<EOL><INDENT>return interfaces._impl<EOL><DEDENT>


if self.lazy_evals is None:<EOL><INDENT>return<EOL><DEDENT>elif isinstance(self.lazy_evals, LazyEval):<EOL><INDENT>self.lazy_evals.get_updated()<EOL><DEDENT>else:<EOL><INDENT>for lz in self.lazy_evals:<EOL><INDENT>lz.get_updated()<EOL><DEDENT><DEDENT>

Update all LazyEvals in self

        self.lzy_evals must be set to LazyEval object(s) enough to
        update all owned LazyEval objects.


if self.is_model():<EOL><INDENT>return self.get_fullname()<EOL><DEDENT>else:<EOL><INDENT>return self.parent.evalrepr + "<STR_LIT:.>" + self.name<EOL><DEDENT>


self.system = system<EOL>

Called after unpickling to restore some attributes manually.


return self._impl.name<EOL>


return self._impl.get_fullname()<EOL>

Dotted name of the object.

        Names joined by dots, such as 'Model1.Space1.Cells1',
        each element in the string is the name of the parent object
        of the next one joined by a dot.


if self._impl.parent is None:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return self._impl.parent.interface<EOL><DEDENT>

The parent of this object. None for models.

        The parent object of a cells is a space that contains the cells.
        The parent object of a space is either a model or another space
        that contains the space.


return self._impl.model.interface<EOL>

The model this object belongs to.

        This is a property of Model, Space and Cells.
        For models, this property is themselves.


return self._impl.allow_none<EOL>

Whether a cells can have None as its value.

        This is a property of Model, Space and Cells.
        If ``allow_none`` of a cells is False,
        the cells cannot have None as its value.
        Assigning None to the cells
        or its formula returning None raises an Error.
        If True, the cells can have None as their value.
        If set to None, ``allow_none`` of its parent is looked up,
        and the search continues until True or False is found.

        Returns:
            True if the cells can have None, False if it cannot,
            or None if a default value from the parent is to be used.


result = {<EOL>"<STR_LIT:type>": type(self).__name__,<EOL>"<STR_LIT:id>": id(self),<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.fullname,<EOL>"<STR_LIT>": self._get_repr(),<EOL>}<EOL>return result<EOL>


result = self._baseattrs<EOL>for attr in attrs:<EOL><INDENT>if hasattr(self, attr):<EOL><INDENT>result[attr] = getattr(self, attr)._to_attrdict(attrs)<EOL><DEDENT><DEDENT>return result<EOL>


self.get_updated()<EOL>return self.data<EOL>


result = {"<STR_LIT:type>": type(self).__name__}<EOL>try:<EOL><INDENT>result["<STR_LIT>"] = {<EOL>name: item._baseattrs<EOL>for name, item in self.items()<EOL>if name[<NUM_LIT:0>] != "<STR_LIT:_>"<EOL>}<EOL><DEDENT>except:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % self)<EOL><DEDENT>return result<EOL>


LazyEval.__init__(self, [])<EOL>self.owner = owner<EOL>self.namespace_impl = owner._namespace_impl  <EOL>self.observe(self.namespace_impl)<EOL>self.altfunc = None<EOL>self.set_update()<EOL>

Create altered function from owner's formula.

        owner is a StaticSpaceImpl or CellsImpl, which has formula, and
        namespace_impl as its members.


func = self.owner.formula.func<EOL>codeobj = func.__code__<EOL>name = func.__name__  <EOL>namespace_impl = self.owner._namespace_impl.get_updated()<EOL>namespace = namespace_impl.interfaces<EOL>selfnode = get_node(self.owner, None, None)<EOL>for name in self.owner.formula.srcnames:<EOL><INDENT>if name in namespace_impl and isinstance(<EOL>namespace_impl[name], ReferenceImpl<EOL>):<EOL><INDENT>refnode = get_node(namespace_impl[name], None, None)<EOL>self.owner.model.lexdep.add_path([selfnode, refnode])<EOL><DEDENT><DEDENT>closure = func.__closure__  <EOL>if closure is not None:  <EOL><INDENT>closure = create_closure(self.owner.interface)<EOL><DEDENT>self.altfunc = FunctionType(<EOL>codeobj, namespace, name=name, closure=closure<EOL>)<EOL>


return self._impl.spaces.interfaces<EOL>

A mapping of the names of child spaces to the Space objects


if name is None:<EOL><INDENT>return self._impl.model.currentspace.interface<EOL><DEDENT>else:<EOL><INDENT>self._impl.model.currentspace = self._impl.spaces[name]<EOL>return self.cur_space()<EOL><DEDENT>

Set the current space to Space ``name`` and return it.

        If called without arguments, the current space is returned.
        Otherwise, the current space is set to the space named ``name``
        and the space is returned.


result = super()._baseattrs<EOL>result["<STR_LIT>"] = self.spaces._baseattrs<EOL>return result<EOL>


def new_space(self, name=None, bases=None, formula=None, refs=None):

space = self._impl.model.currentspace = self._impl.new_space(<EOL>name=name, bases=get_impls(bases), formula=formula, refs=refs<EOL>)<EOL>return space.interface<EOL>

Create a child space.

        Args:
            name (str, optional): Name of the space. Defaults to ``SpaceN``,
                where ``N`` is a number determined automatically.
            bases (optional): A space or a sequence of spaces to be the base
                space(s) of the created space.
            formula (optional): Function to specify the parameters of
                dynamic child spaces. The signature of this function is used
                for setting parameters for dynamic child spaces.
                This function should return a mapping of keyword arguments
                to be passed to this method when the dynamic child spaces
                are created.

        Returns:
            The new child space.


def import_module(self, module=None, recursive=False, **params):

if module is None:<EOL><INDENT>if "<STR_LIT>" in params:<EOL><INDENT>warnings.warn(<EOL>"<STR_LIT>")<EOL>module = params.pop("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>if "<STR_LIT>" in params:<EOL><INDENT>params["<STR_LIT>"] = get_impls(params["<STR_LIT>"])<EOL><DEDENT>space = (<EOL>self._impl.model.currentspace<EOL>) = self._impl.new_space_from_module(<EOL>module, recursive=recursive, **params<EOL>)<EOL>return get_interfaces(space)<EOL>

Create a child space from an module.

        Args:
            module: a module object or name of the module object.
            recursive: Not yet implemented.
            **params: arguments to pass to ``new_space``

        Returns:
            The new child space created from the module.


def new_space_from_module(self, module, recursive=False, **params):

if "<STR_LIT>" in params:<EOL><INDENT>params["<STR_LIT>"] = get_impls(params["<STR_LIT>"])<EOL><DEDENT>space = (<EOL>self._impl.model.currentspace<EOL>) = self._impl.new_space_from_module(<EOL>module, recursive=recursive, **params<EOL>)<EOL>return get_interfaces(space)<EOL>

Create a child space from an module.

        Alias to :py:meth:`import_module`.

        Args:
            module: a module object or name of the module object.
            recursive: Not yet implemented.
            **params: arguments to pass to ``new_space``

        Returns:
            The new child space created from the module.


Create a child space from an Excel range.

        To use this method, ``openpyxl`` package must be installed.

        Args:
            book (str): Path to an Excel file.
            range_ (str): Range expression, such as "A1", "$G4:$K10",
                or named range "NamedRange1".
            sheet (str): Sheet name (case ignored).
            name (str, optional): Name of the space. Defaults to ``SpaceN``,
                where ``N`` is a number determined automatically.
            names_row (optional): an index number indicating
                what row contains the names of cells and parameters.
                Defaults to the top row (0).
            param_cols (optional): a sequence of index numbers
                indicating parameter columns.
                Defaults to only the leftmost column ([0]).
            names_col (optional): an index number, starting from 0,
                indicating what column contains additional parameters.
            param_rows (optional): a sequence of index numbers, starting from
                0, indicating rows of additional parameters, in case cells are
                defined in two dimensions.
            transpose (optional): Defaults to ``False``.
                If set to ``True``, "row(s)" and "col(s)" in the parameter
                names are interpreted inversely, i.e.
                all indexes passed to "row(s)" parameters are interpreted
                as column indexes,
                and all indexes passed to "col(s)" parameters as row indexes.
            space_param_order: a sequence to specify space parameters and
                their orders. The elements of the sequence denote the indexes
                of ``param_cols`` elements, and optionally the index of
                ``param_rows`` elements shifted by the length of
                ``param_cols``. The elements of this parameter and
                ``cell_param_order`` must not overlap.
            cell_param_order (optional): a sequence to reorder the parameters.
                The elements of the sequence denote the indexes of
                ``param_cols`` elements, and optionally the index of
                ``param_rows`` elements shifted by the length of
                ``param_cols``. The elements of this parameter and
                ``cell_space_order`` must not overlap.

        Returns:
            The new child space created from the Excel range.


for space in self._spaces.values():<EOL><INDENT>space.restore_state(system)<EOL><DEDENT>


raise NotImplementedError<EOL>


from modelx.core.space import StaticSpaceImpl<EOL>if name is None:<EOL><INDENT>name = self.spacenamer.get_next(self.namespace, prefix)<EOL><DEDENT>if name in self.namespace:<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>if not prefix and not is_valid_name(name):<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>space = self._new_space(<EOL>name=name,<EOL>formula=formula,<EOL>refs=refs,<EOL>source=source,<EOL>is_derived=is_derived,<EOL>)<EOL>self._set_space(space)<EOL>self.model.spacegraph.add_space(space)<EOL>if bases is not None:<EOL><INDENT>if isinstance(bases, StaticSpaceImpl):<EOL><INDENT>bases = [bases]<EOL><DEDENT>space.add_bases(bases)<EOL><DEDENT>return space<EOL>

Create a new child space.

        Args:
            name (str): Name of the space. If omitted, the space is
                created automatically.
            bases: If specified, the new space becomes a derived space of
                the `base` space.
            formula: Function whose parameters used to set space parameters.
            refs: a mapping of refs to be added.
            arguments: ordered dict of space parameter names to their values.
            source: A source module from which cell definitions are read.
            prefix: Prefix to the autogenerated name when name is None.


Configure Python ``sys`` settings for modelx.

    This function is called implicitly when importing modelx.
    To restore the Python settings, call :py:func:`restore_python`


Restore Python ``sys`` settings for modelx.

    Restore ``sys`` settings to the original states before
    importing modelx.


Set up IPython shell for modelx.

    Suppress IPython's default traceback messages upon error.


Restore IPython' default error message.

    Bring back IPython's default traceback message upon error for debugging.


_system.callstack.maxdepth = maxdepth<EOL>

Set formula recursion limit.

    Args:
        maxdepth: The maximum depth of the modelx interpreter stack.


return _system.new_model(name).interface<EOL>

Create and return a new model.

    The current model is set set to the created model.

    Args:
        name (:obj:`str`, optional): The name of the model to create.
            Defaults to ``ModelN``, with ``N``
            being an automatically assigned integer.

    Returns:
        The new model.


def new_space(name=None, bases=None, formula=None):

return cur_model().new_space(name, bases, formula)<EOL>

Create and return a new space in the current model.

    The ``currentspace`` of the current model is set to the created model.

    Args:
        name (:obj:`str`, optional): The name of the space to create.
            Defaults to ``SpaceN``, with ``N``
            being an automatically assigned integer.

    Returns:
        The new space.


def defcells(space=None, name=None, *funcs):

if isinstance(space, _FunctionType) and name is None:<EOL><INDENT>func = space<EOL>return _system.currentspace.new_cells(formula=func).interface<EOL><DEDENT>elif (isinstance(space, _Space) or space is None) and (<EOL>isinstance(name, str) or name is None<EOL>):<EOL><INDENT>if space is None:<EOL><INDENT>space = _system.currentspace.interface<EOL><DEDENT>return _CellsMaker(space=space._impl, name=name)<EOL><DEDENT>elif all(<EOL>isinstance(func, _FunctionType) for func in (space, name) + funcs<EOL>):<EOL><INDENT>return [defcells(func) for func in (space, name) + funcs]<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>

Decorator/function to create cells from Python functions.

    Convenience decorator/function to create new cells directly from function
    definitions or function objects substituting for calling
    :py:meth:`new_cells <modelx.core.space.StaticSpace.new_cells>`
    method of the parent space.

    There are 3 ways to use ``defcells`` to define cells from functions.

    **1. As a decorator without arguments**

    To create a cells from a function definition in the current space of the
    current model with the same name as the function's::

        @defcells
        def foo(x):
            return x

    **2. As a decorator with arguments**

    To create a cells from a function definition in a given space and/or with
    a given name::

        @defcells(space=space, name=name)
        def foo(x):
            return x

    **3. As a function**

    To create a multiple cells from a multiple function definitions::

        def foo(x):
            return x

        def bar(y):
            return foo(y)

        foo, bar = defcells(foo, bar)

    Args:
        space(optional): For the 2nd usage, a space to create the cells in.
            Defaults to the current space of the current model.
        name(optional): For the 2nd usage, a name of the created cells.
            Defaults to the function name.
        *funcs: For the 3rd usage, function objects. (``space`` and ``name``
            also take function objects for the 3rd usage.)

    Returns:
        For the 1st and 2nd usage, the newly created single cells is returned.
        For the 3rd usage, a list of newly created cells are returned.


return _get_interfaces(_system.models)<EOL>

Returns a dict that maps model names to models.


<EOL>elms = name.split("<STR_LIT:.>")<EOL>parent = get_models()[elms.pop(<NUM_LIT:0>)]<EOL>while len(elms) > <NUM_LIT:0>:<EOL><INDENT>obj = elms.pop(<NUM_LIT:0>)<EOL>parent = getattr(parent, obj)<EOL><DEDENT>return parent<EOL>


obj = get_object(name)<EOL>args = ast.literal_eval(args)<EOL>if not isinstance(args, tuple):<EOL><INDENT>args = (args,)<EOL><DEDENT>return obj.node(*args)<EOL>

Get node from object name and arg string

    Not Used. Left for future reference purpose.


if model is None:<EOL><INDENT>if _system.currentmodel is not None:<EOL><INDENT>return _system.currentmodel.interface<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if isinstance(model, _Model):<EOL><INDENT>_system.currentmodel = model._impl<EOL><DEDENT>else:<EOL><INDENT>_system.currentmodel = _system.models[model]<EOL><DEDENT>return _system.currentmodel.interface<EOL><DEDENT>

Get and/or set the current model.

    If ``model`` is given, set the current model to ``model`` and return it.
    ``model`` can be the name of a model object, or a model object itself.
    If ``model`` is not given, the current model is returned.


if space is None:<EOL><INDENT>if _system.currentmodel is not None:<EOL><INDENT>if _system.currentmodel.currentspace is not None:<EOL><INDENT>return _system.currentmodel.currentspace.interface<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if isinstance(space, _Space):<EOL><INDENT>cur_model(space.model)<EOL>_system.currentmodel.currentspace = space._impl<EOL><DEDENT>else:<EOL><INDENT>_system.currentmodel.currentspace = _system.currentmodel.spaces[<EOL>space<EOL>]<EOL><DEDENT>return cur_space()<EOL><DEDENT>

Get and/or set the current space of the current model.

    If ``name`` is given, the current space of the current model is
    set to ``name`` and return it.
    If ``name`` is not given, the current space of the current model
    is returned.


return _system.open_model(path, name)<EOL>

Load a model saved from a file and return it.

    Args:
        path (:obj:`str`): Path to the file to load the model from.
        name (optional): If specified, the model is renamed to this name.

    Returns:
        A new model created from the file.


if args is None and kwargs is None:<EOL><INDENT>return (obj,)<EOL><DEDENT>if kwargs is None:<EOL><INDENT>kwargs = {}<EOL><DEDENT>return obj, _bind_args(obj, args, kwargs)<EOL>

Create a node from arguments and return it


obj = node[OBJ]<EOL>key = node[KEY]<EOL>boundargs = obj.formula.signature.bind(*key)<EOL>boundargs.apply_defaults()<EOL>return boundargs.arguments<EOL>

Return an ordered mapping from params to args


def tuplize_key(obj, key, remove_extra=False):

paramlen = len(obj.formula.parameters)<EOL>if isinstance(key, str):<EOL><INDENT>key = (key,)<EOL><DEDENT>elif not isinstance(key, Sequence):<EOL><INDENT>key = (key,)<EOL><DEDENT>if not remove_extra:<EOL><INDENT>return key<EOL><DEDENT>else:<EOL><INDENT>arglen = len(key)<EOL>if arglen:<EOL><INDENT>return key[: min(arglen, paramlen)]<EOL><DEDENT>else:<EOL><INDENT>return key<EOL><DEDENT><DEDENT>


def reorderChild(self, parent, newitem):

source = self.getItem(parent).childItems<EOL>target = newitem.childItems<EOL>i = <NUM_LIT:0><EOL>while i < len(source):<EOL><INDENT>if source[i] == target[i]:<EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>i0 = i<EOL>j0 = source.index(target[i0])<EOL>j = j0 + <NUM_LIT:1><EOL>while j < len(source):<EOL><INDENT>if source[j] == target[j - j0 + i0]:<EOL><INDENT>j += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>self.moveRows(parent, i0, j0, j - j0)<EOL>i += j - j0<EOL><DEDENT><DEDENT>

Reorder a list to match target by moving a sequence at a time.

        Written for QtAbstractItemModel.moveRows.


def moveRows(self, parent, index_to, index_from, length):

source = self.getItem(parent).childItems<EOL>self.beginMoveRows(<EOL>parent, index_from, index_from + length - <NUM_LIT:1>, parent, index_to<EOL>)<EOL>sublist = [source.pop(index_from) for _ in range(length)]<EOL>for _ in range(length):<EOL><INDENT>source.insert(index_to, sublist.pop())<EOL><DEDENT>self.endMoveRows()<EOL>

Move a sub sequence in a list

        index_to must be smaller than index_from


if model is None:<EOL><INDENT>model = mx.cur_model()<EOL><DEDENT>treemodel = ModelTreeModel(model._baseattrs)<EOL>view = QTreeView()<EOL>view.setModel(treemodel)<EOL>view.setWindowTitle("<STR_LIT>" % model.name)<EOL>view.setAlternatingRowColors(True)<EOL>return view<EOL>


Get QTreeView object containing the model tree.

    Args:
        model: :class:`Model <modelx.core.model.Model>` object.
            Defaults to the current model.


if model is None:<EOL><INDENT>model = mx.cur_model()<EOL><DEDENT>view = get_modeltree(model)<EOL>app = QApplication.instance()<EOL>if not app:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>view.show()<EOL>app.exec_()<EOL>

Display the model tree window.

    Args:
        model: :class:`Model <modelx.core.model.Model>` object.
            Defaults to the current model.

    Warnings:
        For this function to work with Spyder, *Graphics backend* option
        of Spyder must be set to *inline*.


return self.predecessors(node)<EOL>


seqs = [self.get_mro(base) for base<EOL>in self.get_bases(space)] + [list(self.get_bases(space))]<EOL>res = []<EOL>while True:<EOL><INDENT>non_empty = list(filter(None, seqs))<EOL>if not non_empty:<EOL><INDENT>res.insert(<NUM_LIT:0>, space)<EOL>return res<EOL><DEDENT>for seq in non_empty:  <EOL><INDENT>candidate = seq[<NUM_LIT:0>]<EOL>not_head = [s for s in non_empty if candidate in s[<NUM_LIT:1>:]]<EOL>if not_head:<EOL><INDENT>candidate = None<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not candidate:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>")<EOL><DEDENT>res.append(candidate)<EOL>for seq in non_empty:<EOL><INDENT>if seq[<NUM_LIT:0>] == candidate:<EOL><INDENT>del seq[<NUM_LIT:0>]<EOL><DEDENT><DEDENT><DEDENT>

Calculate the Method Resolution Order of bases using the C3 algorithm.

        Code modified from
        http://code.activestate.com/recipes/577748-calculate-the-mro-of-a-class/

        Args:
            bases: sequence of direct base spaces.

        Returns:
            mro as a list of bases including node itself


i = <NUM_LIT:0><EOL>while i < len(source):<EOL><INDENT>if source[i] == targetorder[i]:<EOL><INDENT>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>i0 = i<EOL>j0 = source.index(targetorder[i0])<EOL>j = j0 + <NUM_LIT:1><EOL>while j < len(source):<EOL><INDENT>if source[j] == targetorder[j - j0 + i0]:<EOL><INDENT>j += <NUM_LIT:1><EOL>continue<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>move_elements(source, i0, j0, j - j0)<EOL>i += j - j0<EOL><DEDENT><DEDENT>

Reorder a list to match target by moving a sequence at a time.

    Written for QtAbstractItemModel.moveRows.


def move_elements(source, index_to, index_from, length):

sublist = [source.pop(index_from) for _ in range(length)]<EOL>for _ in range(length):<EOL><INDENT>source.insert(index_to, sublist.pop())<EOL><DEDENT>


PyCode_New = ctypes.pythonapi.PyCode_New<EOL>PyCode_New.argtypes = (<EOL>ctypes.c_int,<EOL>ctypes.c_int,<EOL>ctypes.c_int,<EOL>ctypes.c_int,<EOL>ctypes.c_int,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.py_object,<EOL>ctypes.c_int,<EOL>ctypes.py_object)<EOL>PyCode_New.restype = ctypes.py_object<EOL>args = [<EOL>[code.co_argcount, '<STR_LIT>'],<EOL>[code.co_kwonlyargcount, '<STR_LIT>'],<EOL>[code.co_nlocals, '<STR_LIT>'],<EOL>[code.co_stacksize, '<STR_LIT>'],<EOL>[code.co_flags, '<STR_LIT>'],<EOL>[code.co_code, '<STR_LIT>'],<EOL>[code.co_consts, '<STR_LIT>'],<EOL>[code.co_names, '<STR_LIT>'],<EOL>[code.co_varnames, '<STR_LIT>'],<EOL>[code.co_freevars, '<STR_LIT>'],<EOL>[code.co_cellvars, '<STR_LIT>'],<EOL>[code.co_filename, '<STR_LIT>'],<EOL>[code.co_name, '<STR_LIT>'],<EOL>[code.co_firstlineno, '<STR_LIT>'],<EOL>[code.co_lnotab, '<STR_LIT>']]<EOL>for arg in args:<EOL><INDENT>if arg[<NUM_LIT:1>] in attrs:<EOL><INDENT>arg[<NUM_LIT:0>] = attrs[arg[<NUM_LIT:1>]]<EOL><DEDENT><DEDENT>return PyCode_New(<EOL>args[<NUM_LIT:0>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:1>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:2>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:3>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:4>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:5>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:6>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:7>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:8>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:9>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:10>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:11>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT:12>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT>][<NUM_LIT:0>],  <EOL>args[<NUM_LIT>][<NUM_LIT:0>])<EOL>

Create a new code object by altering some of ``code`` attributes

    Args:
        code: code objcect
        attrs: a mapping of names of code object attrs to their values


def alter_freevars(func, globals_=None, **vars):

if globals_ is None:<EOL><INDENT>globals_ = func.__globals__<EOL><DEDENT>frees = tuple(vars.keys())<EOL>oldlocs = func.__code__.co_names<EOL>newlocs = tuple(name for name in oldlocs if name not in frees)<EOL>code = _alter_code(func.__code__,<EOL>co_freevars=frees,<EOL>co_names=newlocs,<EOL>co_flags=func.__code__.co_flags | inspect.CO_NESTED)<EOL>closure = _create_closure(*vars.values())<EOL>return FunctionType(code, globals_, closure=closure)<EOL>

Replace local variables with free variables

    Warnings:
        This function does not work.


with open(path.join(here, '<STR_LIT>'), '<STR_LIT:r>') as f:<EOL><INDENT>data = f.read()<EOL><DEDENT>return data<EOL>


n = payback_period<EOL>return principal / pv_payments(n)<EOL>

How much amount per payment for a given payback period.


if n == <NUM_LIT:1>:<EOL><INDENT>return <NUM_LIT:1> / (<NUM_LIT:1> + interest_rate)<EOL><DEDENT>else:<EOL><INDENT>return pv_payments(n - <NUM_LIT:1>) + <NUM_LIT:1> / (<NUM_LIT:1> + interest_rate) ** n<EOL><DEDENT>

Present value of repayments of 1 for n years.


n = <NUM_LIT:1><EOL>while annual_payment * pv_payments(n) < principal:<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Payback period for a given annual payment


def __init__(self, hostname: str = '<STR_LIT>',<EOL>port: int = <NUM_LIT>, retries: int = <NUM_LIT:10>,<EOL>log_errors: bool = False) -> None:

self.hostname = hostname<EOL>self.port = port<EOL>self.retries = retries<EOL>self.log_errors = log_errors<EOL>self._log_fp = Path(tempfile.gettempdir(), '<STR_LIT>')<EOL>if self._log_fp.is_file():<EOL><INDENT>self._log_fp.open('<STR_LIT:w>').close()<EOL><DEDENT>

:param hostname: The IP address of the TweeboParser API server.
:param port: The Port that the TweeboParser API server is attached to.
:param retries: Number of times to retry json decoding the
                returned data.


if self.log_errors:<EOL><INDENT>with self._log_fp.open('<STR_LIT>') as log_file:<EOL><INDENT>log_file.write(f'<STR_LIT>')<EOL><DEDENT><DEDENT>

Given some error text it will log the text if self.log_errors is True

:param text: Error text to log


def parse_conll(self, texts: List[str], retry_count: int = <NUM_LIT:0>) -> List[str]:

post_data = {'<STR_LIT>': texts, '<STR_LIT>': '<STR_LIT>'}<EOL>try:<EOL><INDENT>response = requests.post(f'<STR_LIT>',<EOL>json=post_data,<EOL>headers={'<STR_LIT>': '<STR_LIT>'})<EOL>response.raise_for_status()<EOL><DEDENT>except (requests.exceptions.ConnectionError,<EOL>requests.exceptions.Timeout) as server_error:<EOL><INDENT>raise ServerError(server_error, self.hostname, self.port)<EOL><DEDENT>except requests.exceptions.HTTPError as http_error:<EOL><INDENT>raise http_error<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return response.json()<EOL><DEDENT>except json.JSONDecodeError as json_exception:<EOL><INDENT>if retry_count == self.retries:<EOL><INDENT>self.log_error(response.text)<EOL>raise Exception('<STR_LIT>'<EOL>f'<STR_LIT>')<EOL><DEDENT>return self.parse_conll(texts, retry_count + <NUM_LIT:1>)<EOL><DEDENT><DEDENT>

Processes the texts using TweeboParse and returns them in CoNLL format.

:param texts: The List of Strings to be processed by TweeboParse.
:param retry_count: The number of times it has retried for. Default
                    0 does not require setting, main purpose is for
                    recursion.
:return: A list of CoNLL formated strings.
:raises ServerError: Caused when the server is not running.
:raises :py:class:`requests.exceptions.HTTPError`: Caused when the
        input texts is not formated correctly e.g. When you give it a
        String not a list of Strings.
:raises :py:class:`json.JSONDecodeError`: Caused if after self.retries
        attempts to parse the data it cannot decode the data.

:Example:


def parse_stanford(self, texts: List[str], retry_count: int = <NUM_LIT:0><EOL>) -> List[Dict[str, Union[str, int]]]:

post_data = {'<STR_LIT>': texts, '<STR_LIT>': '<STR_LIT>'}<EOL>try:<EOL><INDENT>response = requests.post(f'<STR_LIT>',<EOL>json=post_data,<EOL>headers={'<STR_LIT>': '<STR_LIT>'})<EOL>response.raise_for_status()<EOL><DEDENT>except (requests.exceptions.ConnectionError,<EOL>requests.exceptions.Timeout,<EOL>requests.exceptions.InvalidSchema) as server_error:<EOL><INDENT>raise ServerError(server_error, self.hostname, self.port)<EOL><DEDENT>except requests.exceptions.HTTPError as http_error:<EOL><INDENT>raise http_error<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return response.json()<EOL><DEDENT>except json.JSONDecodeError as json_exception:<EOL><INDENT>if retry_count == self.retries:<EOL><INDENT>self.log_error(response.text)<EOL>raise Exception('<STR_LIT>'<EOL>f'<STR_LIT>')<EOL><DEDENT>return self.parse_stanford(texts, retry_count + <NUM_LIT:1>)<EOL><DEDENT><DEDENT>

Processes the texts using TweeboParse and returns them in a Stanford
styled format (as in the same format as the json return of the Stanford
CoreNLP server dependency parser).

:param texts: The List of Strings to be processed by TweeboParse.
:param retry_count: The number of times it has retried for. Default
                    0 does not require setting, main purpose is for
                    recursion.
:return: A list of dicts.
:raises ServerError: Caused when the server is not running.
:raises :py:class:`requests.exceptions.HTTPError`: Caused when the
        input texts is not formated correctly e.g. When you give it a
        String not a list of Strings.
:raises :py:class:`json.JSONDecodeError`: Caused if after self.retries
        attempts to parse the data it cannot decode the data.

:Example:
::
    from tweebo_parser import API
    tweebo_api = API()
    text_data = ['hello how are you', 'Where are we going']
    result = tweebo_api.parse_stanford(text_data)
    print(result)
    [{}]


def __init__(self, excpetion: requests.exceptions.RequestException,<EOL>hostname: str, port: int) -> None:

message = f'<STR_LIT>'<EOL>if isinstance(excpetion, requests.exceptions.Timeout):<EOL><INDENT>message = '<STR_LIT>'f'<STR_LIT>'<EOL><DEDENT>elif isinstance(excpetion, requests.exceptions.ConnectionError):<EOL><INDENT>message = '<STR_LIT>'f'<STR_LIT>'<EOL><DEDENT>self.message = message<EOL>

:param exception: The requests exception instance that is raised.
:param hostname: The IP address of the API server.
:param port: The Port that the API server is attached to.


if len(kwargs) != <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>[(typ, string)] = kwargs.items()<EOL>types = {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}<EOL>if typ not in types:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>if to not in types:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>def pascal_iter(string):<EOL><INDENT>yield from (m.group(<NUM_LIT:0>) for m in re.finditer(r'<STR_LIT>', string))<EOL><DEDENT>def snake_iter(string):<EOL><INDENT>yield from (m.group(<NUM_LIT:2>) for m in re.finditer(r'<STR_LIT>', string))<EOL><DEDENT>inputs = {<EOL>'<STR_LIT>': pascal_iter,<EOL>'<STR_LIT>': pascal_iter,<EOL>'<STR_LIT>': snake_iter,<EOL>'<STR_LIT>': snake_iter,<EOL>}<EOL>def out_fun(sep, case=None, case_fst=None):<EOL><INDENT>if case is None:<EOL><INDENT>case = lambda x: x<EOL><DEDENT>if case_fst is None:<EOL><INDENT>case_fst = case<EOL><DEDENT>return lambda tokens: sep.join(case_fst(token) if i == <NUM_LIT:0> else case(token) for i, token in enumerate(tokens))<EOL><DEDENT>outputs = {<EOL>'<STR_LIT>': out_fun('<STR_LIT>', str.capitalize),<EOL>'<STR_LIT>': out_fun('<STR_LIT>', str.capitalize, str.lower),<EOL>'<STR_LIT>': out_fun('<STR_LIT:_>', str.lower),<EOL>'<STR_LIT>': out_fun('<STR_LIT:_>', str.upper),<EOL>}<EOL>tokens = inputs[typ](string)<EOL>return outputs[to](tokens)<EOL>

Converts an identifier from one case type to another.
    An identifier is an ASCII string consisting of letters, digits and underscores, not starting with a digit.
    The supported case types are camelCase, PascalCase, snake_case, and CONSTANT_CASE,
    identified as camel, pascal, snake, and constant.
    The input identifier is given as a keyword argument with one of these names,
    and the output type is given as a string in the `to` keyword argument.
    If a given string does not conform to the specified case type (such as underscores in camel or pascal case strings,
    or double__underscores in general), the result may not be as desired,
    although things like snaKe_casE or CONStaNT_CASe will generally work.


return LatexFixer(*args, **kwargs).tostring()<EOL>

Transform a unicode string into another more compatible with latex,
    fixing some common typographical errors


return self.data<EOL>


not_sentence_end_chars = ['<STR_LIT:U+0020>']<EOL>abbreviations = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>titles = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for abbrev in abbreviations:<EOL><INDENT>for x in not_sentence_end_chars:<EOL><INDENT>self._str_replacement(abbrev + x, abbrev + '<STR_LIT>')<EOL><DEDENT><DEDENT>for title in titles:<EOL><INDENT>for x in not_sentence_end_chars:<EOL><INDENT>self._str_replacement(title + x, title + '<STR_LIT>')<EOL><DEDENT><DEDENT>

Fix common spacing errors caused by LaTeX's habit
        of using an inter-sentence space after any full stop.


pass<EOL>

Fix errors where inter-sentence spacing
        is not used after after a word ending with a capital letter.


substs = [('<STR_LIT>', '<STR_LIT>'), <EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT>', '<STR_LIT>')]<EOL>for sub in substs:<EOL><INDENT>self._str_replacement(*sub)<EOL><DEDENT>

Replace unicode symbols with latex commands
        where those symbols cannot be represented in utf mode.


problematic_hyphens = [(r'<STR_LIT>', r'<STR_LIT>'),<EOL>(r'<STR_LIT>', '<STR_LIT>'),<EOL>(r'<STR_LIT>', '<STR_LIT>')]<EOL>for problem_case in problematic_hyphens:<EOL><INDENT>self._regex_replacement(*problem_case)<EOL><DEDENT>

Transform hyphens to various kinds of dashes


def _str_replacement(self, target, replacement):

self.data = self.data.replace(target, replacement)<EOL>


def _regex_replacement(self, target, replacement):

match = re.compile(target)<EOL>self.data = match.sub(replacement, self.data)<EOL>

Regex substitute target with replacement


for f, callback, transferred, key, ov in events:<EOL><INDENT>try:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(callback))<EOL>value = callback(transferred, key, ov)<EOL><DEDENT>except OSError:<EOL><INDENT>self._logger.warning('<STR_LIT>', exc_info=sys.exc_info())<EOL><DEDENT>else:<EOL><INDENT>f.set_result(value)<EOL><DEDENT><DEDENT>


if not self.__events:<EOL><INDENT>self._poll(timeout)<EOL><DEDENT>tmp = self.__events<EOL>self.__events = []<EOL>return tmp<EOL>

Override in order to handle events in a threadsafe manner.


if timeout is None:<EOL><INDENT>ms = UINT32_MAX  <EOL><DEDENT>elif timeout < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>ms = math.ceil(timeout * <NUM_LIT>)<EOL>if ms >= UINT32_MAX:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>with QtCore.QMutexLocker(self._lock):<EOL><INDENT>while True:<EOL><INDENT>status = _overlapped.GetQueuedCompletionStatus(self._iocp, ms)<EOL>if status is None:<EOL><INDENT>break<EOL><DEDENT>err, transferred, key, address = status<EOL>try:<EOL><INDENT>f, ov, obj, callback = self._cache.pop(address)<EOL><DEDENT>except KeyError:<EOL><INDENT>if key not in (<NUM_LIT:0>, _overlapped.INVALID_HANDLE_VALUE):<EOL><INDENT>_winapi.CloseHandle(key)<EOL><DEDENT>ms = <NUM_LIT:0><EOL>continue<EOL><DEDENT>if obj in self._stopped_serving:<EOL><INDENT>f.cancel()<EOL><DEDENT>elif not f.done():<EOL><INDENT>self.__events.append((f, callback, transferred, key, ov))<EOL><DEDENT>ms = <NUM_LIT:0><EOL><DEDENT><DEDENT>


attr_name = '<STR_LIT>'<EOL>cls_name = cls.__qualname__<EOL>module = cls.__module__<EOL>if module is not None:<EOL><INDENT>cls_name = module + '<STR_LIT:.>' + cls_name<EOL><DEDENT>else:<EOL><INDENT>raise AssertionError<EOL><DEDENT>setattr(cls, attr_name, logging.getLogger(cls_name))<EOL>return cls<EOL>

Class decorator to add a logger to a class.


if isinstance(fileobj, int):<EOL><INDENT>fd = fileobj<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>fd = int(fileobj.fileno())<EOL><DEDENT>except (AttributeError, TypeError, ValueError) as ex:<EOL><INDENT>raise ValueError("<STR_LIT>".format(fileobj)) from ex<EOL><DEDENT><DEDENT>if fd < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>".format(fd))<EOL><DEDENT>return fd<EOL>

Return a file descriptor from a file object.

Parameters:
fileobj -- file object or file descriptor

Returns:
corresponding file descriptor

Raises:
ValueError if the object is invalid


raise NotImplementedError<EOL>

Implement abstract method even though we don't need it.


try:<EOL><INDENT>return _fileobj_to_fd(fileobj)<EOL><DEDENT>except ValueError:<EOL><INDENT>for key in self._fd_to_key.values():<EOL><INDENT>if key.fileobj is fileobj:<EOL><INDENT>return key.fd<EOL><DEDENT><DEDENT>raise<EOL><DEDENT>

Return a file descriptor from a file object.

        This wraps _fileobj_to_fd() to do an exhaustive search in case
        the object is invalid but we still have it in our map.  This
        is used by unregister() so we can unregister an object that
        was previously registered even if it is closed.  It is also
        used by _SelectorMapping.


try:<EOL><INDENT>return self._fd_to_key[fd]<EOL><DEDENT>except KeyError:<EOL><INDENT>return None<EOL><DEDENT>

Return the key associated to a given file descriptor.

Parameters:
fd -- file descriptor

Returns:
corresponding key, or None if not found


self._logger.debug('<STR_LIT>'.format(key, mask))<EOL>fileobj, (reader, writer) = key.fileobj, key.data<EOL>if mask & selectors.EVENT_READ and reader is not None:<EOL><INDENT>if reader._cancelled:<EOL><INDENT>self.remove_reader(fileobj)<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(reader))<EOL>reader._run()<EOL><DEDENT><DEDENT>if mask & selectors.EVENT_WRITE and writer is not None:<EOL><INDENT>if writer._cancelled:<EOL><INDENT>self.remove_writer(fileobj)<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(writer))<EOL>writer._run()<EOL><DEDENT><DEDENT>


@functools.wraps(fn)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>f = asyncio.ensure_future(fn(*args, **kwargs))<EOL>while not f.done():<EOL><INDENT>QApplication.instance().processEvents()<EOL><DEDENT><DEDENT>return wrapper<EOL>

Allow to run async code before application is closed.


def outer_decorator(fn):<EOL><INDENT>@Slot(*args)<EOL>@functools.wraps(fn)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>asyncio.ensure_future(fn(*args, **kwargs))<EOL><DEDENT>return wrapper<EOL><DEDENT>return outer_decorator<EOL>

Make a Qt async slot run on asyncio loop.


self.__is_running = True<EOL>self._before_run_forever()<EOL>try:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>rslt = self.__app.exec_()<EOL>self._logger.debug('<STR_LIT>'.format(rslt))<EOL>return rslt<EOL><DEDENT>finally:<EOL><INDENT>self._after_run_forever()<EOL>self.__is_running = False<EOL><DEDENT>


self._logger.debug('<STR_LIT>'.format(future))<EOL>future = asyncio.ensure_future(future, loop=self)<EOL>def stop(*args): self.stop()  <EOL>future.add_done_callback(stop)<EOL>try:<EOL><INDENT>self.run_forever()<EOL><DEDENT>finally:<EOL><INDENT>future.remove_done_callback(stop)<EOL><DEDENT>self.__app.processEvents()  <EOL>if not future.done():<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>self._logger.debug('<STR_LIT>'.format(future))<EOL>return future.result()<EOL>


if not self.__is_running:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>return<EOL><DEDENT>self._logger.debug('<STR_LIT>')<EOL>self.__is_running = False<EOL>self.__app.exit()<EOL>self._logger.debug('<STR_LIT>')<EOL>


return self.__is_running<EOL>

Return True if the event loop is running, False otherwise.


if self.is_running():<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>')<EOL>if self.__default_executor is not None:<EOL><INDENT>self.__default_executor.shutdown()<EOL><DEDENT>super().close()<EOL>self._timer.stop()<EOL>self.__app = None<EOL>for notifier in itertools.chain(self._read_notifiers.values(), self._write_notifiers.values()):<EOL><INDENT>notifier.setEnabled(False)<EOL><DEDENT>self._read_notifiers = None<EOL>self._write_notifiers = None<EOL>

Release all resources used by the event loop.

The loop cannot be restarted after it has been closed.


def call_later(self, delay, callback, *args, context=None):

if asyncio.iscoroutinefunction(callback):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if not callable(callback):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(callback).__name__))<EOL><DEDENT>self._logger.debug(<EOL>'<STR_LIT>'<EOL>.format(callback, args, delay))<EOL>if sys.version_info >= (<NUM_LIT:3>, <NUM_LIT:7>):<EOL><INDENT>return self._add_callback(asyncio.Handle(callback, args, self, context=context), delay)<EOL><DEDENT>return self._add_callback(asyncio.Handle(callback, args, self), delay)<EOL>

Register callback to be invoked after a certain delay.


def call_soon(self, callback, *args, context=None):

return self.call_later(<NUM_LIT:0>, callback, *args, context=context)<EOL>

Register a callback to be run on the next iteration of the event loop.


def call_at(self, when, callback, *args, context=None):

return self.call_later(when - self.time(), callback, *args, context=context)<EOL>

Register callback to be invoked at a certain time.


return time.monotonic()<EOL>

Get time according to event loop's clock.


def add_reader(self, fd, callback, *args):

self._check_closed()<EOL>try:<EOL><INDENT>existing = self._read_notifiers[fd]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>existing.setEnabled(False)<EOL>existing.activated.disconnect()<EOL><DEDENT>notifier = QtCore.QSocketNotifier(fd, QtCore.QSocketNotifier.Read)<EOL>notifier.setEnabled(True)<EOL>self._logger.debug('<STR_LIT>'.format(fd))<EOL>notifier.activated.connect(<EOL>lambda: self.__on_notifier_ready(<EOL>self._read_notifiers, notifier, fd, callback, args)  <EOL>)<EOL>self._read_notifiers[fd] = notifier<EOL>

Register a callback for when a file descriptor is ready for reading.


if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>'.format(fd))<EOL>try:<EOL><INDENT>notifier = self._read_notifiers.pop(fd)<EOL><DEDENT>except KeyError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>notifier.setEnabled(False)<EOL>return True<EOL><DEDENT>


def add_writer(self, fd, callback, *args):

self._check_closed()<EOL>try:<EOL><INDENT>existing = self._write_notifiers[fd]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>existing.setEnabled(False)<EOL>existing.activated.disconnect()<EOL><DEDENT>notifier = QtCore.QSocketNotifier(fd, QtCore.QSocketNotifier.Write)<EOL>notifier.setEnabled(True)<EOL>self._logger.debug('<STR_LIT>'.format(fd))<EOL>notifier.activated.connect(<EOL>lambda: self.__on_notifier_ready(<EOL>self._write_notifiers, notifier, fd, callback, args)  <EOL>)<EOL>self._write_notifiers[fd] = notifier<EOL>

Register a callback for when a file descriptor is ready for writing.


if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>'.format(fd))<EOL>try:<EOL><INDENT>notifier = self._write_notifiers.pop(fd)<EOL><DEDENT>except KeyError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>notifier.setEnabled(False)<EOL>return True<EOL><DEDENT>


def call_soon_threadsafe(self, callback, *args, context=None):

self.__call_soon_signal.emit(callback, args)<EOL>


def run_in_executor(self, executor, callback, *args):

self._logger.debug('<STR_LIT>'.format(callback, args))<EOL>if isinstance(callback, asyncio.Handle):<EOL><INDENT>assert not args<EOL>assert not isinstance(callback, asyncio.TimerHandle)<EOL>if callback._cancelled:<EOL><INDENT>f = asyncio.Future()<EOL>f.set_result(None)<EOL>return f<EOL><DEDENT>callback, args = callback.callback, callback.args<EOL><DEDENT>if executor is None:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>executor = self.__default_executor<EOL><DEDENT>if executor is None:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>executor = self.__default_executor = QThreadExecutor()<EOL><DEDENT>return asyncio.wrap_future(executor.submit(callback, *args))<EOL>

Run callback in executor.

        If no executor is provided, the default executor will be used, which defers execution to
        a background thread.


self._logger.debug('<STR_LIT>')<EOL>message = context.get('<STR_LIT:message>')<EOL>if not message:<EOL><INDENT>message = '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>exception = context['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>exc_info = False<EOL><DEDENT>else:<EOL><INDENT>exc_info = (type(exception), exception, exception.__traceback__)<EOL><DEDENT>log_lines = [message]<EOL>for key in [k for k in sorted(context) if k not in {'<STR_LIT:message>', '<STR_LIT>'}]:<EOL><INDENT>log_lines.append('<STR_LIT>'.format(key, context[key]))<EOL><DEDENT>self.__log_error('<STR_LIT:\n>'.join(log_lines), exc_info=exc_info)<EOL>

Handle exceptions.

        This is the default exception handler.

        This is called when an exception occurs and no exception
        handler is set, and can be called by a custom exception
        handler that wants to defer to the default behavior.

        context parameter has the same meaning as in
        `call_exception_handler()`.


def assert_tags(self, field, expected_value, result_value):

<EOL>expected_value = set(expected_value)<EOL>msg = ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>self.assertEqual(len(result_value), len(expected_value), msg=msg)<EOL>for tag in result_value:<EOL><INDENT>self.assertTrue(tag in expected_value)<EOL><DEDENT>


full_id = self.id().split('<STR_LIT:.>')<EOL>test, module, cls, func = full_id<EOL>path = os.path.join(<EOL>os.path.dirname(CURRENT_PATH),<EOL>'<STR_LIT>',<EOL>"<STR_LIT:data>",<EOL>module.partition('<STR_LIT>')[<NUM_LIT:2>],<EOL>"<STR_LIT>" % func)<EOL>path = os.path.abspath(path)<EOL>content = FileHelper.loadResourceFile(path)<EOL>self.data = json.loads(content)<EOL>


def runArticleAssertions(self, article, fields):

for field in fields:<EOL><INDENT>expected_value = self.data['<STR_LIT>'][field]<EOL>result_value = getattr(article, field, None)<EOL>if field in ['<STR_LIT>']:<EOL><INDENT>self.assertEqual(type(result_value), type(datetime.today()))<EOL>result_value = result_value.isoformat(sep='<STR_LIT:U+0020>')<EOL><DEDENT>assertion = '<STR_LIT>' % field<EOL>if hasattr(self, assertion):<EOL><INDENT>getattr(self, assertion)(field, expected_value, result_value)<EOL>continue<EOL><DEDENT>msg = "<STR_LIT>" % (field, expected_value, result_value)<EOL>self.assertEqual(expected_value, result_value, msg=msg)<EOL><DEDENT>


<EOL>self.loadData()<EOL>self.loadHtml()<EOL>config = self.getConfig()<EOL>if config is not None:<EOL><INDENT>if isinstance(config_, dict):<EOL><INDENT>for k, v in list(config_.items()):<EOL><INDENT>if hasattr(config, k):<EOL><INDENT>setattr(config, k, v)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.parser = config.get_parser()<EOL>target_language = self.data.get('<STR_LIT>')<EOL>if target_language:<EOL><INDENT>config.target_language = target_language<EOL>config.use_meta_language = False<EOL><DEDENT>with open('<STR_LIT>'.format(CURRENT_PATH), '<STR_LIT:rb>') as fobj:<EOL><INDENT>img_content = fobj.read()<EOL><DEDENT>with open('<STR_LIT>'.format(CURRENT_PATH), '<STR_LIT:rb>') as fobj:<EOL><INDENT>blank_img = fobj.read()<EOL><DEDENT>g = Goose(config=config)<EOL>with requests_mock.Mocker(real_http=False) as m:<EOL><INDENT>m.get('<STR_LIT>', content=blank_img)<EOL>m.get('<STR_LIT>', content=blank_img)<EOL>m.get('<STR_LIT>', content=img_content)<EOL>if "<STR_LIT:url>" in self.data:<EOL><INDENT>m.get(self.data['<STR_LIT:url>'], text=self.html)<EOL>return g.extract(url=self.data['<STR_LIT:url>'])<EOL><DEDENT>else:<EOL><INDENT>return g.extract(raw_html=self.html)<EOL><DEDENT><DEDENT>


<EOL>if "<STR_LIT>" in list(self.article.opengraph.keys()):<EOL><INDENT>site_name = self.article.opengraph['<STR_LIT>']<EOL>title = title.replace(site_name, '<STR_LIT>').strip()<EOL><DEDENT>elif (self.article.schema and "<STR_LIT>" in self.article.schema and<EOL>"<STR_LIT:name>" in self.article.schema["<STR_LIT>"]):<EOL><INDENT>site_name = self.article.schema["<STR_LIT>"]["<STR_LIT:name>"]<EOL>title = title.replace(site_name, '<STR_LIT>').strip()<EOL><DEDENT>if self.article.domain:<EOL><INDENT>pattern = re.compile(self.article.domain, re.IGNORECASE)<EOL>title = pattern.sub("<STR_LIT>", title).strip()<EOL><DEDENT>title_words = title.split()<EOL>if title_words and title_words[<NUM_LIT:0>] in TITLE_SPLITTERS:<EOL><INDENT>title_words.pop(<NUM_LIT:0>)<EOL><DEDENT>if not title_words:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>if title_words[-<NUM_LIT:1>] in TITLE_SPLITTERS:<EOL><INDENT>title_words.pop(-<NUM_LIT:1>)<EOL><DEDENT>title = "<STR_LIT:U+0020>".join(title_words).strip()<EOL>return title<EOL>

Clean title with the use of og:site_name
        in this case try to get rid of site name
        and use TITLE_SPLITTERS to reformat title


title = '<STR_LIT>'<EOL>if "<STR_LIT:title>" in list(self.article.opengraph.keys()):<EOL><INDENT>return self.clean_title(self.article.opengraph['<STR_LIT:title>'])<EOL><DEDENT>elif self.article.schema and "<STR_LIT>" in self.article.schema:<EOL><INDENT>return self.clean_title(self.article.schema['<STR_LIT>'])<EOL><DEDENT>meta_headline = self.parser.getElementsByTag(self.article.doc,<EOL>tag="<STR_LIT>",<EOL>attr="<STR_LIT:name>",<EOL>value="<STR_LIT>")<EOL>if meta_headline is not None and len(meta_headline) > <NUM_LIT:0>:<EOL><INDENT>title = self.parser.getAttribute(meta_headline[<NUM_LIT:0>], '<STR_LIT:content>')<EOL>return self.clean_title(title)<EOL><DEDENT>title_element = self.parser.getElementsByTag(self.article.doc, tag='<STR_LIT:title>')<EOL>if title_element is not None and len(title_element) > <NUM_LIT:0>:<EOL><INDENT>title = self.parser.getText(title_element[<NUM_LIT:0>])<EOL>return self.clean_title(title)<EOL><DEDENT>return title<EOL>

\
        Fetch the article title and analyze it


video = Video()<EOL>video._embed_code = self.get_embed_code(node)<EOL>video._embed_type = self.get_embed_type(node)<EOL>video._width = self.get_width(node)<EOL>video._height = self.get_height(node)<EOL>video._src = self.get_src(node)<EOL>video._provider = self.get_provider(video.src)<EOL>return video<EOL>

Create a video object from a video embed


return Video()<EOL>


<EOL>if self.config.use_meta_language:<EOL><INDENT>if self.article.meta_lang:<EOL><INDENT>return self.article.meta_lang[:<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return self.config.target_language<EOL>

Returns the language is by the article or
the configuration language


para = "<STR_LIT:p>"<EOL>steps_away = <NUM_LIT:0><EOL>minimum_stopword_count = <NUM_LIT:5><EOL>max_stepsaway_from_node = <NUM_LIT:3><EOL>nodes = self.walk_siblings(node)<EOL>for current_node in nodes:<EOL><INDENT>current_node_tag = self.parser.getTag(current_node)<EOL>if current_node_tag == para:<EOL><INDENT>if steps_away >= max_stepsaway_from_node:<EOL><INDENT>return False<EOL><DEDENT>para_text = self.parser.getText(current_node)<EOL>word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(para_text)<EOL>if word_stats.get_stopword_count() > minimum_stopword_count:<EOL><INDENT>return True<EOL><DEDENT>steps_away += <NUM_LIT:1><EOL><DEDENT><DEDENT>return False<EOL>

\
        alot of times the first paragraph might be the caption under an image
        so we'll want to make sure if we're going to boost a parent node that
        it should be connected to other paragraphs,
        at least for the first n paragraphs so we'll want to make sure that
        the next sibling is a paragraph and has at
        least some substatial weight to it


def get_siblings_content(self, current_sibling, baselinescore_siblings_para):

if current_sibling.tag == '<STR_LIT:p>' and self.parser.getText(current_sibling):<EOL><INDENT>tmp = current_sibling<EOL>if tmp.tail:<EOL><INDENT>tmp = deepcopy(tmp)<EOL>tmp.tail = '<STR_LIT>'<EOL><DEDENT>return [tmp]<EOL><DEDENT>else:<EOL><INDENT>potential_paragraphs = self.parser.getElementsByTag(current_sibling, tag='<STR_LIT:p>')<EOL>if potential_paragraphs is None:<EOL><INDENT>return None<EOL><DEDENT>paragraphs = list()<EOL>for first_paragraph in potential_paragraphs:<EOL><INDENT>text = self.parser.getText(first_paragraph)<EOL>if text:  <EOL><INDENT>word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text)<EOL>paragraph_score = word_stats.get_stopword_count()<EOL>sibling_baseline_score = float(<NUM_LIT>)<EOL>high_link_density = self.is_highlink_density(first_paragraph)<EOL>score = float(baselinescore_siblings_para * sibling_baseline_score)<EOL>if score < paragraph_score and not high_link_density:<EOL><INDENT>para = self.parser.createElement(tag='<STR_LIT:p>', text=text, tail=None)<EOL>paragraphs.append(para)<EOL><DEDENT><DEDENT><DEDENT>return paragraphs<EOL><DEDENT>

adds any siblings that may have a decent score to this node


base = <NUM_LIT><EOL>paragraphs_number = <NUM_LIT:0><EOL>paragraphs_score = <NUM_LIT:0><EOL>nodes_to_check = self.parser.getElementsByTag(top_node, tag='<STR_LIT:p>')<EOL>for node in nodes_to_check:<EOL><INDENT>text_node = self.parser.getText(node)<EOL>word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text_node)<EOL>high_link_density = self.is_highlink_density(node)<EOL>if word_stats.get_stopword_count() > <NUM_LIT:2> and not high_link_density:<EOL><INDENT>paragraphs_number += <NUM_LIT:1><EOL>paragraphs_score += word_stats.get_stopword_count()<EOL><DEDENT><DEDENT>if paragraphs_number > <NUM_LIT:0>:<EOL><INDENT>base = paragraphs_score // paragraphs_number<EOL><DEDENT>return base<EOL>

we could have long articles that have tons of paragraphs
so if we tried to calculate the base score against
the total text score of those paragraphs it would be unfair.
So we need to normalize the score based on the average scoring
of the paragraphs within the top node.
For example if our total score of 10 paragraphs was 1000
but each had an average value of 100 then 100 should be our base.


def update_score(self, node, add_to_score):

current_score = <NUM_LIT:0><EOL>score_string = self.parser.getAttribute(node, '<STR_LIT>')<EOL>if score_string:<EOL><INDENT>current_score = int(score_string)<EOL><DEDENT>new_score = current_score + int(add_to_score)<EOL>self.parser.setAttribute(node, "<STR_LIT>", str(new_score))<EOL>

adds a score to the gravityScore Attribute we put on divs
we'll get the current score then add the score
we're passing in to the current


def update_node_count(self, node, add_to_count):

current_score = <NUM_LIT:0><EOL>count_string = self.parser.getAttribute(node, '<STR_LIT>')<EOL>if count_string:<EOL><INDENT>current_score = int(count_string)<EOL><DEDENT>new_score = current_score + add_to_count<EOL>self.parser.setAttribute(node, "<STR_LIT>", str(new_score))<EOL>

\
        stores how many decent nodes are under a parent node


links = self.parser.getElementsByTag(element, tag='<STR_LIT:a>')<EOL>if not links:<EOL><INDENT>return False<EOL><DEDENT>text = self.parser.getText(element)<EOL>words = text.split('<STR_LIT:U+0020>')<EOL>words_number = float(len(words))<EOL>link_text_parts = []<EOL>for link in links:<EOL><INDENT>link_text_parts.append(self.parser.getText(link))<EOL><DEDENT>link_text = '<STR_LIT>'.join(link_text_parts)<EOL>link_words = link_text.split('<STR_LIT:U+0020>')<EOL>number_of_link_words = float(len(link_words))<EOL>number_of_links = float(len(links))<EOL>link_divisor = float(number_of_link_words / words_number)<EOL>score = float(link_divisor * number_of_links)<EOL>if score >= <NUM_LIT:1.0>:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

checks the density of links within a node,
is there not much text and most of it contains linky shit?
if so it's no good


return self.get_node_gravity_score(node) or <NUM_LIT:0><EOL>

returns the gravityScore as an integer from this node


nodes_to_check = []<EOL>for doc in docs:<EOL><INDENT>for tag in ['<STR_LIT:p>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>items = self.parser.getElementsByTag(doc, tag=tag)<EOL>nodes_to_check += items<EOL><DEDENT><DEDENT>return nodes_to_check<EOL>

\
        returns a list of nodes we want to search
        on like paragraphs and tables


parse_tags = ['<STR_LIT:p>']<EOL>if self.config.parse_lists:<EOL><INDENT>parse_tags.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>if self.config.parse_headers:<EOL><INDENT>parse_tags.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>target_node = self.article.top_node<EOL>node = self.add_siblings(target_node)<EOL>for elm in self.parser.getChildren(node):<EOL><INDENT>e_tag = self.parser.getTag(elm)<EOL>if e_tag not in parse_tags:<EOL><INDENT>if (self.is_highlink_density(elm) or self.is_table_and_no_para_exist(elm) or<EOL>not self.is_nodescore_threshold_met(node, elm)):<EOL><INDENT>self.parser.remove(elm)<EOL><DEDENT><DEDENT><DEDENT>return node<EOL>

\
        remove any divs that looks like non-content,
        clusters of links, or paras with no gusto


def check_large_images(self, node, parent_depth_level, sibling_depth_level):

good_images = self.get_image_candidates(node)<EOL>if good_images:<EOL><INDENT>scored_images = self.fetch_images(good_images, parent_depth_level)<EOL>if scored_images:<EOL><INDENT>highscore_image = sorted(list(scored_images.items()),<EOL>key=lambda x: x[<NUM_LIT:1>], reverse=True)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>main_image = Image()<EOL>main_image._src = highscore_image.src<EOL>main_image._width = highscore_image.width<EOL>main_image._height = highscore_image.height<EOL>main_image._extraction_type = "<STR_LIT>"<EOL>score_len = len(scored_images)<EOL>main_image._confidence_score = <NUM_LIT:100> / score_len if score_len > <NUM_LIT:0> else <NUM_LIT:0><EOL>return main_image<EOL><DEDENT><DEDENT>depth_obj = self.get_depth_level(node, parent_depth_level, sibling_depth_level)<EOL>if depth_obj:<EOL><INDENT>return self.check_large_images(depth_obj.node, depth_obj.parent_depth,<EOL>depth_obj.sibling_depth)<EOL><DEDENT>return None<EOL>

\
        although slow the best way to determine the best image is to download
        them and check the actual dimensions of the image when on disk
        so we'll go through a phased approach...
        1. get a list of ALL images from the parent node
        2. filter out any bad image names that we know of (gifs, ads, etc..)
        3. do a head request on each file to make sure it meets
           our bare requirements
        4. any images left over let's do a full GET request,
           download em to disk and check their dimensions
        5. Score images based on different factors like height/width
           and possibly things like color density


def fetch_images(self, images, depth_level):

image_results = {}<EOL>initial_area = float(<NUM_LIT:0.0>)<EOL>total_score = float(<NUM_LIT:0.0>)<EOL>cnt = float(<NUM_LIT:1.0>)<EOL>min_width = <NUM_LIT:50><EOL>for image in images[:<NUM_LIT:30>]:<EOL><INDENT>src = self.parser.getAttribute(image, attr='<STR_LIT:src>')<EOL>src = self.build_image_path(src)<EOL>src = self.add_schema_if_none(src)<EOL>local_image = self.get_local_image(src)<EOL>width = local_image.width<EOL>height = local_image.height<EOL>src = local_image.src<EOL>file_extension = local_image.file_extension<EOL>if file_extension != '<STR_LIT>' or file_extension != '<STR_LIT>':<EOL><INDENT>if (depth_level >= <NUM_LIT:1> and local_image.width > <NUM_LIT>) or depth_level < <NUM_LIT:1>:<EOL><INDENT>if not self.is_banner_dimensions(width, height):<EOL><INDENT>if width > min_width:<EOL><INDENT>sequence_score = float(<NUM_LIT:1.0> / cnt)<EOL>area = float(width * height)<EOL>total_score = float(<NUM_LIT:0.0>)<EOL>if initial_area == <NUM_LIT:0>:<EOL><INDENT>initial_area = area * float(<NUM_LIT>)<EOL>total_score = <NUM_LIT:1.0><EOL><DEDENT>else:<EOL><INDENT>area_difference = float(area / initial_area)<EOL>total_score = sequence_score * area_difference<EOL><DEDENT>image_results.update({local_image: total_score})<EOL>cnt += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return image_results<EOL>

\
        download the images to temp disk and set their dimensions
        - we're going to score the images in the order in which
          they appear so images higher up will have more importance,
        - we'll count the area of the 1st image as a score
          of 1 and then calculate how much larger or small each image after it is
        - we'll also make sure to try and weed out banner
          type ad blocks that have big widths and small heights or vice versa
        - so if the image is 3rd found in the dom it's
          sequence score would be 1 / 3 = .33 * diff
          in area from the first image


if width == height:<EOL><INDENT>return False<EOL><DEDENT>if width > height:<EOL><INDENT>diff = float(width / height)<EOL>if diff > <NUM_LIT:5>:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>if height > width:<EOL><INDENT>diff = float(height / width)<EOL>if diff > <NUM_LIT:5>:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

\
        returns true if we think this is kind of a bannery dimension
        like 600 / 100 = 6 may be a fishy dimension for a good image


good_images = []<EOL>for image in images:<EOL><INDENT>if self.is_valid_filename(image):<EOL><INDENT>good_images.append(image)<EOL><DEDENT><DEDENT>return good_images if len(good_images) > <NUM_LIT:0> else None<EOL>

\
        takes a list of image elements
        and filters out the ones with bad names


src = self.parser.getAttribute(image_node, attr='<STR_LIT:src>')<EOL>if not src:<EOL><INDENT>return False<EOL><DEDENT>if self.badimages_names_re.search(src):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

\
        will check the image src against a list
        of bad image files we know of like buttons, etc...


cnt = <NUM_LIT:0><EOL>max_bytes_size = <NUM_LIT><EOL>good_images = []<EOL>for image in images:<EOL><INDENT>if cnt > <NUM_LIT:30>:<EOL><INDENT>return good_images<EOL><DEDENT>src = self.parser.getAttribute(image, attr='<STR_LIT:src>')<EOL>src = self.build_image_path(src)<EOL>src = self.add_schema_if_none(src)<EOL>local_image = self.get_local_image(src)<EOL>if local_image:<EOL><INDENT>filesize = local_image.bytes<EOL>if (filesize == <NUM_LIT:0> or filesize > self.images_min_bytes) and filesize < max_bytes_size:<EOL><INDENT>good_images.append(image)<EOL><DEDENT>else:<EOL><INDENT>images.remove(image)<EOL><DEDENT><DEDENT>cnt += <NUM_LIT:1><EOL><DEDENT>return good_images if len(good_images) > <NUM_LIT:0> else None<EOL>

\
        loop through all the images and find the ones
        that have the best bytez to even make them a candidate


node = self.article.raw_doc<EOL>meta = self.parser.getElementsByTag(node, tag='<STR_LIT>', attr='<STR_LIT>', value='<STR_LIT>')<EOL>for item in meta:<EOL><INDENT>src = self.parser.getAttribute(item, attr='<STR_LIT>')<EOL>if src:<EOL><INDENT>return self.get_image(src, extraction_type='<STR_LIT>')<EOL><DEDENT><DEDENT>return None<EOL>

\
        checks to see if we were able to
        find open link_src on this page


if '<STR_LIT:image>' in self.article.opengraph:<EOL><INDENT>return self.get_image(self.article.opengraph["<STR_LIT:image>"],<EOL>extraction_type='<STR_LIT>')<EOL><DEDENT>elif (self.article.schema and '<STR_LIT:image>' in self.article.schema and<EOL>"<STR_LIT:url>" in self.article.schema["<STR_LIT:image>"]):<EOL><INDENT>return self.get_image(self.article.schema["<STR_LIT:image>"]["<STR_LIT:url>"],<EOL>extraction_type='<STR_LIT>')<EOL><DEDENT>return None<EOL>

\
        checks to see if we were able to find the image via known schemas:

        Supported Schemas
         - Open Graph
         - schema.org


return ImageUtils.store_image(self.fetcher, self.article.link_hash, src, self.config)<EOL>

\
        returns the bytes of the image file on disk