parser = ArgumentParser()<EOL>parser.add_argument(<EOL>"<STR_LIT:-c>", "<STR_LIT>", dest="<STR_LIT>",<EOL>help="<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>parser.add_argument(<EOL>"<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>",<EOL>help="<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>parser.add_argument(<EOL>"<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>",<EOL>help="<STR_LIT>" +<EOL>"<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>parser.add_argument(<EOL>"<STR_LIT>", "<STR_LIT>", dest="<STR_LIT:type>", default='<STR_LIT>',<EOL>help="<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>" +<EOL>"<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>parser.add_argument(<EOL>"<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>",<EOL>help="<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>parser.add_argument(<EOL>"<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>",<EOL>help="<STR_LIT>",<EOL>metavar="<STR_LIT>")<EOL>options = parser.parse_args()<EOL>version = os.popen("<STR_LIT>").read().strip()<EOL>if options.inputfile:<EOL><INDENT>if options.inputfile.endswith('<STR_LIT>'):<EOL><INDENT>input_genome_file = open(options.inputfile)<EOL><DEDENT>elif options.inputfile.endswith('<STR_LIT>'):<EOL><INDENT>input_genome_file = gzip.open(options.inputfile)<EOL><DEDENT>elif options.inputfile.endswith('<STR_LIT>'):<EOL><INDENT>input_genome_file = bz2.BZ2File(options.inputfile)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>elif not sys.stdin.isatty():<EOL><INDENT>input_genome_file = sys.stdin<EOL><DEDENT>else:<EOL><INDENT>sys.stderr.write("<STR_LIT>")<EOL>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if options.build and options.build in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>build = options.build<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>if (not (options.clinvarfile or options.clinvardir) or<EOL>(options.clinvarfile and options.clinvardir)):<EOL><INDENT>sys.stderr.write("<STR_LIT>")<EOL>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if options.clinvarfile:<EOL><INDENT>clinvarfilename = options.clinvarfile<EOL><DEDENT>elif options.clinvardir:<EOL><INDENT>clinvarfilename = get_latest_vcf_file(target_dir=options.clinvardir,<EOL>build=build)<EOL><DEDENT>if clinvarfilename.endswith('<STR_LIT>'):<EOL><INDENT>input_clinvar_file = open(options.clinvarfile)<EOL><DEDENT>elif clinvarfilename.endswith('<STR_LIT>'):<EOL><INDENT>input_clinvar_file = gzip.open(clinvarfilename)<EOL><DEDENT>elif clinvarfilename.endswith('<STR_LIT>'):<EOL><INDENT>input_clinvar_file = bz2.BZ2File(clinvarfilename)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>" +<EOL>"<STR_LIT>")<EOL><DEDENT>if options.type not in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>if options.type == "<STR_LIT>":<EOL><INDENT>csv_report(input_genome_file=input_genome_file,<EOL>input_clinvar_file=input_clinvar_file,<EOL>build=build,<EOL>version=version)<EOL><DEDENT>elif options.type == "<STR_LIT>":<EOL><INDENT>notes_json = {}<EOL>if options.notes:<EOL><INDENT>notes_json["<STR_LIT>"] = options.notes<EOL>try:<EOL><INDENT>notes_json = json.loads(options.notes)<EOL><DEDENT>except:<EOL><INDENT>sys.stderr.write("<STR_LIT>")<EOL><DEDENT><DEDENT>json_report(input_genome_file=input_genome_file,<EOL>input_clinvar_file=input_clinvar_file,<EOL>build=build,<EOL>notes=notes_json,<EOL>version=version)<EOL><DEDENT>

Parse command line argument and
output appropriate file type (csv or JSON)


(self.clnalleleid, self.hgvs, self.clnsig,<EOL>self.clndn, self.clndisdb, self.clnvi) = [<EOL>kwargs[x] for x in<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']]<EOL>super(ClinVarAllele, self).__init__(*args, **kwargs)<EOL>

Initialize ClinVarAllele object

A ClinVarAllele is an allele for a genomic position that has data
from ClinVar associated with it.

Required arguments:
sequence:  String of DNA letters (A, C, G, or T) for the allele;
           may be empty (to represent a deletion)
frequency: Preferred allele frequency
alleleid:  ClinVar Allele ID
clnhgvs:   HGVS nomenclature for this allele
clnsig:    ClinVar clinical significance
clndn:     ClinVar disease name
clndisdb:  Database IDs of disease database entries (tag-value pairs)
clnvi:     Database IDs of clinical sources (tag-value pairs)


self_as_dict = super(ClinVarAllele, self).as_dict(*args, **kwargs)<EOL>self_as_dict['<STR_LIT>'] = self.hgvs<EOL>self_as_dict['<STR_LIT>'] = self.clnalleleid<EOL>self_as_dict['<STR_LIT>'] = self.clnsig<EOL>self_as_dict['<STR_LIT>'] = self.clndn<EOL>self_as_dict['<STR_LIT>'] = self.clndisdb<EOL>self_as_dict['<STR_LIT>'] = self.clnvi<EOL>return self_as_dict<EOL>

Return ClinVarAllele data as dict object.


kwargs['<STR_LIT>'] = False<EOL>super(ClinVarVCFLine, self).__init__(self, *args, **kwargs)<EOL>


return {'<STR_LIT>': self.chrom,<EOL>'<STR_LIT:start>': self.start,<EOL>'<STR_LIT>': self.ref_allele,<EOL>'<STR_LIT>': self.alt_alleles,<EOL>'<STR_LIT:info>': self.info,<EOL>'<STR_LIT>': [x.as_dict() for x in self.alleles]}<EOL>

Dict representation of parsed ClinVar VCF line


frequencies = OrderedDict([<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT>', '<STR_LIT>')])<EOL>pref_freq = '<STR_LIT>'<EOL>for source in frequencies.keys():<EOL><INDENT>freq_key = '<STR_LIT>' + source<EOL>if freq_key in self.info:<EOL><INDENT>frequencies[source] = self.info[freq_key]<EOL>if pref_freq == '<STR_LIT>':<EOL><INDENT>pref_freq = frequencies[source]<EOL><DEDENT><DEDENT><DEDENT>return pref_freq, frequencies<EOL>


<EOL>pref_freq, frequencies = self._parse_frequencies()<EOL>info_clnvar_single_tags = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>cln_data = {x.lower(): self.info[x] if x in self.info else None<EOL>for x in info_clnvar_single_tags}<EOL>cln_data.update(<EOL>{'<STR_LIT>': [x.split('<STR_LIT:U+002C>') for x in<EOL>self.info['<STR_LIT>'].split('<STR_LIT:|>')]<EOL>if '<STR_LIT>' in self.info else []})<EOL>cln_data.update({'<STR_LIT>': self.info['<STR_LIT>'].split('<STR_LIT:|>') if<EOL>'<STR_LIT>' in self.info else []})<EOL>cln_data.update({'<STR_LIT>': self.info['<STR_LIT>'].split('<STR_LIT:U+002C>')<EOL>if '<STR_LIT>' in self.info else []})<EOL>try:<EOL><INDENT>sequence = self.alt_alleles[<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>sequence = self.ref_allele<EOL><DEDENT>allele = ClinVarAllele(frequency=pref_freq, sequence=sequence,<EOL>**cln_data)<EOL>if not cln_data['<STR_LIT>']:<EOL><INDENT>return []<EOL><DEDENT>return [allele]<EOL>

Parse alleles for ClinVar VCF, overrides parent method.


sequence = kwargs['<STR_LIT>']<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>frequency = kwargs['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>frequency = '<STR_LIT>'<EOL><DEDENT>if not (re.match(r'<STR_LIT>', sequence) or<EOL>re.match(r'<STR_LIT>', sequence)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.sequence = sequence<EOL>if frequency:<EOL><INDENT>try:<EOL><INDENT>if (float(frequency) < <NUM_LIT:0.0> or<EOL>float(frequency) > <NUM_LIT:1.0>):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>if not frequency == '<STR_LIT>':<EOL><INDENT>raise ValueError('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>self.frequency = frequency<EOL><DEDENT>

Initialize Allele object

Required arguments:
sequence:  Short string of DNA letters (ACGT) for the allele.
           May be empty (to represent a deletion).

Optional arguments:
frequency: a string representation of a float between 0 and 1


return self.as_json()<EOL>

Print Allele object as dict object data.


return self.as_json()<EOL>


self_as_dict = dict()<EOL>self_as_dict['<STR_LIT>'] = self.sequence<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>self_as_dict['<STR_LIT>'] = self.frequency<EOL><DEDENT>return self_as_dict<EOL>


return json.dumps(self.as_dict())<EOL>


vcf_line = kwargs['<STR_LIT>']<EOL>skip_info = ('<STR_LIT>' in kwargs and kwargs['<STR_LIT>'])<EOL>vcf_fields = vcf_line.strip().split('<STR_LIT:\t>')<EOL>self.chrom = vcf_fields[<NUM_LIT:0>]<EOL>self.start = int(vcf_fields[<NUM_LIT:1>])<EOL>self.ref_allele = vcf_fields[<NUM_LIT:3>]<EOL>if vcf_fields[<NUM_LIT:4>] == '<STR_LIT:.>':<EOL><INDENT>self.alt_alleles = []<EOL><DEDENT>else:<EOL><INDENT>self.alt_alleles = vcf_fields[<NUM_LIT:4>].split('<STR_LIT:U+002C>')<EOL><DEDENT>if not skip_info:<EOL><INDENT>self.info = self._parse_info(vcf_fields[<NUM_LIT:7>])<EOL><DEDENT>self.alleles = self._parse_allele_data()<EOL>


return [Allele(sequence=x) for x in<EOL>[self.ref_allele] + self.alt_alleles]<EOL>

Create list of Alleles from VCF line data


info = dict()<EOL>for item in info_field.split('<STR_LIT:;>'):<EOL><INDENT>info_item_data = item.split('<STR_LIT:=>')<EOL>if len(info_item_data) == <NUM_LIT:1>:<EOL><INDENT>info[info_item_data[<NUM_LIT:0>]] = True<EOL><DEDENT>elif len(info_item_data) == <NUM_LIT:2>:<EOL><INDENT>info[info_item_data[<NUM_LIT:0>]] = info_item_data[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>return info<EOL>


return json.dumps(self.as_dict(), ensure_ascii=True)<EOL>

String representation of parsed VCF data


self_as_dict = {'<STR_LIT>': self.chrom,<EOL>'<STR_LIT:start>': self.start,<EOL>'<STR_LIT>': self.ref_allele,<EOL>'<STR_LIT>': self.alt_alleles,<EOL>'<STR_LIT>': [x.as_dict() for x in self.alleles]}<EOL>try:<EOL><INDENT>self_as_dict['<STR_LIT:info>'] = self.info<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>return self_as_dict<EOL>


if not vcf_line:<EOL><INDENT>return None<EOL><DEDENT>vcf_data = vcf_line.strip().split('<STR_LIT:\t>')<EOL>return_data = dict()<EOL>return_data['<STR_LIT>'] = CHROM_INDEX[vcf_data[<NUM_LIT:0>]]<EOL>return_data['<STR_LIT>'] = int(vcf_data[<NUM_LIT:1>])<EOL>return return_data<EOL>

Very lightweight parsing of a vcf line to get position.

Returns a dict containing:
'chrom': index of chromosome (int), indicates sort order
'pos': position on chromosome (int)


clin_curr_line = _next_line(clin_file)<EOL>genome_curr_line = _next_line(genome_file)<EOL>while clin_curr_line.startswith('<STR_LIT:#>'):<EOL><INDENT>clin_curr_line = _next_line(clin_file)<EOL><DEDENT>while genome_curr_line.startswith('<STR_LIT:#>'):<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL><DEDENT>while clin_curr_line and genome_curr_line:<EOL><INDENT>clin_curr_pos = VCFLine.get_pos(clin_curr_line)<EOL>genome_curr_pos = VCFLine.get_pos(genome_curr_line)<EOL>try:<EOL><INDENT>if clin_curr_pos['<STR_LIT>'] > genome_curr_pos['<STR_LIT>']:<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL>continue<EOL><DEDENT>elif clin_curr_pos['<STR_LIT>'] < genome_curr_pos['<STR_LIT>']:<EOL><INDENT>clin_curr_line = _next_line(clin_file)<EOL>continue<EOL><DEDENT>if clin_curr_pos['<STR_LIT>'] > genome_curr_pos['<STR_LIT>']:<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL>continue<EOL><DEDENT>elif clin_curr_pos['<STR_LIT>'] < genome_curr_pos['<STR_LIT>']:<EOL><INDENT>clin_curr_line = _next_line(clin_file)<EOL>continue<EOL><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>genome_vcf_line = GenomeVCFLine(vcf_line=genome_curr_line,<EOL>skip_info=True)<EOL>if not genome_vcf_line.genotype_allele_indexes:<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL>continue<EOL><DEDENT>clinvar_vcf_line = ClinVarVCFLine(vcf_line=clin_curr_line)<EOL>if not genome_vcf_line.ref_allele == clinvar_vcf_line.ref_allele:<EOL><INDENT>try:<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL>clin_curr_line = _next_line(clin_file)<EOL>continue<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT><DEDENT>genotype_allele_indexes = genome_vcf_line.genotype_allele_indexes<EOL>genome_alleles = [genome_vcf_line.alleles[x] for<EOL>x in genotype_allele_indexes]<EOL>if len(genome_alleles) == <NUM_LIT:1>:<EOL><INDENT>zygosity = '<STR_LIT>'<EOL><DEDENT>elif len(genome_alleles) == <NUM_LIT:2>:<EOL><INDENT>if genome_alleles[<NUM_LIT:0>].sequence == genome_alleles[<NUM_LIT:1>].sequence:<EOL><INDENT>zygosity = '<STR_LIT>'<EOL>genome_alleles = [genome_alleles[<NUM_LIT:0>]]<EOL><DEDENT>else:<EOL><INDENT>zygosity = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' +<EOL>str(genome_vcf_line))<EOL><DEDENT>for genome_allele in genome_alleles:<EOL><INDENT>for allele in clinvar_vcf_line.alleles:<EOL><INDENT>if genome_allele.sequence == allele.sequence:<EOL><INDENT>if hasattr(allele, '<STR_LIT>'):<EOL><INDENT>yield (genome_vcf_line, allele, zygosity)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>genome_curr_line = _next_line(genome_file)<EOL>clin_curr_line = _next_line(clin_file)<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Match a genome VCF to variants in the ClinVar VCF file

Acts as a generator, yielding tuples of:
(ClinVarVCFLine, ClinVarAllele, zygosity)

'zygosity' is a string and corresponds to the genome's zygosity for that
ClinVarAllele. It can be either: 'Het' (heterozygous), 'Hom' (homozygous),
or 'Hem' (hemizygous, e.g. X chromosome in XY individuals).


if build == '<STR_LIT>':<EOL><INDENT>ftp.cwd(DIR_CLINVAR_VCF_B37)<EOL><DEDENT>elif build == '<STR_LIT>':<EOL><INDENT>ftp.cwd(DIR_CLINVAR_VCF_B38)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>

Navigate an open ftplib.FTP to appropriate directory for ClinVar VCF files.

Args:
    ftp:   (type: ftplib.FTP) an open connection to ftp.ncbi.nlm.nih.gov
    build: (type: string) genome build, either 'b37' or 'b38'


format_col = vcf_fields[<NUM_LIT:8>].split('<STR_LIT::>')<EOL>genome_data = vcf_fields[<NUM_LIT:9>].split('<STR_LIT::>')<EOL>try:<EOL><INDENT>gt_idx = format_col.index('<STR_LIT>')<EOL><DEDENT>except ValueError:<EOL><INDENT>return []<EOL><DEDENT>return [int(x) for x in re.split(r'<STR_LIT>', genome_data[gt_idx]) if<EOL>x != '<STR_LIT:.>']<EOL>


for item in table:<EOL><INDENT>res = {<EOL>k.lower(): nfd(v) if isinstance(v, text_type) else v for k, v in item.items()}<EOL>for extra in res.pop('<STR_LIT>', []):<EOL><INDENT>k, _, v = extra.partition('<STR_LIT::>')<EOL>res[k.strip()] = v.strip()<EOL><DEDENT>yield res<EOL><DEDENT>

Auxiliary function for iterating over a data table.


return self.resolve_sound(sound)<EOL>

Return a Sound instance matching the specification.


from lingpy.sequence.sound_classes import token2class<EOL>from lingpy.data import Model<EOL>columns = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>bipa = TranscriptionSystem('<STR_LIT>')<EOL>for src, rows in args.repos.iter_sources(type='<STR_LIT>'):<EOL><INDENT>args.log.info('<STR_LIT>'.format(src['<STR_LIT>']))<EOL>uritemplate = URITemplate(src['<STR_LIT>']) if src['<STR_LIT>'] else None<EOL>out = [['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'] + columns]<EOL>graphemes = set()<EOL>for row in rows:<EOL><INDENT>if row['<STR_LIT>'] in graphemes:<EOL><INDENT>args.log.warn('<STR_LIT>'.format(row['<STR_LIT>']))<EOL>continue<EOL><DEDENT>graphemes.add(row['<STR_LIT>'])<EOL>if not row['<STR_LIT>']:<EOL><INDENT>bipa_sound = bipa[row['<STR_LIT>']]<EOL>explicit = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>bipa_sound = bipa[row['<STR_LIT>']]<EOL>explicit = '<STR_LIT:+>'<EOL><DEDENT>generated = '<STR_LIT:+>' if bipa_sound.generated else '<STR_LIT>'<EOL>if is_valid_sound(bipa_sound, bipa):<EOL><INDENT>bipa_grapheme = bipa_sound.s<EOL>bipa_name = bipa_sound.name<EOL><DEDENT>else:<EOL><INDENT>bipa_grapheme, bipa_name = '<STR_LIT>', '<STR_LIT>'<EOL><DEDENT>url = uritemplate.expand(**row) if uritemplate else row.get('<STR_LIT>', '<STR_LIT>')<EOL>out.append(<EOL>[bipa_grapheme, bipa_name, generated, explicit, row['<STR_LIT>'],<EOL>url] + [<EOL>row.get(c, '<STR_LIT>') for c in columns])<EOL><DEDENT>found = len([o for o in out if o[<NUM_LIT:0>] != '<STR_LIT>'])<EOL>args.log.info('<STR_LIT>'.format(<EOL>found, len(out), found / len(out) * <NUM_LIT:100>))<EOL>with UnicodeWriter(<EOL>pkg_path('<STR_LIT>', '<STR_LIT>'.format(src['<STR_LIT>'])), delimiter='<STR_LIT:\t>'<EOL>) as writer:<EOL><INDENT>writer.writerows(out)<EOL><DEDENT><DEDENT>count = <NUM_LIT:0><EOL>with UnicodeWriter(pkg_path('<STR_LIT>', '<STR_LIT>'), delimiter='<STR_LIT:\t>') as writer:<EOL><INDENT>writer.writerow(['<STR_LIT>', '<STR_LIT>'] + SOUNDCLASS_SYSTEMS)<EOL>for grapheme, sound in sorted(bipa.sounds.items()):<EOL><INDENT>if not sound.alias:<EOL><INDENT>writer.writerow(<EOL>[sound.name, grapheme] + [token2class(<EOL>grapheme, Model(cls)) for cls in SOUNDCLASS_SYSTEMS])<EOL>count += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>args.log.info('<STR_LIT>'.format(count))<EOL>

Prepare transcriptiondata from the transcription sources.


sound = sound if isinstance(sound, Sound) else self.system[sound]<EOL>if sound.name in self.data:<EOL><INDENT>return '<STR_LIT>'.join([x['<STR_LIT>'] for x in self.data[sound.name]])<EOL><DEDENT>raise KeyError("<STR_LIT>")<EOL>

Function tries to identify a sound in the data.

        Notes
        -----
        The function tries to resolve sounds to take a sound with less complex
        features in order to yield the next approximate sound class, if the
        transcription data are sound classes.


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>assert id_<EOL>system = pkg_path('<STR_LIT>', id_)<EOL>if not (system.exists() and system.is_dir()):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(id_))<EOL><DEDENT>self.system = TableGroup.from_file(<EOL>pkg_path('<STR_LIT>', '<STR_LIT>'))<EOL>self.system._fname = system / '<STR_LIT>'<EOL>self.features = {'<STR_LIT>': {}, '<STR_LIT>': {}, '<STR_LIT>': {}}<EOL>self._feature_values = {}<EOL>features = jsonlib.load(pkg_path('<STR_LIT>', '<STR_LIT>'))<EOL>self.diacritics = dict(<EOL>consonant={}, vowel={}, click={}, diphthong={}, tone={}, cluster={})<EOL>for dia in itertable(self.system.tabledict['<STR_LIT>']):<EOL><INDENT>if not dia['<STR_LIT>'] and not dia['<STR_LIT>']:<EOL><INDENT>self.features[dia['<STR_LIT:type>']][dia['<STR_LIT:value>']] = dia['<STR_LIT>']<EOL><DEDENT>self._feature_values[dia['<STR_LIT:value>']] = dia['<STR_LIT>']<EOL>self.diacritics[dia['<STR_LIT:type>']][dia['<STR_LIT>']] = dia['<STR_LIT:value>']<EOL><DEDENT>self.sound_classes = {}<EOL>self.columns = {}  <EOL>self.sounds = {}  <EOL>self._covered = {}<EOL>aliases = []<EOL>for cls in [Consonant, Vowel, Tone, Marker]:  <EOL><INDENT>type_ = cls.__name__.lower()<EOL>self.sound_classes[type_] = cls<EOL>self.columns[type_] = [<EOL>c['<STR_LIT:name>'].lower() for c in<EOL>self.system.tabledict['<STR_LIT>'.format(type_)]<EOL>.asdict()['<STR_LIT>']['<STR_LIT>']]<EOL>for l, item in enumerate(itertable(<EOL>self.system.tabledict['<STR_LIT>'.format(type_)])):<EOL><INDENT>if item['<STR_LIT>'] in self.sounds:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(<EOL>type_ + '<STR_LIT>', l + <NUM_LIT:2>, item['<STR_LIT>']))<EOL><DEDENT>sound = cls(ts=self, **item)<EOL>for key, value in item.items():<EOL><INDENT>if key not in {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'} andvalue and value not in self._feature_values:<EOL><INDENT>self._feature_values[value] = key<EOL>if type_ != '<STR_LIT>' and value not in features[type_][key]:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>".format(<EOL>key, value, l + <NUM_LIT:2>))<EOL><DEDENT><DEDENT><DEDENT>self.sounds[item['<STR_LIT>']] = sound<EOL>if not sound.alias:<EOL><INDENT>if sound.featureset in self.features:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(<EOL>type_ + '<STR_LIT>', l + <NUM_LIT:2>, sound.name))<EOL><DEDENT>self.features[sound.featureset] = sound<EOL><DEDENT>else:<EOL><INDENT>aliases += [(l, sound.type, sound.featureset)]<EOL><DEDENT><DEDENT><DEDENT>if [x for x in aliases if x[<NUM_LIT:2>] not in self.features]:  <EOL><INDENT>error = '<STR_LIT:U+002CU+0020>'.join(<EOL>text_type(x[<NUM_LIT:0>] + <NUM_LIT:2>) + '<STR_LIT:/>' + text_type(x[<NUM_LIT:1>])<EOL>for x in aliases if x[<NUM_LIT:2>] not in self.features)<EOL>raise ValueError(<EOL>'<STR_LIT>'.format(error))<EOL><DEDENT>self._regex = None<EOL>self._update_regex()<EOL>self._normalize = {<EOL>norm(r['<STR_LIT:source>']): norm(r['<STR_LIT:target>'])<EOL>for r in itertable(self.system.tabledict['<STR_LIT>'])}<EOL>

:param system: The name of a transcription system or a directory containing one.


nstring = norm(string)<EOL>if "<STR_LIT:/>" in string:<EOL><INDENT>s, t = string.split('<STR_LIT:/>')<EOL>nstring = t<EOL><DEDENT>return self.normalize(nstring)<EOL>

Extended normalization: normalize by list of norm-characers, split
        by character "/".


return '<STR_LIT>'.join([self._normalize.get(x, x) for x in nfd(string)])<EOL>

Normalize the string according to normalization list


components = string.split('<STR_LIT:U+0020>')<EOL>if frozenset(components) in self.features:<EOL><INDENT>return self.features[frozenset(components)]<EOL><DEDENT>rest, sound_class = components[:-<NUM_LIT:1>], components[-<NUM_LIT:1>]<EOL>if sound_class in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if string.startswith('<STR_LIT>') and '<STR_LIT>' in string:<EOL><INDENT>extension = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>'}[sound_class]<EOL>string_ = '<STR_LIT:U+0020>'.join(string.split('<STR_LIT:U+0020>')[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>from_, to_ = string_.split('<STR_LIT>')<EOL>v1, v2 = frozenset(from_.split('<STR_LIT:U+0020>') + [extension]), frozenset(<EOL>to_.split('<STR_LIT:U+0020>') + [extension])<EOL>if v1 in self.features and v2 in self.features:<EOL><INDENT>s1, s2 = (self.features[v1], self.features[v2])<EOL>if sound_class == '<STR_LIT>':<EOL><INDENT>return Diphthong.from_sounds(s1 + s2, s1, s2, self)  <EOL><DEDENT>else:<EOL><INDENT>return Cluster.from_sounds(s1 + s2, s1, s2, self)  <EOL><DEDENT><DEDENT>else:<EOL><INDENT>s1, s2 = self._from_name(from_ + '<STR_LIT:U+0020>' + extension), self._from_name(<EOL>to_ + '<STR_LIT:U+0020>' + extension)<EOL>if not (isinstance(<EOL>s1, UnknownSound) or isinstance(s2, UnknownSound)):  <EOL><INDENT>if sound_class == '<STR_LIT>':<EOL><INDENT>return Diphthong.from_sounds(  <EOL>s1 + s2, s1, s2, self)<EOL><DEDENT>return Cluster.from_sounds(s1 + s2, s1, s2, self)  <EOL><DEDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>if sound_class not in self.sound_classes:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>args = {self._feature_values.get(comp, '<STR_LIT:?>'): comp for comp in rest}<EOL>if '<STR_LIT:?>' in args:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>args['<STR_LIT>'] = '<STR_LIT>'<EOL>args['<STR_LIT>'] = self<EOL>sound = self.sound_classes[sound_class](**args)<EOL>if sound.featureset not in self.features:<EOL><INDENT>sound.generated = True<EOL>return sound<EOL><DEDENT>return self.features[sound.featureset]<EOL>


nstring = self._norm(string)<EOL>if nstring in self.sounds:<EOL><INDENT>sound = self.sounds[nstring]<EOL>sound.normalized = nstring != string<EOL>sound.source = string<EOL>return sound<EOL><DEDENT>match = list(self._regex.finditer(nstring))<EOL>if len(match) == <NUM_LIT:2>:<EOL><INDENT>sound1 = self._parse(nstring[:match[<NUM_LIT:1>].start()])<EOL>sound2 = self._parse(nstring[match[<NUM_LIT:1>].start():])<EOL>if '<STR_LIT>' not in (sound1.type, sound2.type) andsound1.type == sound2.type:<EOL><INDENT>if sound1.type == '<STR_LIT>':<EOL><INDENT>return Diphthong.from_sounds(  <EOL>string, sound1, sound2, self)<EOL><DEDENT>elif sound1.type == '<STR_LIT>' andsound1.manner in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>') andsound2.manner in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>return Cluster.from_sounds(  <EOL>string, sound1, sound2, self)<EOL><DEDENT><DEDENT>return UnknownSound(grapheme=nstring, source=string, ts=self)  <EOL><DEDENT>if len(match) != <NUM_LIT:1>:<EOL><INDENT>return UnknownSound(grapheme=nstring, source=string, ts=self)  <EOL><DEDENT>pre, mid, post = nstring.partition(nstring[match[<NUM_LIT:0>].start():match[<NUM_LIT:0>].end()])<EOL>base_sound = self.sounds[mid]<EOL>if isinstance(base_sound, Marker):  <EOL><INDENT>assert pre or post<EOL>return UnknownSound(grapheme=nstring, source=string, ts=self)  <EOL><DEDENT>features = attr.asdict(base_sound)<EOL>features.update(<EOL>source=string,<EOL>generated=True,<EOL>normalized=nstring != string,<EOL>base=base_sound.grapheme)<EOL>grapheme, sound = '<STR_LIT>', '<STR_LIT>'<EOL>for dia in [p + EMPTY for p in pre]:<EOL><INDENT>feature = self.diacritics[base_sound.type].get(dia, {})<EOL>if not feature:<EOL><INDENT>return UnknownSound(  <EOL>grapheme=nstring, source=string, ts=self)<EOL><DEDENT>features[self._feature_values[feature]] = feature<EOL>grapheme += dia[<NUM_LIT:0>]<EOL>sound += self.features[base_sound.type][feature][<NUM_LIT:0>]<EOL><DEDENT>grapheme += base_sound.grapheme<EOL>sound += base_sound.s<EOL>for dia in [EMPTY + p for p in post]:<EOL><INDENT>feature = self.diacritics[base_sound.type].get(dia, {})<EOL>if not feature:<EOL><INDENT>return UnknownSound(  <EOL>grapheme=nstring, source=string, ts=self)<EOL><DEDENT>features[self._feature_values[feature]] = feature<EOL>grapheme += dia[<NUM_LIT:1>]<EOL>sound += self.features[base_sound.type][feature][<NUM_LIT:1>]<EOL><DEDENT>features['<STR_LIT>'] = sound<EOL>new_sound = self.sound_classes[base_sound.type](**features)<EOL>if text_type(new_sound) != sound:<EOL><INDENT>new_sound.alias = True<EOL><DEDENT>if grapheme != sound:<EOL><INDENT>new_sound.alias = True<EOL>new_sound.grapheme = grapheme<EOL><DEDENT>return new_sound<EOL>

Parse a string and return its features.

        :param string: A one-symbol string in NFD

        Notes
        -----
        Strategy is rather simple: we determine the base part of a string and
        then search left and right of this part for the additional features as
        expressed by the diacritics. Fails if a segment has more than one basic
        part.


if isinstance(sound, (Marker, UnknownSound)):<EOL><INDENT>return False<EOL><DEDENT>s1 = ts[sound.name]<EOL>s2 = ts[sound.s]<EOL>return s1.name == s2.name and s1.s == s2.s<EOL>

Check the consistency of a given transcription system conversino


return self.ts.id == other.ts.id and self.grapheme == other.grapheme<EOL>

In the absence of features, we consider symbols equal, if they belong to the same
system and are represented by the same grapheme.


try:<EOL><INDENT>return '<STR_LIT>'.join(unicodedata.name(ss) for ss in self.__unicode__())<EOL><DEDENT>except TypeError:<EOL><INDENT>return '<STR_LIT:->'<EOL><DEDENT>except ValueError:<EOL><INDENT>return '<STR_LIT:?>'<EOL><DEDENT>

Return unicode name(s) for a character set.


return '<STR_LIT:U+0020>'.join('<STR_LIT>' + ('<STR_LIT>' + hex(ord(x))[<NUM_LIT:2>:])[-<NUM_LIT:4>:] for x in self.__unicode__())<EOL>

Return unicode codepoint(s) for a grapheme.


<EOL>if not self.generated:<EOL><INDENT>if not self.alias and self.grapheme in self.ts.sounds:<EOL><INDENT>return self.grapheme<EOL><DEDENT>elif self.alias and self.featureset in self.ts.features:<EOL><INDENT>return text_type(self.ts.features[self.featureset])<EOL><DEDENT>raise ValueError(<EOL>'<STR_LIT>'.format(self.grapheme))  <EOL><DEDENT>elements = [f for f in self._features() if f not in EXCLUDE_FEATURES] + [self.type]<EOL>base_str = self.base or '<STR_LIT>'<EOL>base_graphemes = []<EOL>while elements:<EOL><INDENT>base = self.ts.features.get(frozenset(elements))<EOL>if base:<EOL><INDENT>base_graphemes.append(base.grapheme)<EOL><DEDENT>elements.pop(<NUM_LIT:0>)<EOL><DEDENT>base_str = base_graphemes[-<NUM_LIT:1>] if base_graphemes else base_str or '<STR_LIT>'<EOL>base_vals = {<EOL>self.ts._feature_values[elm] for elm in<EOL>self.ts.sounds[base_str].name.split('<STR_LIT:U+0020>')[:-<NUM_LIT:1>]} ifbase_str != '<STR_LIT>' else {}<EOL>out = []<EOL>for p in self._write_order['<STR_LIT>']:<EOL><INDENT>if p not in base_vals and getattr(self, p, '<STR_LIT>') in self._features():<EOL><INDENT>out.append(<EOL>norm(self.ts.features[self.type].get(getattr(self, p, '<STR_LIT>'), '<STR_LIT>')))<EOL><DEDENT><DEDENT>out.append(base_str)<EOL>for p in self._write_order['<STR_LIT>']:<EOL><INDENT>if p not in base_vals and getattr(self, p, '<STR_LIT>') in self._features():<EOL><INDENT>out.append(<EOL>norm(self.ts.features[self.type].get(getattr(self, p, '<STR_LIT>'), '<STR_LIT>')))<EOL><DEDENT><DEDENT>return '<STR_LIT>'.join(out)<EOL>

Return the reference representation of the sound.

Note
----
We first try to return the non-alias value in our data. If this fails,
we create the sound based on it's feature representation.


tbl = []<EOL>features = [<EOL>f for f in self._name_order if f not in self.ts.columns[self.type]]<EOL>if self.generated and self.s != self.source:<EOL><INDENT>tbl += [self.__unicode__() + '<STR_LIT>' + self.source]<EOL><DEDENT>else:<EOL><INDENT>tbl += [self.__unicode__()]<EOL><DEDENT>for name in self.ts.columns[self.type][<NUM_LIT:1>:]:<EOL><INDENT>if name != '<STR_LIT>' and name != '<STR_LIT>':<EOL><INDENT>tbl += [getattr(self, name) or '<STR_LIT>']<EOL><DEDENT>elif name == '<STR_LIT>':<EOL><INDENT>tbl += ['<STR_LIT:+>' if getattr(self, name) else '<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>bundle = []<EOL>for f in features:<EOL><INDENT>val = getattr(self, f)<EOL>if val:<EOL><INDENT>bundle += ['<STR_LIT>'.format(f, val)]<EOL><DEDENT><DEDENT>tbl += ['<STR_LIT:U+002C>'.join(bundle)]<EOL><DEDENT><DEDENT>return tbl<EOL>

Returns the tabular representation of the sound as given in our data


return [self.grapheme, self.from_sound.name, self.to_sound.name]<EOL>

Overwrite the table attribute for complex sounds


sound = sound if isinstance(sound, Symbol) else self.system[sound]<EOL>if sound.name in self.data:<EOL><INDENT>return self.data[sound.name]['<STR_LIT>']<EOL><DEDENT>if not sound.type == '<STR_LIT>':<EOL><INDENT>if sound.type in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>return self.resolve_sound(sound.from_sound)<EOL><DEDENT>name = [<EOL>s for s in sound.name.split('<STR_LIT:U+0020>') if<EOL>self.system._feature_values.get(s, '<STR_LIT>') not in<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']]<EOL>while len(name) >= <NUM_LIT:4>:<EOL><INDENT>sound = self.system.get('<STR_LIT:U+0020>'.join(name))<EOL>if sound and sound.name in self.data:<EOL><INDENT>return self.resolve_sound(sound)<EOL><DEDENT>name.pop(<NUM_LIT:0>)<EOL><DEDENT><DEDENT>raise KeyError("<STR_LIT>")<EOL>


return {p.strip() for p in (header or '<STR_LIT>').split('<STR_LIT:;>')}<EOL>

Normalize a CSP header for consistent comparisons.


self.limiter = None<EOL>self.talisman = None<EOL>if app:<EOL><INDENT>self.init_app(app, **kwargs)<EOL><DEDENT>

r"""Extension initialization.

        :param app: An instance of :class:`~flask.Flask`.
        :param \**kwargs: Keyword arguments are passed to ``init_app`` method.


<EOL>self.init_config(app)<EOL>self.limiter = Limiter(app, key_func=get_ipaddr)<EOL>if app.config['<STR_LIT>']:<EOL><INDENT>self.talisman = Talisman(<EOL>app, **app.config.get('<STR_LIT>', {})<EOL>)<EOL><DEDENT>if app.config['<STR_LIT>']:<EOL><INDENT>blueprint = Blueprint('<STR_LIT>', __name__)<EOL>@blueprint.route('<STR_LIT>')<EOL>def ping():<EOL><INDENT>"""<STR_LIT>"""<EOL>return '<STR_LIT:OK>'<EOL><DEDENT>ping.talisman_view_options = {'<STR_LIT>': False}<EOL>app.register_blueprint(blueprint)<EOL><DEDENT>requestid_header = app.config.get('<STR_LIT>')<EOL>if requestid_header:<EOL><INDENT>@app.before_request<EOL>def set_request_id():<EOL><INDENT>"""<STR_LIT>"""<EOL>request_id = request.headers.get(requestid_header)<EOL>if request_id:<EOL><INDENT>g.request_id = request_id[:<NUM_LIT:200>]<EOL><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>from flask_debugtoolbar import DebugToolbarExtension<EOL>app.extensions['<STR_LIT>'] = DebugToolbarExtension(app)<EOL><DEDENT>except ImportError:<EOL><INDENT>app.logger.debug('<STR_LIT>')<EOL><DEDENT>app.extensions['<STR_LIT>'] = self<EOL>

Initialize application object.

        :param app: An instance of :class:`~flask.Flask`.


config_apps = ['<STR_LIT>', '<STR_LIT>']<EOL>flask_talisman_debug_mode = ["<STR_LIT>"]<EOL>for k in dir(config):<EOL><INDENT>if any([k.startswith(prefix) for prefix in config_apps]):<EOL><INDENT>app.config.setdefault(k, getattr(config, k))<EOL><DEDENT><DEDENT>if app.config['<STR_LIT>']:<EOL><INDENT>app.config.setdefault('<STR_LIT>', {})<EOL>headers = app.config['<STR_LIT>']<EOL>if headers.get('<STR_LIT>') != {}:<EOL><INDENT>headers.setdefault('<STR_LIT>', {})<EOL>csp = headers['<STR_LIT>']<EOL>if csp.get('<STR_LIT>') != []:<EOL><INDENT>csp.setdefault('<STR_LIT>', [])<EOL>csp['<STR_LIT>'] += flask_talisman_debug_mode<EOL><DEDENT><DEDENT><DEDENT>

Initialize configuration.

        :param app: An instance of :class:`~flask.Flask`.


if current_app:<EOL><INDENT>return current_app.config.get('<STR_LIT>', None)<EOL><DEDENT>


<EOL>local_templates_path = os.path.join(app.instance_path, '<STR_LIT>')<EOL>if os.path.exists(local_templates_path):<EOL><INDENT>app.jinja_loader = ChoiceLoader([<EOL>FileSystemLoader(local_templates_path),<EOL>app.jinja_loader,<EOL>])<EOL><DEDENT>app.jinja_options = dict(<EOL>app.jinja_options,<EOL>cache_size=<NUM_LIT:1000>,<EOL>bytecode_cache=BytecodeCache(app)<EOL>)<EOL>invenio_config_loader(app, **kwargs_config)<EOL>

Configuration loader.

    Adds support for loading templates from the Flask application's instance
    folder (``<instance_folder>/templates``).


try:<EOL><INDENT>pkg_resources.get_distribution('<STR_LIT>')<EOL>from invenio_files_rest.app import Flask as FlaskBase<EOL><DEDENT>except pkg_resources.DistributionNotFound:<EOL><INDENT>from flask import Flask as FlaskBase<EOL><DEDENT>class Request(TrustedHostsMixin, FlaskBase.request_class):<EOL><INDENT>pass<EOL><DEDENT>class Flask(FlaskBase):<EOL><INDENT>request_class = Request<EOL><DEDENT>return Flask<EOL>

Create Flask application class.

    Invenio-Files-REST needs to patch the Werkzeug form parsing in order to
    support streaming large file uploads. This is done by subclassing the Flask
    application class.


if type(option) is str:<EOL><INDENT>option=option.lower()<EOL><DEDENT>if type(value) is str:<EOL><INDENT>value=value.lower()<EOL><DEDENT>return (option,value)<EOL>

Enforces lower case options and option values where appropriate


try:<EOL><INDENT>value=value.__str__()<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>return (option,value)<EOL>

Converts any values to strings when appropriate


if type(value) is str:<EOL><INDENT>try:<EOL><INDENT>value=float(value)<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return (option,value)<EOL>

Converts string values to floats when appropriate


if type(value) is str:<EOL><INDENT>if value.lower() == '<STR_LIT:true>':<EOL><INDENT>value=True<EOL><DEDENT>elif value.lower() == '<STR_LIT:false>':<EOL><INDENT>value=False<EOL><DEDENT><DEDENT>return (option,value)<EOL>

Converts string values to booleans when appropriate


self._rules = rules or []<EOL>self._dict = config or {}<EOL>self.enforce_rules()<EOL>

:param config: Old configuration
:type config: behaving like dict or Config


Set options but only if they haven't already been defined

:param defaults: Options and values to be set
:type defaults: behaving like _dict or Config


fork=deepcopy(self)<EOL>self[name]=fork<EOL>return fork<EOL>

Create fork and store it in current instance


sub = Config()<EOL>self[name]=sub<EOL>return sub<EOL>

Return empty Config object and store this object in current Config


N=len(times)<EOL>times = times.flatten()<EOL>p0 = np.log(S0)<EOL>if dW is None:<EOL><INDENT>dW=np.sqrt(times[<NUM_LIT:1>:]-times[:-<NUM_LIT:1>])[None,:,None]*np.random.normal(size=(M,N-<NUM_LIT:1>,d))<EOL><DEDENT>if np.squeeze(sigma).ndim<=<NUM_LIT:1>:<EOL><INDENT>dF = sigma*dW<EOL>ito_correction = np.squeeze(sigma**<NUM_LIT:2>/<NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>dF = np.einsum('<STR_LIT>',sigma,dW)<EOL>ito_correction = np.sum(sigma**<NUM_LIT:2>,<NUM_LIT:1>)/<NUM_LIT:2><EOL><DEDENT>drift  = (r-ito_correction)*times[None,:,None]<EOL>diffusion = integral(dF=dF,axis=<NUM_LIT:1>,cumulative = True)<EOL>return np.exp(p0 + drift + diffusion)<EOL>

Return M Euler-Maruyama sample paths with N time steps of S_t, where
    dS_t = S_t*r*dt+S_t*sigma*dW_t
    S(0)=S0

:rtype: M x N x d array


d_nu = <NUM_LIT:1> if nu_1d else d<EOL>nu = np.zeros((M,len(times),d_nu))<EOL>S = np.zeros((M,len(times),d))<EOL>nu[:,<NUM_LIT:0>,:] = nu0<EOL>S[:,<NUM_LIT:0>,:] = S0<EOL>if <NUM_LIT:2>*kappa*theta<=xi**<NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>test = np.std(np.diff(times.flatten())) <EOL>if test><NUM_LIT>:<EOL><INDENT>raise ValueError<EOL><DEDENT>dt = times[<NUM_LIT:1>]-times[<NUM_LIT:0>]<EOL>N = len(times)<EOL>if d == <NUM_LIT:1>:<EOL><INDENT>if np.array(rho).size ==<NUM_LIT:1>:<EOL><INDENT>rho = np.array([[<NUM_LIT:1>,rho],[rho,<NUM_LIT:1>]])<EOL><DEDENT><DEDENT>chol = np.linalg.cholesky(rho)<EOL>dW = np.sqrt(dt)*np.einsum('<STR_LIT>',chol,np.random.normal(size=(M,N-<NUM_LIT:1>,d+d_nu)))<EOL>for i in range(<NUM_LIT:1>,N):<EOL><INDENT>dt = times[i]-times[i-<NUM_LIT:1>]<EOL>nu[:,i,:] = np.abs(nu[:,i-<NUM_LIT:1>,:] + kappa*(theta-nu[:,i-<NUM_LIT:1>,:])*dt+xi*np.sqrt(nu[:,i-<NUM_LIT:1>,:])*dW[:,i-<NUM_LIT:1>,d:])<EOL><DEDENT>S = S0*np.exp(integral(np.sqrt(nu),dF = dW[:,:,:d],axis=<NUM_LIT:1>,cumulative = True)+integral(mu - <NUM_LIT:0.5>*nu,F = times,axis=<NUM_LIT:1>,trapez=False,cumulative = True))<EOL>return np.concatenate((S,nu),axis=-<NUM_LIT:1>)<EOL>

Return M Euler-Maruyama sample paths with N time steps of (S_t,v_t), where
    (S_t,v_t) follows the Heston model of mathematical finance

:rtype: M x N x d array


def fBrown(H,T,N,M,dW = None,cholesky = False):

alpha = <NUM_LIT:0.5>-H<EOL>times = np.linspace(<NUM_LIT:0>, T, N)<EOL>dt = T/(N-<NUM_LIT:1>)<EOL>if cholesky:<EOL><INDENT>if dW is not None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>times = times[<NUM_LIT:1>:]<EOL>tdt = times/np.reshape(times,(-<NUM_LIT:1>,<NUM_LIT:1>))<EOL>tdt[np.tril_indices(N-<NUM_LIT:1>,-<NUM_LIT:1>)]=<NUM_LIT:0><EOL>cov = np.reshape(times,(-<NUM_LIT:1>,<NUM_LIT:1>))**(<NUM_LIT:1>-<NUM_LIT:2>*alpha)*(<NUM_LIT:1>/(<NUM_LIT:1>-alpha))*(tdt-<NUM_LIT:1>)**(-alpha)*scipy.special.hyp2f1(alpha,<NUM_LIT:1>-alpha,<NUM_LIT:2>-alpha,<NUM_LIT:1>/(<NUM_LIT:1>-tdt))<EOL>cov[<NUM_LIT:0>,:] = <NUM_LIT:0><EOL>np.fill_diagonal(cov,times**(<NUM_LIT:1>-<NUM_LIT:2>*alpha)/(<NUM_LIT:1>-<NUM_LIT:2>*alpha))<EOL>cov[np.tril_indices(N-<NUM_LIT:1>,-<NUM_LIT:1>)] = cov.T[np.tril_indices(N-<NUM_LIT:1>,-<NUM_LIT:1>)]<EOL>L = scipy.linalg.cholesky(cov)<EOL>return np.concatenate((np.zeros((<NUM_LIT:1>,M)),L.T@np.random.normal(size=(N-<NUM_LIT:1>,M))))<EOL><DEDENT>if dW is None:<EOL><INDENT>dW = np.sqrt(dt)*np.random.normal(size=(N-<NUM_LIT:1>,M))<EOL><DEDENT>if H == <NUM_LIT:0.5>:<EOL><INDENT>return integral(dF = dW,cumulative = True)  <EOL><DEDENT>a = <NUM_LIT:1>/dt/(<NUM_LIT:1>-alpha)*((T-times[N-<NUM_LIT:2>::-<NUM_LIT:1>])**(<NUM_LIT:1>-alpha)-(T-times[:<NUM_LIT:0>:-<NUM_LIT:1>])**(<NUM_LIT:1>-alpha))<EOL>out = toeplitz_multiplication(a,np.zeros_like(a),dW[::-<NUM_LIT:1>])[::-<NUM_LIT:1>]<EOL>out -=a[<NUM_LIT:0>]*dW<EOL>cov = np.array([[ dt**(<NUM_LIT:1>-<NUM_LIT:2>*alpha)/(<NUM_LIT:1>-<NUM_LIT:2>*alpha),dt**(<NUM_LIT:1>-alpha)/(<NUM_LIT:1>-alpha)],[dt**(<NUM_LIT:1>-alpha)/(<NUM_LIT:1>-alpha),dt]])<EOL>var = cov[<NUM_LIT:0>,<NUM_LIT:0>]-cov[<NUM_LIT:0>,<NUM_LIT:1>]**<NUM_LIT:2>/cov[<NUM_LIT:1>,<NUM_LIT:1>]<EOL>out += cov[<NUM_LIT:0>,<NUM_LIT:1>]/cov[<NUM_LIT:1>,<NUM_LIT:1>]*dW <EOL>out += np.sqrt(var)*np.random.normal(size = (N-<NUM_LIT:1>,M))<EOL>out = np.concatenate((np.zeros((<NUM_LIT:1>,M)),out))<EOL>return out<EOL>

Sample fractional Brownian motion with differentiability index H
on interval [0,T] (H=1/2 yields standard Brownian motion)

:param H: Differentiability, larger than 0
:param T: Final time
:param N: Number of time steps
:param M: Number of samples
:param dW: Driving noise, optional


def r_bergomi(H,T,eta,xi,rho,S0,r,N,M,dW=None,dW_orth=None,cholesky = False,return_v=False):

times = np.linspace(<NUM_LIT:0>, T, N)<EOL>dt = T/(N-<NUM_LIT:1>)<EOL>times = np.reshape(times,(-<NUM_LIT:1>,<NUM_LIT:1>))<EOL>if dW is None:<EOL><INDENT>dW = np.sqrt(dt)*np.random.normal(size=(N-<NUM_LIT:1>,M))<EOL><DEDENT>if dW_orth is None:<EOL><INDENT>dW_orth = np.sqrt(dt)*np.random.normal(size=(N-<NUM_LIT:1>,M))<EOL><DEDENT>dZ = rho*dW+np.sqrt(<NUM_LIT:1>-rho**<NUM_LIT:2>)*dW_orth<EOL>Y = eta*np.sqrt(<NUM_LIT:2>*H)*fBrown(H,T,N,M,dW =dW,cholesky = cholesky)<EOL>v = xi*np.exp(Y-<NUM_LIT:0.5>*(eta**<NUM_LIT:2>)*times**(<NUM_LIT:2>*H))<EOL>S = S0*np.exp(integral(np.sqrt(v),dF = dZ,axis=<NUM_LIT:0>,cumulative = True)+integral(r - <NUM_LIT:0.5>*v,F = times,axis=<NUM_LIT:0>,trapez=False,cumulative = True))<EOL>if return_v:<EOL><INDENT>return np.array([S,v]).T<EOL><DEDENT>else:<EOL><INDENT>return np.array([S]).T<EOL><DEDENT>

Return M Euler-Maruyama sample paths with N time steps of (S_t,v_t), where
    (S_t,v_t) follows the rBergomi model of mathematical finance

:rtype: M x N x d array


subdirs = []<EOL>def prepare_string(s):<EOL><INDENT>s = str(s)<EOL>s = re.sub('<STR_LIT>'+f"<STR_LIT>",'<STR_LIT:_>',s)<EOL>if into=='<STR_LIT:file>':<EOL><INDENT>s = s.replace('<STR_LIT:_>', '<STR_LIT:U+0020>')<EOL><DEDENT>if '<STR_LIT:U+0020>' in s:<EOL><INDENT>s = s.title()<EOL>s = s.replace('<STR_LIT:U+0020>','<STR_LIT>')<EOL><DEDENT>return s<EOL><DEDENT>if isinstance(keywords,set):<EOL><INDENT>keywords_list = sorted(keywords)<EOL>for property in keywords_list:<EOL><INDENT>subdirs.append(prepare_string(property))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>keywords_list = sorted(keywords.items())<EOL>for property,value in keywords_list:  <EOL><INDENT>if Bool.valid(value):<EOL><INDENT>subdirs.append(('<STR_LIT>' if value else ('<STR_LIT>' if into=='<STR_LIT:path>' else '<STR_LIT>'))+prepare_string(property))<EOL><DEDENT>elif (Float|Integer).valid(value):<EOL><INDENT>subdirs.append('<STR_LIT>'.format(prepare_string(property),prepare_string(value)))<EOL><DEDENT>else:<EOL><INDENT>subdirs.append('<STR_LIT>'.format(prepare_string(property),'<STR_LIT:_>' if into == '<STR_LIT:path>' else '<STR_LIT>',prepare_string(value)))<EOL><DEDENT><DEDENT><DEDENT>if into == '<STR_LIT:path>':<EOL><INDENT>out = os.path.join(*subdirs)<EOL><DEDENT>else:<EOL><INDENT>out = '<STR_LIT:_>'.join(subdirs)<EOL><DEDENT>return out<EOL>

turns keyword pairs into path or filename

if `into=='path'`, then keywords are separted by underscores, else keywords are used to create a directory hierarchy


if not path:<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>result = []<EOL>for root, __, files in os.walk(path):<EOL><INDENT>for name in files:<EOL><INDENT>if fnmatch.fnmatch(name if match_name else os.path.join(root,name),pattern):<EOL><INDENT>result.append(os.path.join(root, name))<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

https://stackoverflow.com/questions/1724693/find-a-file-in-python

WARNING: pattern is by default matched to entire path not to file names


if not path:<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>result = []<EOL>for root, __, __ in os.walk(path):<EOL><INDENT>match_against = os.path.basename(root) if match_name else root<EOL>try:<EOL><INDENT>does_match = pattern.match(match_against)<EOL><DEDENT>except AttributeError:<EOL><INDENT>does_match = fnmatch.fnmatch(match_against,pattern)<EOL><DEDENT>if does_match:<EOL><INDENT>result.append(root)<EOL><DEDENT><DEDENT>return result<EOL>

WARNING: pattern is matched to entire path, not directory names, unless
match_name = True


src_path = Path(source_dir).expanduser().resolve()<EOL>with ZipFile(zip_name, '<STR_LIT:w>', ZIP_DEFLATED) as zf:<EOL><INDENT>for file in src_path.rglob('<STR_LIT:*>'):<EOL><INDENT>path_in_zip = str(file.relative_to(src_path.parent))<EOL>if rename_source_dir != False:<EOL><INDENT>_,tail = path_in_zip.split(os.sep,<NUM_LIT:1>)<EOL>path_in_zip=os.sep.join([rename_source_dir,tail])<EOL><DEDENT>zf.write(str(file.resolve()), path_in_zip)<EOL><DEDENT><DEDENT>


has = []<EOL>return [x for x in seq if not (x in has or has.append(x))]<EOL>


dict.__init__(self)<EOL>if init_dict:<EOL><INDENT>for key in init_dict:<EOL><INDENT>self[key] = init_dict[key]<EOL><DEDENT><DEDENT>

:param init_dict: Initial state of function
:type init_dict: Dictionary whos values support addition and scalar multiplication


for x in X:<EOL><INDENT>self[x] = None<EOL><DEDENT>

Expand domain

:param X: New elements of domain
:type X: Iterable


F = RFunction()<EOL>for key in self.keys():<EOL><INDENT>F[key] = self[key]<EOL><DEDENT>for key in other.keys():<EOL><INDENT>if key in F.keys():<EOL><INDENT>F[key] += other[key]<EOL><DEDENT>else:<EOL><INDENT>F[key] = other[key]<EOL><DEDENT><DEDENT>return F<EOL>

Vector space operation: Add two real-valued functions


if other == <NUM_LIT:0>:<EOL><INDENT>return self<EOL><DEDENT>else: <EOL><INDENT>return self.__add__(other)<EOL><DEDENT>

When iterables of functions are added, the first function is added to 0
using __radd__


F = RFunction()<EOL>for key in self.keys():<EOL><INDENT>F[key] = other * self[key]<EOL><DEDENT>return F<EOL>

Vector space operation: Multiply real-valued function with real


self.default = default<EOL>super().__init__()<EOL>

:param default: Default values for unknown keys
:type default: Function


rejection_subreason = None<EOL>if spec is None:<EOL><INDENT>return arg<EOL><DEDENT>try:<EOL><INDENT>return spec._validate(arg)<EOL><DEDENT>except Exception as e:<EOL><INDENT>rejection_subreason = e<EOL><DEDENT>try:<EOL><INDENT>lenience = spec.lenience<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>for level in range(<NUM_LIT:1>, lenience + <NUM_LIT:1>):<EOL><INDENT>temp = None<EOL>try:<EOL><INDENT>temp = spec.forgive(arg=arg, level=level)<EOL><DEDENT>except Exception:<EOL><INDENT>pass  <EOL><DEDENT>if temp is not None and temp is not arg:<EOL><INDENT>arg = temp<EOL>try:<EOL><INDENT>return spec._validate(arg)<EOL><DEDENT>except Exception as e:<EOL><INDENT>rejection_subreason = e<EOL><DEDENT><DEDENT><DEDENT><DEDENT>rejection_reason = '<STR_LIT>'.format(arg, spec)<EOL>rejection_subreason = '<STR_LIT>'.format(rejection_subreason.__class__.__name__, rejection_subreason) if rejection_subreason is not None else '<STR_LIT>'<EOL>raise ValidationError(rejection_reason + rejection_subreason)<EOL>

Make sure `arg` adheres to specification

:param arg: Anything
:param spec: Specification
:type spec: Specification

:return: Validated object


def _validate_many(args, specs, defaults,passed_conditions,value_conditions,<EOL>allow_unknowns,unknowns_spec):

validated_args = builtins.dict() <EOL>passed_but_not_specified = set(args.keys()) - set(specs.keys())<EOL>if passed_but_not_specified:<EOL><INDENT>if not allow_unknowns:<EOL><INDENT>raise ValueError(('<STR_LIT>' + <EOL>'<STR_LIT>'.format(passed_but_not_specified)))<EOL><DEDENT>else:<EOL><INDENT>for arg in passed_but_not_specified:<EOL><INDENT>if unknowns_spec is not None:<EOL><INDENT>specs[arg] = unknowns_spec<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if passed_conditions:<EOL><INDENT>validate(args, Dict(passed_conditions=passed_conditions))<EOL><DEDENT>for arg in specs:<EOL><INDENT>if (not arg in args) or NotPassed(args[arg]):<EOL><INDENT>if arg in defaults:<EOL><INDENT>if isinstance(defaults[arg],DefaultGenerator):<EOL><INDENT>validated_args[arg] = defaults[arg]()<EOL><DEDENT>else:<EOL><INDENT>validated_args[arg] = defaults[arg]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validated_args[arg] = NotPassed<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validated_args[arg] = validate(args[arg], specs[arg])<EOL><DEDENT><DEDENT>if value_conditions:<EOL><INDENT>validated_args = validate(validated_args, value_conditions)<EOL><DEDENT>return validated_args<EOL>

Similar to validate but validates multiple objects at once, each with their own specification.

Fill objects that were specified but not provided with NotPassed or default values
Apply `value_condition` to object dictionary as a whole


def integral(A=None,dF=None,F=None,axis = <NUM_LIT:0>,trapez = False,cumulative = False):

ndim = max(v.ndim for v in (A,dF,F) if v is not None)<EOL>def broadcast(x):<EOL><INDENT>new_shape = [<NUM_LIT:1>]*ndim<EOL>new_shape[axis] = -<NUM_LIT:1><EOL>return np.reshape(x,new_shape)<EOL><DEDENT>if F is not None:<EOL><INDENT>assert(dF is None)<EOL>if F.ndim<ndim:<EOL><INDENT>F = broadcast(F)<EOL><DEDENT>N = F.shape[axis]<EOL>dF = F.take(indices = range(<NUM_LIT:1>,N),axis = axis)-F.take(indices = range(N-<NUM_LIT:1>),axis = axis)<EOL><DEDENT>elif dF is not None:<EOL><INDENT>if dF.ndim<ndim:<EOL><INDENT>dF = broadcast(dF)<EOL><DEDENT>N = dF.shape[axis]+<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>if A.ndim<ndim:<EOL><INDENT>A = broadcast(A)<EOL><DEDENT>N = A.shape[axis]<EOL><DEDENT>if A is not None:<EOL><INDENT>if trapez:<EOL><INDENT>midA = (A.take(indices = range(<NUM_LIT:1>,N),axis = axis)+A.take(indices = range(N-<NUM_LIT:1>),axis = axis))/<NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>midA = A.take(indices=range(N-<NUM_LIT:1>),axis=axis)<EOL><DEDENT>if dF is not None:<EOL><INDENT>dY = midA*dF<EOL><DEDENT>else:<EOL><INDENT>dY = midA<EOL><DEDENT><DEDENT>else:<EOL><INDENT>dY = dF<EOL><DEDENT>pad_shape = list(dY.shape)<EOL>pad_shape[axis] = <NUM_LIT:1><EOL>pad = np.zeros(pad_shape)<EOL>if cumulative:<EOL><INDENT>return np.concatenate((pad,np.cumsum(dY,axis = axis)),axis = axis)<EOL><DEDENT>else:<EOL><INDENT>return np.sum(dY,axis = axis)<EOL><DEDENT>

Turns an array A of length N (the function values in N points)
and an array dF of length N-1 (the masses of the N-1 intervals)
into an array of length N (the integral \int A dF at N points, with first entry 0)

:param A: Integrand (optional, default ones, length N)
:param dF: Integrator (optional, default ones, length N-1)
:param F: Alternative to dF (optional, length N)
:param trapez: Use trapezoidal rule (else left point)


a = np.reshape(a,(-<NUM_LIT:1>))<EOL>b = np.reshape(b,(-<NUM_LIT:1>))<EOL>n = len(a)<EOL>c = np.concatenate((a[[<NUM_LIT:0>]],b[<NUM_LIT:1>:],np.zeros(<NUM_LIT:1>),a[-<NUM_LIT:1>:<NUM_LIT:0>:-<NUM_LIT:1>]))<EOL>p = ifft(fft(c)*fft(v.T,n=<NUM_LIT:2>*n)).T<EOL>if np.all(np.isreal(a)) and np.all(np.isreal(b)) and np.all(np.isreal(v)):<EOL><INDENT>return np.real(p[:n])<EOL><DEDENT>else:<EOL><INDENT>return p[:n]<EOL><DEDENT>

Multiply Toeplitz matrix with first row a and first column b with vector v

Normal matrix multiplication would require storage and runtime O(n^2);
embedding into a circulant matrix and using FFT yields O(log(n)n)


def grid_evaluation(X, Y, f,vectorized=True):

XX = np.reshape(np.concatenate([X[..., None], Y[..., None]], axis=<NUM_LIT:2>), (X.size, <NUM_LIT:2>), order='<STR_LIT:C>')<EOL>if vectorized:<EOL><INDENT>ZZ = f(XX)<EOL><DEDENT>else:<EOL><INDENT>ZZ = np.array([f(x) for x in XX])<EOL><DEDENT>return np.reshape(ZZ, X.shape, order='<STR_LIT:C>')<EOL>

Evaluate function on given grid and return values in grid format

Assume X and Y are 2-dimensional arrays containing x and y coordinates,
respectively, of a two-dimensional grid, and f is a function that takes
1-d arrays with two entries. This function evaluates f on the grid points
described by X and Y and returns another 2-dimensional array of the shape
of X and Y that contains the values of f.

:param X: 2-dimensional array of x-coordinates
:param Y: 2-dimensional array of y-coordinates
:param f: function to be evaluated on grid
:param vectorized: `f` can handle arrays of inputs
:return: 2-dimensional array of values of f


_, _, V = np.linalg.svd(np.array([v]))<EOL>return V[<NUM_LIT:1>:]<EOL>

Return orthonormal basis of complement of vector.

:param v: 1-dimensional numpy array
:return: Matrix whose .dot() computes coefficients w.r.t. an orthonormal basis of the complement of v
    (i.e. whose row vectors form an orthonormal basis of the complement of v)


if len(values) == <NUM_LIT:1>:<EOL><INDENT>return values[<NUM_LIT:0>]<EOL><DEDENT>if len(values) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>values = [float(value) for value in values]<EOL>indices_sorted = np.argsort(values)<EOL>values = [values[ind] for ind in indices_sorted]<EOL>weights = [weights[ind] for ind in indices_sorted]<EOL>total_weight = sum(weights)<EOL>below_weight = <NUM_LIT:0><EOL>i = -<NUM_LIT:1><EOL>while below_weight < total_weight / <NUM_LIT:2>:<EOL><INDENT>i += <NUM_LIT:1><EOL>below_weight += weights[i]<EOL><DEDENT>return values[i]<EOL>

Returns element such that sum of weights below and above are (roughly) equal

:param values: Values whose median is sought
:type values: List of reals
:param weights: Weights of each value
:type weights: List of positive reals
:return: value of weighted median
:rtype: Real


def __init__(self, wrapped, tight=False):

self.__tight = tight<EOL>self.__wrapped = array(wrapped) if tight else wrapped<EOL>self.__hash = int(sha1(wrapped.view(uint8)).hexdigest(), <NUM_LIT:16>)<EOL>

r'''Creates a new hashable object encapsulating an ndarray.

            wrapped
                The wrapped ndarray.

            tight
                Optional. If True, a copy of the input ndaray is created.
                Defaults to False.


if self.__tight:<EOL><INDENT>return array(self.__wrapped)<EOL><DEDENT>return self.__wrapped<EOL>

r'''Returns the encapsulated ndarray.

            If the wrapper is "tight", a copy of the encapsulated ndarray is
            returned. Otherwise, the encapsulated ndarray itself is returned.


for i in range(int(<NUM_LIT> * value)):<EOL><INDENT>_ = <NUM_LIT:2> ** (i / value)<EOL><DEDENT>

time.sleep() substitute
Keep busy for some time (very roughly and depending on machine, `value` is in ms)

:param value: Time
:type value: Number


@validate_args(warnings=False)<EOL>def EasyHPC(backend:In('<STR_LIT>', '<STR_LIT>')|Function='<STR_LIT>',<EOL>n_tasks:In('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:count>')='<STR_LIT>',<EOL>n_results:In('<STR_LIT>', '<STR_LIT>')='<STR_LIT>',<EOL>aux_output:Bool=True,  <EOL>reduce:Function=None,<EOL>split_job=NotPassed,<EOL>parallel = True,<EOL>method = None,<EOL>pool = None<EOL>):

self = argparse.Namespace()<EOL>direct_call =  (~String&Function).valid(backend)<EOL>if direct_call:<EOL><INDENT>f = backend<EOL>backend = '<STR_LIT>'<EOL><DEDENT>if backend == '<STR_LIT>': <EOL><INDENT>self.processor = _MPI_processor<EOL>self.finalizer = _MPI_finalizer<EOL><DEDENT>if backend == '<STR_LIT>':<EOL><INDENT>self.processor = _MP_processor<EOL>self.finalizer = None<EOL><DEDENT>self.info = argparse.Namespace()<EOL>self.info.n_tasks = n_tasks<EOL>self.info.n_results = n_results<EOL>self.info.parallel = parallel<EOL>self.info.reduce = reduce<EOL>self.info.wrap_MPI = False<EOL>self.info.aux_output = aux_output <EOL>self.info.method = method<EOL>self.info.pool = pool or Pool()<EOL>self.info.split_job = split_job<EOL>if self.info.n_tasks == '<STR_LIT>':<EOL><INDENT>if self.info.n_results == '<STR_LIT>':<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if NotPassed(self.info.split_job):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>if direct_call:<EOL><INDENT>def _lam(*args,**kwargs):<EOL><INDENT>return _MultiProcessorWrapper_call(args,kwargs,f,self.processor,self.finalizer,self.info)<EOL><DEDENT>return _lam<EOL><DEDENT>return lambda f: _easy_hpc_call(f,self)<EOL>

:param n_tasks: How many tasks does the decorated function handle?
:param n_results: If the decorated function handles many tasks at once, are the results reduced (n_results = 'one') or not (as many results as tasks)?
:param reduce: Function that reduces multiple outputs to a single output
:param splitjob: Function that converts an input (to the decorated function) that represents one large job to two smaller jobs

NOTE: don't turn this into a class, you'll run into strange pickling errors


def plot_indices(mis, dims=None, weights=None, groups=<NUM_LIT:1>,legend = True,index_labels=None, colors = None,axis_labels = None,size_exponent=<NUM_LIT:0.1>,ax=None):

if weights is None:<EOL><INDENT>weights = {mi: <NUM_LIT:1> for mi in mis}<EOL><DEDENT>if Function.valid(weights):<EOL><INDENT>weights = {mi:weights(mi) for mi in mis}<EOL><DEDENT>values = list(weights.values())<EOL>if Integer.valid(groups):<EOL><INDENT>N_g = groups<EOL>groups = [[mi for mi in mis if (weights[mi] > np.percentile(values, <NUM_LIT:100>/groups*g) or g==<NUM_LIT:0>) and weights[mi] <= np.percentile(values, <NUM_LIT:100>/groups*(g+<NUM_LIT:1>))] for g in range(N_g)]<EOL>group_names = ['<STR_LIT>'.format(<NUM_LIT:100>/N_g*(N_g-i-<NUM_LIT:1>),<NUM_LIT:100>/N_g*(N_g-i)) for i in reversed(range(N_g))]<EOL><DEDENT>else:<EOL><INDENT>if Function.valid(groups):<EOL><INDENT>groups = {mi:groups(mi) for mi in mis}<EOL><DEDENT>group_names = unique(list(groups.values()))<EOL>groups = [[mi for mi in mis if groups[mi]==name] for name in group_names]<EOL>N_g = len(group_names)<EOL><DEDENT>if colors is None: <EOL><INDENT>colors = matplotlib.cm.rainbow(np.linspace(<NUM_LIT:0>, <NUM_LIT:1>, N_g))  <EOL><DEDENT>if Dict.valid(mis):<EOL><INDENT>if index_labels is None or weights is None:<EOL><INDENT>temp = list(mis.keys())<EOL>if (List|Tuple).valid(temp[<NUM_LIT:0>]):<EOL><INDENT>if not (index_labels is None and weights is None):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>weights = {mi:mis[mi][<NUM_LIT:0>] for mi in mis}<EOL>index_labels=  {mi:mis[mi][<NUM_LIT:1>] for mi in mis}<EOL><DEDENT>else:<EOL><INDENT>if weights is None:<EOL><INDENT>weights = mis<EOL><DEDENT>else:<EOL><INDENT>index_labels = mis<EOL><DEDENT><DEDENT>mis = temp<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>if dims is None:<EOL><INDENT>try:<EOL><INDENT>dims = len(mis[<NUM_LIT:0>])<EOL><DEDENT>except TypeError:<EOL><INDENT>dims = sorted(list(set.union(*(set(mi.active_dims()) for mi in mis))))   <EOL><DEDENT><DEDENT>if len(dims) > <NUM_LIT:3>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if len(dims) < <NUM_LIT:1>:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL>return<EOL><DEDENT>if ax is None:<EOL><INDENT>fig = plt.figure() <EOL>if len(dims) == <NUM_LIT:3>:<EOL><INDENT>ax = fig.gca(projection='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>ax = fig.gca()<EOL><DEDENT><DEDENT>size_function = lambda mi: sum([weights[mi2] for mi2 in mis if mi.equal_mod(mi2, lambda dim: dim not in dims)]) <EOL>sizes = {mi: np.power(size_function(mi), size_exponent) for mi in mis}<EOL>for i,plot_indices in enumerate(groups):<EOL><INDENT>X = np.array([mi[dims[<NUM_LIT:0>]] for mi in plot_indices])<EOL>if len(dims) > <NUM_LIT:1>:<EOL><INDENT>Y = np.array([mi[dims[<NUM_LIT:1>]] for mi in plot_indices])<EOL><DEDENT>else:<EOL><INDENT>Y = np.array([<NUM_LIT:0> for mi in plot_indices])<EOL><DEDENT>if len(dims) > <NUM_LIT:2>:<EOL><INDENT>Z = np.array([mi[dims[<NUM_LIT:2>]] for mi in plot_indices])<EOL><DEDENT>else:<EOL><INDENT>Z = np.array([<NUM_LIT:0> for mi in plot_indices])   <EOL><DEDENT>sizes_plot = np.array([sizes[mi] for mi in plot_indices])<EOL>if weights:<EOL><INDENT>if len(dims) == <NUM_LIT:3>:<EOL><INDENT>ax.scatter(X, Y, Z, s = <NUM_LIT:50> * sizes_plot / max(sizes.values()), color=colors[i], alpha=<NUM_LIT:1>)            <EOL><DEDENT>else:<EOL><INDENT>ax.scatter(X, Y, s = <NUM_LIT:50> * sizes_plot / max(sizes.values()), color=colors[i], alpha=<NUM_LIT:1>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(dims) == <NUM_LIT:3>:<EOL><INDENT>ax.scatter(X, Y, Z,color = colors[i],alpha=<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>ax.scatter(X, Y,color=colors[i],alpha=<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if True:<EOL><INDENT>if len(dims)==<NUM_LIT:3>:<EOL><INDENT>axs='<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>axs='<STR_LIT>'<EOL><DEDENT>extents = np.array([getattr(ax, '<STR_LIT>'.format(dim))() for dim in axs])<EOL>sz = extents[:,<NUM_LIT:1>] - extents[:,<NUM_LIT:0>]<EOL>maxsize = max(abs(sz))<EOL>for dim in axs:<EOL><INDENT>getattr(ax, '<STR_LIT>'.format(dim))(<NUM_LIT:0>, maxsize)<EOL><DEDENT><DEDENT><DEDENT>if axis_labels is not None:<EOL><INDENT>ax.set_xlabel(axis_labels[<NUM_LIT:0>])<EOL>if len(dims)><NUM_LIT:1>:<EOL><INDENT>ax.set_ylabel(axis_labels[<NUM_LIT:1>])<EOL><DEDENT>if len(dims)><NUM_LIT:1>:<EOL><INDENT>ax.set_zlabel(axis_labels[<NUM_LIT:2>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ax.set_xlabel('<STR_LIT>' + str(dims[<NUM_LIT:0>])+'<STR_LIT:$>',size=<NUM_LIT:20>)<EOL>if len(dims) > <NUM_LIT:1>:<EOL><INDENT>ax.set_ylabel('<STR_LIT>' + str(dims[<NUM_LIT:1>])+'<STR_LIT:$>',size=<NUM_LIT:20>)<EOL><DEDENT>if len(dims) > <NUM_LIT:2>:<EOL><INDENT>ax.set_zlabel('<STR_LIT>' + str(dims[<NUM_LIT:2>])+'<STR_LIT:$>',size=<NUM_LIT:20>)<EOL><DEDENT>plt.grid()<EOL><DEDENT>x_coordinates = [mi[dims[<NUM_LIT:0>]] for mi in mis]<EOL>xticks=list(range(min(x_coordinates),max(x_coordinates)+<NUM_LIT:1>))<EOL>ax.set_xticks(xticks)<EOL>if len(dims)><NUM_LIT:1>:<EOL><INDENT>y_coordinates = [mi[dims[<NUM_LIT:1>]] for mi in mis]<EOL>ax.set_yticks(list(range(min(y_coordinates),max(y_coordinates)+<NUM_LIT:1>)))<EOL><DEDENT>if len(dims)><NUM_LIT:2>:<EOL><INDENT>z_coordinates = [mi[dims[<NUM_LIT:2>]] for mi in mis]<EOL>ax.set_zticks(list(range(min(z_coordinates),max(z_coordinates)+<NUM_LIT:1>)))<EOL><DEDENT>if index_labels:<EOL><INDENT>for mi in index_labels:<EOL><INDENT>ax.annotate('<STR_LIT>'.format(index_labels[mi]),xy=(mi[<NUM_LIT:0>],mi[<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>if legend and len(group_names)><NUM_LIT:1>:<EOL><INDENT>ax.legend([patches.Patch(color=color) for color in np.flipud(colors)],group_names)<EOL><DEDENT>return ax<EOL>

Plot multi-index set

:param mis: Multi-index set
:type mis: Iterable of SparseIndices
:param dims: Which dimensions to use for plotting
:type dims: List of integers.
:param weights: Weights associated with each multi-index
:type weights: Dictionary
:param quantiles: Number of groups plotted in different colors
:type quantiles: Integer>=1 or list of colors

TODO: exchange index_labels and dims, exchange quantiles and dims


def ezplot(f,xlim,ylim=None,ax = None,vectorized=True,N=None,contour = False,args=None,kwargs=None,dry_run=False,show=None,include_endpoints=False):

kwargs = kwargs or {}<EOL>args = args or []<EOL>d = <NUM_LIT:1> if ylim is None else <NUM_LIT:2><EOL>if ax is None:<EOL><INDENT>fig = plt.figure()<EOL>show = show if show is not None else True<EOL>ax = fig.gca() if (d==<NUM_LIT:1> or contour) else fig.gca(projection='<STR_LIT>')<EOL><DEDENT>if d == <NUM_LIT:1>:<EOL><INDENT>if N is None:<EOL><INDENT>N = <NUM_LIT:200><EOL><DEDENT>if include_endpoints:<EOL><INDENT>X = np.linspace(xlim[<NUM_LIT:0>],xlim[<NUM_LIT:1>],N)<EOL><DEDENT>else:<EOL><INDENT>L = xlim[<NUM_LIT:1>] - xlim[<NUM_LIT:0>]<EOL>X = np.linspace(xlim[<NUM_LIT:0>] + L / N, xlim[<NUM_LIT:1>] - L / N, N)<EOL><DEDENT>X = X.reshape((-<NUM_LIT:1>, <NUM_LIT:1>))<EOL>if vectorized:<EOL><INDENT>Z = f(X)<EOL><DEDENT>else:<EOL><INDENT>Z = np.array([f(x) for x in X])<EOL><DEDENT>if not dry_run:<EOL><INDENT>C = ax.plot(X, Z,*args,**kwargs)<EOL><DEDENT><DEDENT>elif d == <NUM_LIT:2>:<EOL><INDENT>if N is None:<EOL><INDENT>N = <NUM_LIT:30><EOL><DEDENT>T = np.zeros((N, <NUM_LIT:2>))<EOL>if include_endpoints:<EOL><INDENT>T[:,<NUM_LIT:0>]=np.linspace(xlim[<NUM_LIT:0>],xlim[<NUM_LIT:1>],N)<EOL>T[:,<NUM_LIT:1>]=np.linspace(ylim[<NUM_LIT:0>],ylim[<NUM_LIT:1>],N)<EOL><DEDENT>else:<EOL><INDENT>L = xlim[<NUM_LIT:1>] - xlim[<NUM_LIT:0>]<EOL>T[:, <NUM_LIT:0>] = np.linspace(xlim[<NUM_LIT:0>] + L / N, xlim[<NUM_LIT:1>] - L / N, N) <EOL>L = ylim[<NUM_LIT:1>] - ylim[<NUM_LIT:0>]<EOL>T[:, <NUM_LIT:1>] = np.linspace(ylim[<NUM_LIT:0>] + L / N, ylim[<NUM_LIT:1>] - L / N, N) <EOL><DEDENT>X, Y = meshgrid(T[:, <NUM_LIT:0>], T[:, <NUM_LIT:1>])<EOL>Z = grid_evaluation(X, Y, f,vectorized=vectorized)<EOL>if contour:<EOL><INDENT>if not dry_run:<EOL><INDENT>N=<NUM_LIT:200><EOL>colors=np.concatenate((np.ones((N,<NUM_LIT:1>)),np.tile(np.linspace(<NUM_LIT:1>,<NUM_LIT:0>,N).reshape(-<NUM_LIT:1>,<NUM_LIT:1>),(<NUM_LIT:1>,<NUM_LIT:2>))),axis=<NUM_LIT:1>)<EOL>colors = [ [<NUM_LIT:1>,<NUM_LIT:1>,<NUM_LIT:1>],*colors,[<NUM_LIT:1>,<NUM_LIT:0>,<NUM_LIT:0>]]<EOL>print('<STR_LIT>',np.max(Z[:]))<EOL>C = ax.contourf(X,Y,Z,levels = [-np.inf,*np.linspace(-<NUM_LIT:20>,<NUM_LIT:20>,N),np.inf],colors=colors)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not dry_run:<EOL><INDENT>C = ax.plot_surface(X, Y, Z)<EOL><DEDENT><DEDENT><DEDENT>if show:<EOL><INDENT>plt.show()<EOL><DEDENT>return ax,C,Z<EOL>

Plot polynomial approximation.

:param vectorized: `f` can handle an array of inputs


fig = plt.figure()<EOL>ax = Axes3D(fig)<EOL>light = LightSource(<NUM_LIT>, <NUM_LIT>)<EOL>illuminated_surface = light.shade(Z, cmap=cm.coolwarm)  <EOL>Xmin = np.amin(X)<EOL>Xmax = np.amax(X)<EOL>Ymin = np.amin(Y)<EOL>Ymax = np.amax(Y)<EOL>Zmin = np.amin(Z)<EOL>Zmax = np.amax(Z)<EOL>ax.contourf(X, Y, Z, zdir='<STR_LIT:x>', offset=Xmin - <NUM_LIT:0.1> * (Xmax - Xmin), cmap=cm.coolwarm, alpha=<NUM_LIT:1>)  <EOL>ax.contourf(X, Y, Z, zdir='<STR_LIT:y>', offset=Ymax + <NUM_LIT:0.1> * (Ymax - Ymin), cmap=cm.coolwarm, alpha=<NUM_LIT:1>)  <EOL>ax.contourf(X, Y, Z, zdir='<STR_LIT:z>', offset=Zmin - <NUM_LIT:0.1> * (Zmax - Zmin), cmap=cm.coolwarm, alpha=<NUM_LIT:1>)  <EOL>ax.plot_surface(X, Y, Z, cstride=<NUM_LIT:5>, rstride=<NUM_LIT:5>, facecolors=illuminated_surface, alpha=<NUM_LIT:0.5>)<EOL>plt.show()<EOL>

Surface plot.

Generate X and Y using, for example
      X,Y = np.mgrid[0:1:50j, 0:1:50j]
    or
      X,Y= np.meshgrid([0,1,2],[1,2,3]).

:param X: 2D-Array of x-coordinates
:param Y: 2D-Array of y-coordinates
:param Z: 2D-Array of z-coordinates


def plot_convergence(times, values, name=None, title=None, reference='<STR_LIT>', convergence_type='<STR_LIT>', expect_residuals=None,<EOL>expect_times=None, plot_rate='<STR_LIT>', base = np.exp(<NUM_LIT:0>),xlabel = '<STR_LIT:x>', p=<NUM_LIT:2>, preasymptotics=True, stagnation=False, marker='<STR_LIT:.>',<EOL>legend='<STR_LIT>',relative = False,ax = None):

name = name or '<STR_LIT>'<EOL>self_reference = (isinstance(reference,str) and reference=='<STR_LIT>') <EOL>ax = ax or plt.gca()<EOL>color = next(ax._get_lines.prop_cycler)['<STR_LIT>']<EOL>ax.tick_params(labeltop=False, labelright=True, right=True, which='<STR_LIT>')<EOL>ax.yaxis.grid(which="<STR_LIT>", linestyle='<STR_LIT:->', alpha=<NUM_LIT:0.5>)<EOL>ax.yaxis.grid(which="<STR_LIT>", linestyle='<STR_LIT:->', alpha=<NUM_LIT>)<EOL>c_ticks = <NUM_LIT:3><EOL>ACCEPT_MISFIT = <NUM_LIT:0.1><EOL>values, times = np.squeeze(values), np.squeeze(times)<EOL>assert(times.ndim == <NUM_LIT:1>)<EOL>assert(len(times) == len(values))<EOL>sorting = np.argsort(times)<EOL>times = times[sorting]<EOL>values = values[sorting]<EOL>if plot_rate == True:<EOL><INDENT>plot_rate = '<STR_LIT>'<EOL><DEDENT>if plot_rate !='<STR_LIT>':<EOL><INDENT>plot_rate = plot_rate*np.log(base)<EOL><DEDENT>if self_reference:<EOL><INDENT>if len(times) <= <NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>limit = values[-<NUM_LIT:1>]<EOL>limit_time = times[-<NUM_LIT:1>]<EOL>times = times[<NUM_LIT:0>:-<NUM_LIT:1>]<EOL>values = values[<NUM_LIT:0>:-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>limit = np.squeeze(reference)<EOL>limit_time = np.Inf<EOL><DEDENT>residuals = np.zeros(len(times))<EOL>N = limit.size<EOL>for L in range(len(times)):<EOL><INDENT>if p < np.Inf:<EOL><INDENT>residuals[L] = np.power(np.sum(np.power(np.abs(values[L] - limit), p) / N), <NUM_LIT:1.> / p)  <EOL><DEDENT>else:<EOL><INDENT>residuals[L] = np.amax(np.abs(values[L] - limit))<EOL><DEDENT><DEDENT>if relative:<EOL><INDENT>if p<np.Inf:<EOL><INDENT>residuals /= np.power(np.sum(np.power(np.abs(limit),p)/N),<NUM_LIT:1.>/p)<EOL><DEDENT>else:<EOL><INDENT>residuals /= np.amax(np.abs(limit))<EOL><DEDENT><DEDENT>try:<EOL><INDENT>remove = np.isnan(times) | np.isinf(times) | np.isnan(residuals) | np.isinf(residuals) | (residuals == <NUM_LIT:0>) | ((times == <NUM_LIT:0>) & (convergence_type == '<STR_LIT>'))<EOL><DEDENT>except TypeError:<EOL><INDENT>print(times,residuals)<EOL><DEDENT>times = times[~remove]<EOL>if sum(~remove) < (<NUM_LIT:2> if self_reference else <NUM_LIT:1>):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>residuals = residuals[~remove]<EOL>if convergence_type == '<STR_LIT>':<EOL><INDENT>x = np.log(times)<EOL>limit_x = np.log(limit_time)<EOL><DEDENT>else:<EOL><INDENT>x = times<EOL>limit_x = limit_time<EOL><DEDENT>max_x = max(x)<EOL>y = np.log(residuals)<EOL>try:<EOL><INDENT>rate, offset, min_x_fit, max_x_fit = _fit_rate(x, y, stagnation, preasymptotics, limit_x, have_rate=False if (plot_rate == '<STR_LIT>' or plot_rate is None) else plot_rate)<EOL><DEDENT>except FitError as e:<EOL><INDENT>warnings.warn(str(e))<EOL>plot_rate = False<EOL>rate = None<EOL><DEDENT>if self_reference:<EOL><INDENT>if rate >= <NUM_LIT:0>:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>real_rate = _real_rate(rate, l_bound=min_x_fit, r_bound=max_x_fit, reference_x=limit_x)<EOL>if (real_rate is None or abs((real_rate - rate) / rate) >= ACCEPT_MISFIT):<EOL><INDENT>warnings.warn(('<STR_LIT>')<EOL>+ ('<STR_LIT>'.format(real_rate) if real_rate else '<STR_LIT>')<EOL>+ ('<STR_LIT>'.format(rate) if rate else '<STR_LIT>'))      <EOL><DEDENT><DEDENT><DEDENT>if plot_rate:<EOL><INDENT>name += '<STR_LIT>' if plot_rate == '<STR_LIT>' else '<STR_LIT>'<EOL>if convergence_type == '<STR_LIT>':<EOL><INDENT>name+='<STR_LIT>'.format(rate) <EOL><DEDENT>else:<EOL><INDENT>base_rate = rate/np.log(base)<EOL>base_rate_str = f'<STR_LIT>'<EOL>if base_rate_str=='<STR_LIT>':<EOL><INDENT>base_rate_str='<STR_LIT:->'<EOL><DEDENT>if base_rate_str =='<STR_LIT:1>':<EOL><INDENT>base_rate_str = '<STR_LIT>'<EOL><DEDENT>name+=f'<STR_LIT>'<EOL><DEDENT>if convergence_type == '<STR_LIT>':<EOL><INDENT>X = np.linspace(np.exp(min_x_fit), np.exp(max_x_fit), c_ticks)<EOL>ax.loglog(X, np.exp(offset) * X ** rate, '<STR_LIT>', color=color)<EOL><DEDENT>else:<EOL><INDENT>X = np.linspace(min_x_fit, max_x_fit, c_ticks)<EOL>ax.semilogy(X, np.exp(offset + rate * X), '<STR_LIT>', color=color)<EOL><DEDENT><DEDENT>max_x_data = max_x<EOL>keep_1 = (x <= max_x_data)<EOL>if convergence_type == '<STR_LIT>':<EOL><INDENT>ax.loglog(np.array(times)[keep_1], np.array(residuals)[keep_1], label=name, marker=marker, color=color)<EOL>ax.loglog(np.array(times), np.array(residuals), marker=marker, color=color, alpha=<NUM_LIT:0.5>)<EOL><DEDENT>else:<EOL><INDENT>ax.semilogy(np.array(times)[keep_1], np.array(residuals)[keep_1], label=name, marker=marker, color=color)<EOL>ax.semilogy(np.array(times), np.array(residuals), marker=marker, color=color, alpha=<NUM_LIT:0.5>)<EOL><DEDENT>if expect_times is not None and expect_residuals is not None:<EOL><INDENT>ax.loglog(expect_times, expect_residuals, '<STR_LIT>', marker=marker, color=color) <EOL><DEDENT>if name:<EOL><INDENT>ax.legend(loc=legend)<EOL><DEDENT>if title:<EOL><INDENT>ax.set_title(title)<EOL><DEDENT>return rate<EOL>

Show loglog or semilogy convergence plot.

Specify :code:`reference` if exact limit is known. Otherwise limit is
taken to be last entry of :code:`values`.

Distance to limit is computed as RMSE (or analogous p-norm if p is specified)

Specify either :code:`plot_rate`(pass number or 'fit') or
:code:`expect_residuals` and :code:`expect_times` to add a second plot with
the expected convergence.

:param times: Runtimes
:type times: List of positive numbers
:param values: Outputs
:type values: List of arrays
:param reference: Exact solution, or 'self' if not available
:type reference: Array or 'self'
:param convergence_type: Convergence type
:type convergence_type: 'algebraic' or 'exponential'
:param expect_residuals: Expected residuals
:type expect_residuals: List of positive numbers
:param expect_times: Expected runtimes
:type expect_times: List of positive numbers
:param plot_rate: Expected convergence order
:type plot_rate: Real or 'fit'
:param preasymptotics: Ignore initial entries for rate fitting
:type preasymptotics: Boolean
:param stagnation: Ignore final entries from rate fitting
:type stagnation: Boolean
:param marker: Marker for data points
:type marker: Matplotlib marker string
:return: fitted convergence order


def wrapper(self,*args,**kwargs):  <EOL><INDENT>self.log.log(group=function.__name__,message='<STR_LIT>') <EOL>function(self,*args,**kwargs)<EOL>self.log.log(group=function.__name__,message='<STR_LIT>') <EOL><DEDENT>return wrapper<EOL>

Decorator that logs function calls in their self.log


def wrapper(*args,**kwargs):  <EOL><INDENT>pr=cProfile.Profile()<EOL>pr.enable()<EOL>output = function(*args,**kwargs)<EOL>pr.disable()<EOL>return pr,output<EOL><DEDENT>return wrapper<EOL>

Decorator that adds a runtime profile object to the output


import memory_profiler<EOL>def wrapper(*args,**kwargs):<EOL><INDENT>m = StringIO()<EOL>temp_func = memory_profiler.profile(func = function,stream=m,precision=<NUM_LIT:4>)<EOL>output = temp_func(*args,**kwargs)<EOL>print(m.getvalue())<EOL>m.close()<EOL>return output<EOL><DEDENT>return wrapper<EOL>

Decorator that prints memory information at each call of the function


import memory_profiler<EOL>def wrapper(*args,**kwargs):<EOL><INDENT>m=StringIO()<EOL>pr=cProfile.Profile()<EOL>pr.enable()<EOL>temp_func = memory_profiler.profile(func=function,stream=m,precision=<NUM_LIT:4>)<EOL>output = temp_func(*args,**kwargs)<EOL>print(m.getvalue())<EOL>pr.disable()<EOL>ps = pstats.Stats(pr)<EOL>ps.sort_stats('<STR_LIT>').print_stats('<STR_LIT>',<NUM_LIT:20>)<EOL>m.close()<EOL>return output<EOL><DEDENT>return wrapper<EOL>

Decorator that prints memory and runtime information at each call of the function


function,name=_strip_function(function)<EOL>if not function.__code__.co_code in [empty_function.__code__.co_code, doc_string_only_function.__code__.co_code]: <EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>def not_implemented_function(*args,**kwargs):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(args[<NUM_LIT:0>],name))<EOL><DEDENT>not_implemented_function.__qualname__=not_implemented_function.__name__ <EOL>return default(not_implemented_function,name=name)<EOL>

Declare abstract function.

Requires function to be empty except for docstring describing semantics.
To apply function, first argument must come with implementation of semantics.


def wrapper(*args,**kwargs):<EOL><INDENT>pr=cProfile.Profile()<EOL>pr.enable()<EOL>output = function(*args,**kwargs)<EOL>pr.disable()<EOL>ps = pstats.Stats(pr)<EOL>ps.sort_stats('<STR_LIT>').print_stats(<NUM_LIT:20>)<EOL>return output<EOL><DEDENT>return wrapper<EOL>

Decorator that prints running time information at each call of the function


def print_peak_memory(func,stream = None):

import time<EOL>import psutil<EOL>import os<EOL>memory_denominator=<NUM_LIT>**<NUM_LIT:2><EOL>memory_usage_refresh=<NUM_LIT><EOL>def wrapper(*args,**kwargs):<EOL><INDENT>from multiprocessing.pool import ThreadPool<EOL>pool = ThreadPool(processes=<NUM_LIT:1>)<EOL>process = psutil.Process(os.getpid())<EOL>start_mem = process.memory_info().rss<EOL>delta_mem = <NUM_LIT:0><EOL>max_memory = <NUM_LIT:0><EOL>async_result = pool.apply_async(func, args,kwargs)<EOL>while(not async_result.ready()):<EOL><INDENT>current_mem = process.memory_info().rss<EOL>delta_mem = current_mem - start_mem<EOL>if delta_mem > max_memory:<EOL><INDENT>max_memory = delta_mem<EOL><DEDENT>time.sleep(memory_usage_refresh)<EOL><DEDENT>return_val = async_result.get()  <EOL>max_memory /= memory_denominator<EOL>if stream is not None:<EOL><INDENT>stream.write(str(max_memory))<EOL><DEDENT>return return_val<EOL><DEDENT>return wrapper<EOL>

Print peak memory usage (in MB) of a function call

:param func: Function to be called
:param stream: Stream to write peak memory usage (defaults to stdout)

https://stackoverflow.com/questions/9850995/tracking-maximum-memory-usage-by-a-python-function


if len(args)==<NUM_LIT:1>:<EOL><INDENT>string_input = True<EOL>string = args[<NUM_LIT:0>].replace('<STR_LIT:U+0020>','<STR_LIT>')<EOL>original_args=string.split('<STR_LIT:U+002C>')<EOL>args = []<EOL>for arg in original_args:<EOL><INDENT>try:<EOL><INDENT>args.append(ast.literal_eval(arg))<EOL><DEDENT>except (ValueError,SyntaxError):<EOL><INDENT>try:<EOL><INDENT>args.append(eval(arg,{'<STR_LIT>':{}}))<EOL><DEDENT>except (NameError,SyntaxError):<EOL><INDENT>args.append(arg)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>string_input = False<EOL><DEDENT>arg_start = args[<NUM_LIT:0>]<EOL>if len(args)><NUM_LIT:2>:<EOL><INDENT>arg_step = args[<NUM_LIT:1>]<EOL>if len(args)><NUM_LIT:3>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>arg_step = None<EOL><DEDENT>arg_end = args[-<NUM_LIT:1>]<EOL>if String.valid(arg_start) and len(arg_start)==<NUM_LIT:1>:<EOL><INDENT>range_type = '<STR_LIT>'<EOL><DEDENT>elif all(Integer.valid(arg) for arg in args):<EOL><INDENT>range_type = '<STR_LIT>'<EOL><DEDENT>else: <EOL><INDENT>if string_input and original_args[<NUM_LIT:0>][<NUM_LIT:0>] in ['<STR_LIT:(>','<STR_LIT:[>']:<EOL><INDENT>range_type = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>range_type = '<STR_LIT:float>'<EOL><DEDENT><DEDENT>if range_type == '<STR_LIT>':<EOL><INDENT>start = ord(arg_start)<EOL>step = (ord(arg_step)- start) if arg_step else <NUM_LIT:1><EOL>end = ord(arg_end)<EOL>out = [chr(i) for i in range(start,end+step,step)]<EOL>if np.sign(step)*(ord(out[-<NUM_LIT:1>])-end)><NUM_LIT:0>:<EOL><INDENT>del out[-<NUM_LIT:1>]<EOL><DEDENT>return out<EOL><DEDENT>elif range_type == '<STR_LIT>':<EOL><INDENT>if string_input:<EOL><INDENT>if len(args)==<NUM_LIT:2> and all('<STR_LIT>' in oa for oa in original_args):<EOL><INDENT>bases,exponents = zip(*[oa.split('<STR_LIT>') for oa in original_args])<EOL>if len(set(bases))==<NUM_LIT:1>:<EOL><INDENT>return [int(bases[<NUM_LIT:0>])**exponent for exponent in smart_range('<STR_LIT:U+002C>'.join(exponents))]<EOL><DEDENT><DEDENT><DEDENT>start = arg_start<EOL>step = (arg_step - arg_start) if arg_step is not None else <NUM_LIT:1><EOL>end = arg_end<EOL>out = list(range(start,end+step,step))<EOL>if np.sign(step)*(out[-<NUM_LIT:1>]-end)><NUM_LIT:0>:<EOL><INDENT>del out[-<NUM_LIT:1>]<EOL><DEDENT>return out<EOL><DEDENT>elif range_type == '<STR_LIT:float>':<EOL><INDENT>if len(args)==<NUM_LIT:2> and all('<STR_LIT>' in oa for oa in original_args):<EOL><INDENT>bases,exponents = zip(*[oa.split('<STR_LIT>') for oa in original_args])<EOL>if len(set(bases))==<NUM_LIT:1>:<EOL><INDENT>return [float(bases[<NUM_LIT:0>])**exponent for exponent in smart_range('<STR_LIT:U+002C>'.join(exponents)) ]<EOL><DEDENT><DEDENT>if len(args) == <NUM_LIT:2>:<EOL><INDENT>raise ValueError()<EOL><DEDENT>start = arg_start<EOL>step = arg_step - arg_start<EOL>end = arg_end<EOL>out = list(np.arange(start,end+<NUM_LIT>*step,step))<EOL>return out<EOL><DEDENT>elif range_type == '<STR_LIT>':<EOL><INDENT>lopen,start = (original_args[<NUM_LIT:0>][<NUM_LIT:0>]=='<STR_LIT:(>'),float(original_args[<NUM_LIT:0>][<NUM_LIT:1>:])<EOL>end,N = original_args[<NUM_LIT:1>].split('<STR_LIT:/>')<EOL>end,ropen = float(end[:-<NUM_LIT:1>]),(end[-<NUM_LIT:1>]=='<STR_LIT:)>')<EOL>N = ast.literal_eval(N)+lopen +ropen<EOL>points = np.linspace(start,end,num=N)<EOL>return points[lopen:len(points)-ropen]<EOL><DEDENT>


if ld:<EOL><INDENT>keys = list(ld[<NUM_LIT:0>])<EOL>dl = {key:[d[key] for d in ld] for key in keys}<EOL>return dl<EOL><DEDENT>else:<EOL><INDENT>return {}<EOL><DEDENT>

Convert list of dictionaries to dictionary of lists


def chained(x):<EOL><INDENT>for f in reversed(fs):<EOL><INDENT>if f:<EOL><INDENT>x=f(x)<EOL><DEDENT><DEDENT>return x<EOL><DEDENT>return chained<EOL>


return shutil.which(cmd) is not None<EOL>

Check whether given command is available on system


npmode = isinstance(l,np.ndarray)<EOL>if npmode:<EOL><INDENT>l=list(l)<EOL><DEDENT>g=np.concatenate((np.array([<NUM_LIT:0>]),np.cumsum(split_integer(len(l),length=N))))<EOL>s=[l[g[i]:g[i+<NUM_LIT:1>]] for i in range(N)]<EOL>if npmode:<EOL><INDENT>s=[np.array(sl) for sl in s]<EOL><DEDENT>return s<EOL>


return '<STR_LIT>'.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))<EOL>

Generate alphanumerical string. Hint: Check whether module tempfile has what you want, especially when you are concerned about race conditions


def random_word(length,dictionary = False):

if dictionary:<EOL><INDENT>try:<EOL><INDENT>with open('<STR_LIT>') as fp:<EOL><INDENT>words = [word.lower()[:-<NUM_LIT:1>] for word in fp.readlines() if re.match('<STR_LIT>'.format('<STR_LIT:{>'+str(length)+'<STR_LIT:}>'),word)]<EOL><DEDENT>return random.choice(words)<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>vowels = list('<STR_LIT>')<EOL>consonants = list('<STR_LIT>')<EOL>pairs = [(random.choice(consonants),random.choice(vowels)) for _ in range(length//<NUM_LIT:2>+<NUM_LIT:1>)] <EOL>return '<STR_LIT>'.join([l for p in pairs for l in p])[:length]<EOL>

Creates random lowercase words from dictionary or by alternating vowels and consonants

The second method chooses from 85**length words.
The dictionary method chooses from 3000--12000 words for 3<=length<=12
(though this of course depends on the available dictionary)

:param length: word length
:param dictionary: Try reading from dictionary, else fall back to artificial words


td = str(timedelta(seconds = seconds))<EOL>parts = td.split('<STR_LIT:.>')<EOL>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>td = td+'<STR_LIT>'<EOL><DEDENT>elif len(parts) == <NUM_LIT:2>:<EOL><INDENT>td = '<STR_LIT:.>'.join([parts[<NUM_LIT:0>],parts[<NUM_LIT:1>][:<NUM_LIT:2>]])<EOL><DEDENT>return td<EOL>

Converts seconds into elapsed time string of form

(X days(s)?,)? HH:MM:SS.YY


def hook():<EOL><INDENT>readline.insert_text(text)<EOL>readline.redisplay()<EOL><DEDENT>try:<EOL><INDENT>readline.set_pre_input_hook(hook)<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>result = input(prompt)<EOL>try:<EOL><INDENT>readline.set_pre_input_hook()<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>return result<EOL>


return s.isidentifier() and not keyword.iskeyword(s)<EOL>


if any(np.isinf(limits)):<EOL><INDENT>return []<EOL><DEDENT>if limits[<NUM_LIT:0>] == limits[<NUM_LIT:1>]:<EOL><INDENT>return np.array([limits[<NUM_LIT:0>]])<EOL><DEDENT>return self.locator.tick_values(limits[<NUM_LIT:0>], limits[<NUM_LIT:1>])<EOL>

Compute breaks

Parameters
----------
limits : tuple
    Minimum and maximum values

Returns
-------
out : array_like
    Sequence of breaks points


if any(np.isinf(limits)):<EOL><INDENT>return []<EOL><DEDENT>n = self.n<EOL>base = self.base<EOL>rng = np.log(limits)/np.log(base)<EOL>_min = int(np.floor(rng[<NUM_LIT:0>]))<EOL>_max = int(np.ceil(rng[<NUM_LIT:1>]))<EOL>dtype = float if _min < <NUM_LIT:0> or _max < <NUM_LIT:0> else int<EOL>if _max == _min:<EOL><INDENT>return base ** _min<EOL><DEDENT>by = int(np.floor((_max-_min)/n)) + <NUM_LIT:1><EOL>for step in range(by, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>breaks = base ** np.arange(_min, _max+<NUM_LIT:1>, step=step, dtype=dtype)<EOL>relevant_breaks = (<EOL>(limits[<NUM_LIT:0>] <= breaks) &<EOL>(breaks <= limits[<NUM_LIT:1>])<EOL>)<EOL>if np.sum(relevant_breaks) >= n-<NUM_LIT:2>:<EOL><INDENT>return breaks<EOL><DEDENT><DEDENT>return _log_sub_breaks(n=n, base=base)(limits)<EOL>


def __call__(self, major, limits=None, n=None):

if len(major) < <NUM_LIT:2>:<EOL><INDENT>return np.array([])<EOL><DEDENT>if limits is None:<EOL><INDENT>limits = min_max(major)<EOL><DEDENT>if n is None:<EOL><INDENT>n = self.n<EOL><DEDENT>diff = np.diff(major)<EOL>step = diff[<NUM_LIT:0>]<EOL>if len(diff) > <NUM_LIT:1> and all(diff == step):<EOL><INDENT>major = np.hstack([major[<NUM_LIT:0>]-step,<EOL>major,<EOL>major[-<NUM_LIT:1>]+step])<EOL><DEDENT>mbreaks = []<EOL>factors = np.arange(<NUM_LIT:1>, n+<NUM_LIT:1>)<EOL>for lhs, rhs in zip(major[:-<NUM_LIT:1>], major[<NUM_LIT:1>:]):<EOL><INDENT>sep = (rhs - lhs)/(n+<NUM_LIT:1>)<EOL>mbreaks.append(lhs + factors * sep)<EOL><DEDENT>minor = np.hstack(mbreaks)<EOL>minor = minor.compress((limits[<NUM_LIT:0>] <= minor) &<EOL>(minor <= limits[<NUM_LIT:1>]))<EOL>return minor<EOL>

Minor breaks

Parameters
----------
major : array_like
    Major breaks
limits : array_like | None
    Limits of the scale. If *array_like*, must be
    of size 2. If **None**, then the minimum and
    maximum of the major breaks are used.
n : int
    Number of minor breaks between the major
    breaks. If **None**, then *self.n* is used.

Returns
-------
out : array_like
    Minor beraks


if not self.trans.dataspace_is_numerical:<EOL><INDENT>raise TypeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if limits is None:<EOL><INDENT>limits = min_max(major)<EOL><DEDENT>if n is None:<EOL><INDENT>n = self.n<EOL><DEDENT>major = self._extend_breaks(major)<EOL>major = self.trans.inverse(major)<EOL>limits = self.trans.inverse(limits)<EOL>minor = minor_breaks(n)(major, limits)<EOL>return self.trans.transform(minor)<EOL>

Minor breaks for transformed scales

Parameters
----------
major : array_like
    Major breaks
limits : array_like | None
    Limits of the scale. If *array_like*, must be
    of size 2. If **None**, then the minimum and
    maximum of the major breaks are used.
n : int
    Number of minor breaks between the major
    breaks. If **None**, then *self.n* is used.

Returns
-------
out : array_like
    Minor breaks


trans = self.trans<EOL>trans = trans if isinstance(trans, type) else trans.__class__<EOL>is_log = trans.__name__.startswith('<STR_LIT>')<EOL>diff = np.diff(major)<EOL>step = diff[<NUM_LIT:0>]<EOL>if is_log and all(diff == step):<EOL><INDENT>major = np.hstack([major[<NUM_LIT:0>]-step, major, major[-<NUM_LIT:1>]+step])<EOL><DEDENT>return major<EOL>

Append 2 extra breaks at either end of major

If breaks of transform space are non-equidistant,
:func:`minor_breaks` add minor breaks beyond the first
and last major breaks. The solutions is to extend those
breaks (in transformed space) before the minor break call
is made. How the breaks depends on the type of transform.


if any(pd.isnull(x) for x in limits):<EOL><INDENT>return []<EOL><DEDENT>ret = self.locator.tick_values(*limits)<EOL>return [num2date(val) for val in ret]<EOL>

Compute breaks

Parameters
----------
limits : tuple
    Minimum and maximum :class:`datetime.datetime` values.

Returns
-------
out : array_like
    Sequence of break points.


if any(pd.isnull(x) for x in limits):<EOL><INDENT>return []<EOL><DEDENT>helper = timedelta_helper(limits)<EOL>scaled_limits = helper.scaled_limits()<EOL>scaled_breaks = self._breaks_func(scaled_limits)<EOL>breaks = helper.numeric_to_timedelta(scaled_breaks)<EOL>return breaks<EOL>

Compute breaks

Parameters
----------
limits : tuple
    Minimum and maximum :class:`datetime.timedelta` values.

Returns
-------
out : array_like
    Sequence of break points.


<EOL>ts_range = self.value(max(sequence)) - self.value(min(sequence))<EOL>package = self.determine_package(sequence[<NUM_LIT:0>])<EOL>if package == '<STR_LIT>':<EOL><INDENT>cuts = [<EOL>(<NUM_LIT>, '<STR_LIT>'),<EOL>(<NUM_LIT>, '<STR_LIT>'),<EOL>(<NUM_LIT>, '<STR_LIT:s>'),<EOL>(<NUM_LIT:9>, '<STR_LIT:m>'),<EOL>(<NUM_LIT:6>, '<STR_LIT:h>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:d>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:w>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:M>'),<EOL>(<NUM_LIT:3>, '<STR_LIT:y>')]<EOL>denomination = NANOSECONDS<EOL>base_units = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>cuts = [<EOL>(<NUM_LIT>, '<STR_LIT:s>'),<EOL>(<NUM_LIT:9>, '<STR_LIT:m>'),<EOL>(<NUM_LIT:6>, '<STR_LIT:h>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:d>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:w>'),<EOL>(<NUM_LIT:4>, '<STR_LIT:M>'),<EOL>(<NUM_LIT:3>, '<STR_LIT:y>')]<EOL>denomination = SECONDS<EOL>base_units = '<STR_LIT>'<EOL><DEDENT>for size, units in reversed(cuts):<EOL><INDENT>if ts_range >= size*denomination[units]:<EOL><INDENT>return units<EOL><DEDENT><DEDENT>return base_units<EOL>

Determine good units for representing a sequence of timedeltas


if self.package == '<STR_LIT>':<EOL><INDENT>return td.value<EOL><DEDENT>else:<EOL><INDENT>return td.total_seconds()<EOL><DEDENT>

Return the numeric value representation on a timedelta


_min = self.limits[<NUM_LIT:0>]/self.factor<EOL>_max = self.limits[<NUM_LIT:1>]/self.factor<EOL>return _min, _max<EOL>

Minimum and Maximum to use for computing breaks


return [self.to_numeric(td) for td in timedeltas]<EOL>

Convert sequence of timedelta to numerics


if self.package == '<STR_LIT>':<EOL><INDENT>return [self.type(int(x*self.factor), units='<STR_LIT>')<EOL>for x in numerics]<EOL><DEDENT>else:<EOL><INDENT>return [self.type(seconds=x*self.factor)<EOL>for x in numerics]<EOL><DEDENT>

Convert sequence of numerics to timedelta


if self.package == '<STR_LIT>':<EOL><INDENT>return td.value/NANOSECONDS[self.units]<EOL><DEDENT>else:<EOL><INDENT>return td.total_seconds()/SECONDS[self.units]<EOL><DEDENT>

Convert timedelta to a number corresponding to the
appropriate units. The appropriate units are those
determined with the object is initialised.


Q = self.Q<EOL>w = self.w<EOL>only_inside = self.only_inside<EOL>simplicity_max = self.simplicity_max<EOL>density_max = self.density_max<EOL>coverage_max = self.coverage_max<EOL>simplicity = self.simplicity<EOL>coverage = self.coverage<EOL>density = self.density<EOL>legibility = self.legibility<EOL>log10 = np.log10<EOL>ceil = np.ceil<EOL>floor = np.floor<EOL>dmin, dmax = limits<EOL>if dmin > dmax:<EOL><INDENT>dmin, dmax = dmax, dmin<EOL><DEDENT>elif dmin == dmax:<EOL><INDENT>return np.array([dmin])<EOL><DEDENT>best_score = -<NUM_LIT:2><EOL>j = <NUM_LIT:1><EOL>while j < float('<STR_LIT>'):<EOL><INDENT>for q in Q:<EOL><INDENT>sm = simplicity_max(q, j)<EOL>if w[<NUM_LIT:0>]*sm + w[<NUM_LIT:1>] + w[<NUM_LIT:2>] + w[<NUM_LIT:3>] < best_score:<EOL><INDENT>j = float('<STR_LIT>')<EOL>break<EOL><DEDENT>k = <NUM_LIT:2><EOL>while k < float('<STR_LIT>'):<EOL><INDENT>dm = density_max(k)<EOL>if w[<NUM_LIT:0>]*sm + w[<NUM_LIT:1>] + w[<NUM_LIT:2>]*dm + w[<NUM_LIT:3>] < best_score:<EOL><INDENT>break<EOL><DEDENT>delta = (dmax-dmin)/(k+<NUM_LIT:1>)/j/q<EOL>z = ceil(log10(delta))<EOL>while z < float('<STR_LIT>'):<EOL><INDENT>step = j*q*(<NUM_LIT:10>**z)<EOL>cm = coverage_max(dmin, dmax, step*(k-<NUM_LIT:1>))<EOL>if w[<NUM_LIT:0>]*sm + w[<NUM_LIT:1>]*cm + w[<NUM_LIT:2>]*dm + w[<NUM_LIT:3>] < best_score:<EOL><INDENT>break<EOL><DEDENT>min_start = int(floor(dmax/step)*j - (k-<NUM_LIT:1>)*j)<EOL>max_start = int(ceil(dmin/step)*j)<EOL>if min_start > max_start:<EOL><INDENT>z = z+<NUM_LIT:1><EOL>break<EOL><DEDENT>for start in range(min_start, max_start+<NUM_LIT:1>):<EOL><INDENT>lmin = start * (step/j)<EOL>lmax = lmin + step*(k-<NUM_LIT:1>)<EOL>lstep = step<EOL>s = simplicity(q, j, lmin, lmax, lstep)<EOL>c = coverage(dmin, dmax, lmin, lmax)<EOL>d = density(k, dmin, dmax, lmin, lmax)<EOL>l = legibility(lmin, lmax, lstep)<EOL>score = w[<NUM_LIT:0>]*s + w[<NUM_LIT:1>]*c + w[<NUM_LIT:2>]*d + w[<NUM_LIT:3>]*l<EOL>if (score > best_score and<EOL>(not only_inside or<EOL>(lmin >= dmin and lmax <= dmax))):<EOL><INDENT>best_score = score<EOL>best = (lmin, lmax, lstep, q, k)<EOL><DEDENT><DEDENT>z = z+<NUM_LIT:1><EOL><DEDENT>k = k+<NUM_LIT:1><EOL><DEDENT><DEDENT>j = j+<NUM_LIT:1><EOL><DEDENT>try:<EOL><INDENT>locs = best[<NUM_LIT:0>] + np.arange(best[<NUM_LIT:4>])*best[<NUM_LIT:2>]<EOL><DEDENT>except UnboundLocalError:<EOL><INDENT>locs = []<EOL><DEDENT>return locs<EOL>

Calculate the breaks

Parameters
----------
limits : array
    Minimum and maximum values.

Returns
-------
out : array_like
    Sequence of break points.


def rescale(x, to=(<NUM_LIT:0>, <NUM_LIT:1>), _from=None):

if _from is None:<EOL><INDENT>_from = np.min(x), np.max(x)<EOL><DEDENT>return np.interp(x, _from, to)<EOL>

Rescale numeric vector to have specified minimum and maximum.

Parameters
----------
x : array_like | numeric
    1D vector of values to manipulate.
to : tuple
    output range (numeric vector of length two)
_from : tuple
    input range (numeric vector of length two).
    If not given, is calculated from the range of x

Returns
-------
out : array_like
    Rescaled values

Examples
--------
>>> x = [0, 2, 4, 6, 8, 10]
>>> rescale(x)
array([0. , 0.2, 0.4, 0.6, 0.8, 1. ])
>>> rescale(x, to=(0, 2))
array([0. , 0.4, 0.8, 1.2, 1.6, 2. ])
>>> rescale(x, to=(0, 2), _from=(0, 20))
array([0. , 0.2, 0.4, 0.6, 0.8, 1. ])


def rescale_mid(x, to=(<NUM_LIT:0>, <NUM_LIT:1>), _from=None, mid=<NUM_LIT:0>):

array_like = True<EOL>try:<EOL><INDENT>len(x)<EOL><DEDENT>except TypeError:<EOL><INDENT>array_like = False<EOL>x = [x]<EOL><DEDENT>if not hasattr(x, '<STR_LIT>'):<EOL><INDENT>x = np.asarray(x)<EOL><DEDENT>if _from is None:<EOL><INDENT>_from = np.array([np.min(x), np.max(x)])<EOL><DEDENT>else:<EOL><INDENT>_from = np.asarray(_from)<EOL><DEDENT>if (zero_range(_from) or zero_range(to)):<EOL><INDENT>out = np.repeat(np.mean(to), len(x))<EOL><DEDENT>else:<EOL><INDENT>extent = <NUM_LIT:2> * np.max(np.abs(_from - mid))<EOL>out = (x - mid) / extent * np.diff(to) + np.mean(to)<EOL><DEDENT>if not array_like:<EOL><INDENT>out = out[<NUM_LIT:0>]<EOL><DEDENT>return out<EOL>

Rescale numeric vector to have specified minimum, midpoint,
and maximum.

Parameters
----------
x : array_like | numeric
    1D vector of values to manipulate.
to : tuple
    output range (numeric vector of length two)
_from : tuple
    input range (numeric vector of length two).
    If not given, is calculated from the range of x
mid	: numeric
    mid-point of input range

Returns
-------
out : array_like
    Rescaled values

Examples
--------
>>> rescale_mid([1, 2, 3], mid=1)
array([0.5 , 0.75, 1.  ])
>>> rescale_mid([1, 2, 3], mid=2)
array([0. , 0.5, 1. ])


def rescale_max(x, to=(<NUM_LIT:0>, <NUM_LIT:1>), _from=None):

array_like = True<EOL>try:<EOL><INDENT>len(x)<EOL><DEDENT>except TypeError:<EOL><INDENT>array_like = False<EOL>x = [x]<EOL><DEDENT>if not hasattr(x, '<STR_LIT>'):<EOL><INDENT>x = np.asarray(x)<EOL><DEDENT>if _from is None:<EOL><INDENT>_from = np.array([np.min(x), np.max(x)])<EOL><DEDENT>out = x/_from[<NUM_LIT:1>] * to[<NUM_LIT:1>]<EOL>if not array_like:<EOL><INDENT>out = out[<NUM_LIT:0>]<EOL><DEDENT>return out<EOL>

Rescale numeric vector to have specified maximum.

Parameters
----------
x : array_like | numeric
    1D vector of values to manipulate.
to : tuple
    output range (numeric vector of length two)
_from : tuple
    input range (numeric vector of length two).
    If not given, is calculated from the range of x.
    Only the 2nd (max) element is essential to the
    output.

Returns
-------
out : array_like
    Rescaled values

Examples
--------
>>> x = [0, 2, 4, 6, 8, 10]
>>> rescale_max(x, (0, 3))
array([0. , 0.6, 1.2, 1.8, 2.4, 3. ])

Only the 2nd (max) element of the parameters ``to``
and ``_from`` are essential to the output.

>>> rescale_max(x, (1, 3))
array([0. , 0.6, 1.2, 1.8, 2.4, 3. ])
>>> rescale_max(x, (0, 20))
array([ 0.,  4.,  8., 12., 16., 20.])

If :python:`max(x) < _from[1]` then values will be
scaled beyond the requested (:python:`to[1]`) maximum.

>>> rescale_max(x, to=(1, 3), _from=(-1, 6))
array([0., 1., 2., 3., 4., 5.])


def squish_infinite(x, range=(<NUM_LIT:0>, <NUM_LIT:1>)):

xtype = type(x)<EOL>if not hasattr(x, '<STR_LIT>'):<EOL><INDENT>x = np.asarray(x)<EOL><DEDENT>x[x == -np.inf] = range[<NUM_LIT:0>]<EOL>x[x == np.inf] = range[<NUM_LIT:1>]<EOL>if not isinstance(x, xtype):<EOL><INDENT>x = xtype(x)<EOL><DEDENT>return x<EOL>

Truncate infinite values to a range.

Parameters
----------
x : array_like
    Values that should have infinities squished.
range : tuple
    The range onto which to squish the infinites.
    Must be of size 2.

Returns
-------
out : array_like
    Values with infinites squished.

Examples
--------
>>> squish_infinite([0, .5, .25, np.inf, .44])
[0.0, 0.5, 0.25, 1.0, 0.44]
>>> squish_infinite([0, -np.inf, .5, .25, np.inf], (-10, 9))
[0.0, -10.0, 0.5, 0.25, 9.0]


def squish(x, range=(<NUM_LIT:0>, <NUM_LIT:1>), only_finite=True):

xtype = type(x)<EOL>if not hasattr(x, '<STR_LIT>'):<EOL><INDENT>x = np.asarray(x)<EOL><DEDENT>finite = np.isfinite(x) if only_finite else True<EOL>x[np.logical_and(x < range[<NUM_LIT:0>], finite)] = range[<NUM_LIT:0>]<EOL>x[np.logical_and(x > range[<NUM_LIT:1>], finite)] = range[<NUM_LIT:1>]<EOL>if not isinstance(x, xtype):<EOL><INDENT>x = xtype(x)<EOL><DEDENT>return x<EOL>

Squish values into range.

Parameters
----------
x : array_like
    Values that should have out of range values squished.
range : tuple
    The range onto which to squish the values.
only_finite: boolean
    When true, only squishes finite values.

Returns
-------
out : array_like
    Values with out of range values squished.

Examples
--------
>>> squish([-1.5, 0.2, 0.5, 0.8, 1.0, 1.2])
[0.0, 0.2, 0.5, 0.8, 1.0, 1.0]

>>> squish([-np.inf, -1.5, 0.2, 0.5, 0.8, 1.0, np.inf], only_finite=False)
[0.0, 0.0, 0.2, 0.5, 0.8, 1.0, 1.0]


def censor(x, range=(<NUM_LIT:0>, <NUM_LIT:1>), only_finite=True):

if not len(x):<EOL><INDENT>return x<EOL><DEDENT>py_time_types = (datetime.datetime, datetime.timedelta)<EOL>np_pd_time_types = (pd.Timestamp, pd.Timedelta,<EOL>np.datetime64, np.timedelta64)<EOL>x0 = first_element(x)<EOL>if type(x0) in py_time_types:<EOL><INDENT>return _censor_with(x, range, '<STR_LIT>')<EOL><DEDENT>if not hasattr(x, '<STR_LIT>') and isinstance(x0, np_pd_time_types):<EOL><INDENT>return _censor_with(x, range, type(x0)('<STR_LIT>'))<EOL><DEDENT>x_array = np.asarray(x)<EOL>if pdtypes.is_number(x0) and not isinstance(x0, np.timedelta64):<EOL><INDENT>null = float('<STR_LIT>')<EOL><DEDENT>elif com.is_datetime_arraylike(x_array):<EOL><INDENT>null = pd.Timestamp('<STR_LIT>')<EOL><DEDENT>elif pdtypes.is_datetime64_dtype(x_array):<EOL><INDENT>null = np.datetime64('<STR_LIT>')<EOL><DEDENT>elif isinstance(x0, pd.Timedelta):<EOL><INDENT>null = pd.Timedelta('<STR_LIT>')<EOL><DEDENT>elif pdtypes.is_timedelta64_dtype(x_array):<EOL><INDENT>null = np.timedelta64('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT:{}>".format(type(x0)))<EOL><DEDENT>if only_finite:<EOL><INDENT>try:<EOL><INDENT>finite = np.isfinite(x)<EOL><DEDENT>except TypeError:<EOL><INDENT>finite = np.repeat(True, len(x))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>finite = np.repeat(True, len(x))<EOL><DEDENT>if hasattr(x, '<STR_LIT>'):<EOL><INDENT>outside = (x < range[<NUM_LIT:0>]) | (x > range[<NUM_LIT:1>])<EOL>bool_idx = finite & outside<EOL>x = x.copy()<EOL>x[bool_idx] = null<EOL><DEDENT>else:<EOL><INDENT>x = [null if not range[<NUM_LIT:0>] <= val <= range[<NUM_LIT:1>] and f else val<EOL>for val, f in zip(x, finite)]<EOL><DEDENT>return x<EOL>

Convert any values outside of range to a **NULL** type object.

Parameters
----------
x : array_like
    Values to manipulate
range : tuple
    (min, max) giving desired output range
only_finite : bool
    If True (the default), will only modify
    finite values.

Returns
-------
x : array_like
    Censored array

Examples
--------
>>> a = [1, 2, np.inf, 3, 4, -np.inf, 5]
>>> censor(a, (0, 10))
[1, 2, inf, 3, 4, -inf, 5]
>>> censor(a, (0, 10), False)
[1, 2, nan, 3, 4, nan, 5]
>>> censor(a, (2, 4))
[nan, 2, inf, 3, 4, -inf, nan]

Notes
-----
All values in ``x`` should be of the same type. ``only_finite`` parameter
is not considered for Datetime and Timedelta types.

The **NULL** type object depends on the type of values in **x**.

- :class:`float` - :py:`float('nan')`
- :class:`int` - :py:`float('nan')`
- :class:`datetime.datetime` : :py:`np.datetime64(NaT)`
- :class:`datetime.timedelta` : :py:`np.timedelta64(NaT)`


return [val if range[<NUM_LIT:0>] <= val <= range[<NUM_LIT:1>] else value<EOL>for val in x]<EOL>

Censor any values outside of range with ``None``