def readComplexCode(self, hskip, alphabet):

stream = self.stream<EOL>lengths = [<NUM_LIT:1>,<NUM_LIT:2>,<NUM_LIT:3>,<NUM_LIT:4>,<NUM_LIT:0>,<NUM_LIT:5>,<NUM_LIT>,<NUM_LIT:6>,<NUM_LIT:16>,<NUM_LIT:7>,<NUM_LIT:8>,<NUM_LIT:9>,<NUM_LIT:10>,<NUM_LIT:11>,<NUM_LIT:12>,<NUM_LIT>,<NUM_LIT>,<NUM_LIT:15>][hskip:]<EOL>codeLengths = {}<EOL>total = <NUM_LIT:0><EOL>lol = LengthOfLengthAlphabet('<STR_LIT>'+alphabet.name)<EOL>lengthCode = LengthAlphabet('<STR_LIT:#>'+alphabet.name)<EOL>lengthIter = iter(lengths)<EOL>lengthsLeft = len(lengths)<EOL>while total<<NUM_LIT:32> and lengthsLeft><NUM_LIT:0>:<EOL><INDENT>lengthsLeft -= <NUM_LIT:1><EOL>newSymbol = next(lengthIter)<EOL>lol.description = str(lengthCode[newSymbol])<EOL>length = self.verboseRead(lol)<EOL>if length:<EOL><INDENT>codeLengths[newSymbol] = length<EOL>total += <NUM_LIT:32>>>length<EOL><DEDENT><DEDENT>if total><NUM_LIT:32>: raise ValueError("<STR_LIT>")<EOL>if len(codeLengths)==<NUM_LIT:1>: codeLengths[list(codeLengths.keys())[<NUM_LIT:0>]] = <NUM_LIT:0><EOL>lengthCode.setLength(codeLengths)<EOL>print("<STR_LIT>".format(alphabet.name))<EOL>lengthCode.showCode()<EOL>symbolLengths = {}<EOL>total = <NUM_LIT:0><EOL>lastLength = <NUM_LIT:8><EOL>alphabetIter = iter(alphabet)<EOL>while total<<NUM_LIT>:<EOL><INDENT>length = lengthCode.decodePeek(<EOL>self.stream.peek(lengthCode.maxLength))[<NUM_LIT:1>].index<EOL>if length==<NUM_LIT:0>:<EOL><INDENT>symbol = next(alphabetIter)<EOL>lengthCode.description = '<STR_LIT>'.format(symbol)<EOL>self.verboseRead(lengthCode)<EOL>continue<EOL><DEDENT>if length==<NUM_LIT:16>:<EOL><INDENT>lengthCode.description ='<STR_LIT>'+str(lastLength)<EOL>extra = self.verboseRead(lengthCode)<EOL>repeat = <NUM_LIT:2><EOL>startSymbol = next(alphabetIter)<EOL>endSymbol = next(alphabetIter)<EOL>symbolLengths[startSymbol.index] =symbolLengths[endSymbol.index] = lastLength<EOL>total += <NUM_LIT:2>*<NUM_LIT>>>lastLength<EOL>while True:<EOL><INDENT>oldRepeat = repeat<EOL>repeat = (repeat-<NUM_LIT:2><<<NUM_LIT:2>)+extra+<NUM_LIT:3><EOL>for i in range(oldRepeat, repeat):<EOL><INDENT>endSymbol = next(alphabetIter)<EOL>symbolLengths[endSymbol.index] = lastLength<EOL><DEDENT>total += (repeat-oldRepeat)*<NUM_LIT>>>lastLength<EOL>if total>=<NUM_LIT>: break<EOL>length = lengthCode.decodePeek(<EOL>self.stream.peek(lengthCode.maxLength))[<NUM_LIT:1>].index<EOL>if length!=<NUM_LIT:16>: break<EOL>lengthCode.description = '<STR_LIT>'.format(<EOL>(repeat-<NUM_LIT:2><<<NUM_LIT:2>)+<NUM_LIT:3>)<EOL>extra = self.verboseRead(lengthCode)<EOL><DEDENT><DEDENT>elif length==<NUM_LIT>:<EOL><INDENT>lengthCode.description = '<STR_LIT>'<EOL>extra = self.verboseRead(lengthCode)<EOL>repeat = <NUM_LIT:2><EOL>startSymbol = next(alphabetIter)<EOL>endSymbol = next(alphabetIter)<EOL>while True:<EOL><INDENT>oldRepeat = repeat<EOL>repeat = (repeat-<NUM_LIT:2><<<NUM_LIT:3>)+extra+<NUM_LIT:3><EOL>for i in range(repeat-oldRepeat):<EOL><INDENT>endSymbol = next(alphabetIter)<EOL><DEDENT>length = lengthCode.decodePeek(<EOL>self.stream.peek(lengthCode.maxLength))[<NUM_LIT:1>].index<EOL>if length!=<NUM_LIT>: break<EOL>lengthCode.description = '<STR_LIT>'.format(<EOL>(repeat-<NUM_LIT:2><<<NUM_LIT:3>)+<NUM_LIT:3>)<EOL>extra = self.verboseRead(lengthCode)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>symbol = next(alphabetIter)<EOL>char = str(symbol)<EOL>if char in '<STR_LIT:{}>': char *= <NUM_LIT:2><EOL>lengthCode.description ='<STR_LIT>'.format(char)<EOL>self.verboseRead(lengthCode)<EOL>symbolLengths[symbol.index] = length<EOL>total += <NUM_LIT>>>length<EOL>lastLength = length<EOL><DEDENT><DEDENT>assert total==<NUM_LIT><EOL>alphabet.setLength(symbolLengths)<EOL>print('<STR_LIT>'+alphabet.name+'<STR_LIT::>')<EOL>alphabet.showCode()<EOL>


print('<STR_LIT>'.format(<EOL>'<STR_LIT>', self.width-<NUM_LIT:10>))<EOL>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.windowSize = self.verboseRead(WindowSizeAlphabet())<EOL>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:=>'))<EOL>self.ISLAST = False<EOL>self.output = bytearray()<EOL>while not self.ISLAST:<EOL><INDENT>self.ISLAST = self.verboseRead(<EOL>BoolCode('<STR_LIT>', description="<STR_LIT>"))<EOL>if self.ISLAST:<EOL><INDENT>if self.verboseRead(<EOL>BoolCode('<STR_LIT>', description="<STR_LIT>")): break<EOL><DEDENT>if self.metablockLength(): continue<EOL>if not self.ISLAST and self.uncompressed(): continue<EOL>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.numberOfBlockTypes = {}<EOL>self.currentBlockCounts = {}<EOL>self.blockTypeCodes = {}<EOL>self.blockCountCodes = {}<EOL>for blockType in (L,I,D): self.blockType(blockType)<EOL>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.NPOSTFIX, self.NDIRECT = self.verboseRead(DistanceParamAlphabet())<EOL>self.readLiteralContextModes()<EOL>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.cmaps = {}<EOL>numberOfTrees = {I: self.numberOfBlockTypes[I]}<EOL>for blockType in (L,D):<EOL><INDENT>numberOfTrees[blockType] = self.contextMap(blockType)<EOL><DEDENT>print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.prefixCodes = {}<EOL>for blockType in (L,I,D):<EOL><INDENT>self.readPrefixArray(blockType, numberOfTrees[blockType])<EOL><DEDENT>self.metablock()<EOL><DEDENT>


def verboseRead(self, alphabet, context='<STR_LIT>', skipExtra=False):

<EOL>stream = self.stream<EOL>pos = stream.pos<EOL>if skipExtra:<EOL><INDENT>length, symbol = alphabet.readTuple(stream)<EOL>extraBits, extra = <NUM_LIT:0>, None<EOL><DEDENT>else:<EOL><INDENT>length, symbol, extraBits, extra = alphabet.readTupleAndExtra(<EOL>stream)<EOL><DEDENT>hexdata = self.makeHexData(pos)<EOL>addressField = '<STR_LIT>'.format(pos+<NUM_LIT:7>>><NUM_LIT:3>) if hexdata else '<STR_LIT>'<EOL>bitdata = self.formatBitData(pos, length, extraBits)<EOL>if '<STR_LIT:|>' in bitdata[<NUM_LIT:1>:]:<EOL><INDENT>self.bitPtr = self.width<EOL><DEDENT>fillWidth = self.bitPtr-(len(hexdata)+len(bitdata))<EOL>if fillWidth<<NUM_LIT:0>: fillWidth = <NUM_LIT:0><EOL>print('<STR_LIT>'.format(<EOL>addressField,<EOL>hexdata+'<STR_LIT:U+0020>'*fillWidth+bitdata, self.width,<EOL>context+alphabet.name,<EOL>symbol if skipExtra else symbol.explanation(extra),<EOL>))<EOL>if bitdata.startswith('<STR_LIT:|>'): self.bitPtr = self.width<EOL>else: self.bitPtr -= len(bitdata)<EOL>return symbol if skipExtra else symbol.value(extra)<EOL>

Read symbol and extra from stream and explain what happens.
        Returns the value of the symbol
        >>> olleke.pos = 0
        >>> l = Layout(olleke)
        >>> l.verboseRead(WindowSizeAlphabet())
        0000  1b                   1011 WSIZE   windowsize=(1<<22)-16=4194288
        4194288


self.MLEN = self.verboseRead(MetablockLengthAlphabet())<EOL>if self.MLEN:<EOL><INDENT>return False<EOL><DEDENT>self.verboseRead(ReservedAlphabet())<EOL>MSKIP = self.verboseRead(SkipLengthAlphabet())<EOL>self.verboseRead(FillerAlphabet(streamPos=self.stream.pos))<EOL>self.stream.pos += <NUM_LIT:8>*MSKIP<EOL>print("<STR_LIT>".format(self.stream.pos>><NUM_LIT:3>))<EOL>return True<EOL>

Read MNIBBLES and meta block length;
        if empty block, skip block and return true.


ISUNCOMPRESSED = self.verboseRead(<EOL>BoolCode('<STR_LIT>', description='<STR_LIT>'))<EOL>if ISUNCOMPRESSED:<EOL><INDENT>self.verboseRead(FillerAlphabet(streamPos=self.stream.pos))<EOL>print('<STR_LIT>')<EOL>self.output += self.stream.readBytes(self.MLEN)<EOL>print(outputFormatter(self.output[-self.MLEN:]))<EOL><DEDENT>return ISUNCOMPRESSED<EOL>


NBLTYPES = self.verboseRead(TypeCountAlphabet(<EOL>'<STR_LIT>'+kind[<NUM_LIT:0>].upper(),<EOL>description='<STR_LIT>'.format(kind),<EOL>))<EOL>self.numberOfBlockTypes[kind] = NBLTYPES<EOL>if NBLTYPES>=<NUM_LIT:2>:<EOL><INDENT>self.blockTypeCodes[kind] = self.readPrefixCode(<EOL>BlockTypeAlphabet('<STR_LIT>'+kind[<NUM_LIT:0>].upper(), NBLTYPES))<EOL>self.blockCountCodes[kind] = self.readPrefixCode(<EOL>BlockCountAlphabet('<STR_LIT>'+kind[<NUM_LIT:0>].upper()))<EOL>blockCount = self.verboseRead(self.blockCountCodes[kind])<EOL><DEDENT>else:<EOL><INDENT>blockCount = <NUM_LIT:1><<<NUM_LIT><EOL><DEDENT>self.currentBlockCounts[kind] = blockCount<EOL>

Read block type switch descriptor for given kind of blockType.


print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:->'))<EOL>self.literalContextModes = []<EOL>for i in range(self.numberOfBlockTypes[L]):<EOL><INDENT>self.literalContextModes.append(<EOL>self.verboseRead(LiteralContextMode(number=i)))<EOL><DEDENT>

Read literal context modes.
        LSB6: lower 6 bits of last char
        MSB6: upper 6 bits of last char
        UTF8: rougly dependent on categories:
            upper 4 bits depend on category of last char:
                control/whitespace/space/ punctuation/quote/%/open/close/
                comma/period/=/digits/ VOWEL/CONSONANT/vowel/consonant
            lower 2 bits depend on category of 2nd last char:
                space/punctuation/digit or upper/lowercase
        signed: hamming weight of last 2 chars


NTREES = self.verboseRead(TypeCountAlphabet(<EOL>kind[<NUM_LIT:0>].upper()+'<STR_LIT>',<EOL>description='<STR_LIT>'.format(kind)))<EOL>mapSize = {L:<NUM_LIT:64>, D:<NUM_LIT:4>}[kind]<EOL>if NTREES<<NUM_LIT:2>:<EOL><INDENT>self.cmaps[kind] = [<NUM_LIT:0>]*mapSize<EOL><DEDENT>else:<EOL><INDENT>RLEMAX = self.verboseRead(RLEmaxAlphabet(<EOL>'<STR_LIT>'+kind[<NUM_LIT:0>].upper(),<EOL>description=kind+'<STR_LIT>'))<EOL>alphabet = TreeAlphabet('<STR_LIT>'+kind[<NUM_LIT:0>].upper(), NTREES=NTREES, RLEMAX=RLEMAX)<EOL>cmapCode = self.readPrefixCode(alphabet)<EOL>tableSize = mapSize*self.numberOfBlockTypes[kind]<EOL>cmap = []<EOL>while len(cmap)<tableSize:<EOL><INDENT>cmapCode.description = '<STR_LIT>'.format(<EOL>*divmod(len(cmap), mapSize))<EOL>count, value = self.verboseRead(cmapCode)<EOL>cmap.extend([value]*count)<EOL><DEDENT>assert len(cmap)==tableSize<EOL>IMTF = self.verboseRead(BoolCode('<STR_LIT>', description='<STR_LIT>'))<EOL>if IMTF:<EOL><INDENT>self.IMTF(cmap)<EOL><DEDENT>if kind==L:<EOL><INDENT>print('<STR_LIT>')<EOL>for i in range(<NUM_LIT:0>, len(cmap), <NUM_LIT:64>):<EOL><INDENT>print(*(<EOL>'<STR_LIT>'.join(map(str, cmap[j:j+<NUM_LIT:8>]))<EOL>for j in range(i, i+<NUM_LIT:64>, <NUM_LIT:8>)<EOL>))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL>print(*(<EOL>'<STR_LIT>'.join(map('<STR_LIT>'.format, cmap[i:i+<NUM_LIT:4>]))<EOL>for i in range(<NUM_LIT:0>, len(cmap), <NUM_LIT:4>)<EOL>))<EOL><DEDENT>self.cmaps[kind] = cmap<EOL><DEDENT>return NTREES<EOL>

Read context maps
        Returns the number of differnt values on the context map
        (In other words, the number of prefix trees)


<EOL>mtf = []<EOL>for i, vi in enumerate(v):<EOL><INDENT>try: value = mtf.pop(vi)<EOL>except IndexError: value = vi<EOL>mtf.insert(<NUM_LIT:0>, value)<EOL>v[i] = value<EOL><DEDENT>

In place inverse move to front transform.


def readPrefixArray(self, kind, numberOfTrees):

prefixes = []<EOL>for i in range(numberOfTrees):<EOL><INDENT>if kind==L: alphabet = LiteralAlphabet(i)<EOL>elif kind==I: alphabet = InsertAndCopyAlphabet(i)<EOL>elif kind==D: alphabet = DistanceAlphabet(<EOL>i, NPOSTFIX=self.NPOSTFIX, NDIRECT=self.NDIRECT)<EOL>self.readPrefixCode(alphabet)<EOL>prefixes.append(alphabet)<EOL><DEDENT>self.prefixCodes[kind] = prefixes<EOL>


print('<STR_LIT>'.center(<NUM_LIT>, '<STR_LIT:=>'))<EOL>self.currentBlockTypes = {L:<NUM_LIT:0>, I:<NUM_LIT:0>, D:<NUM_LIT:0>, pL:<NUM_LIT:1>, pI:<NUM_LIT:1>, pD:<NUM_LIT:1>}<EOL>self.lastDistances = deque([<NUM_LIT>,<NUM_LIT:16>,<NUM_LIT:11>,<NUM_LIT:4>], maxlen=<NUM_LIT:4>)<EOL>self.contextMode = ContextModeKeeper(self.literalContextModes[<NUM_LIT:0>])<EOL>wordList = WordList()<EOL>def distanceCallback(symbol, extra):<EOL><INDENT>"<STR_LIT>"<EOL>index, offset = symbol.value(extra)<EOL>if index:<EOL><INDENT>distance = self.lastDistances[-index]+offset<EOL>return '<STR_LIT>'.format(index, offset, distance)<EOL><DEDENT>if offset<=maxDistance:<EOL><INDENT>return '<STR_LIT>'.format(offset, maxDistance-offset)<EOL><DEDENT>action, word = divmod(offset-maxDistance, <NUM_LIT:1><<wordList.NDBITS[copyLen])<EOL>return '<STR_LIT>'.format(<EOL>offset, maxDistance, copyLen, word, action)<EOL><DEDENT>for dpc in self.prefixCodes[D]: dpc.callback = distanceCallback<EOL>blockLen = <NUM_LIT:0><EOL>while blockLen<self.MLEN:<EOL><INDENT>litLen, copyLen, dist0Flag = self.verboseRead(<EOL>self.prefixCodes[I][<EOL>self.figureBlockType(I)])<EOL>for i in range(litLen):<EOL><INDENT>bt = self.figureBlockType(L)<EOL>cm = self.contextMode.getIndex()<EOL>ct = self.cmaps[L][bt<<<NUM_LIT:6>|cm]<EOL>char = self.verboseRead(<EOL>self.prefixCodes[L][ct],<EOL>context='<STR_LIT>'.format(bt,cm))<EOL>self.contextMode.add(char)<EOL>self.output.append(char)<EOL><DEDENT>blockLen += litLen<EOL>if blockLen>=self.MLEN: return<EOL>maxDistance = min(len(self.output), self.windowSize)<EOL>if dist0Flag:<EOL><INDENT>distance = self.lastDistances[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>bt = self.figureBlockType(D)<EOL>cm = {<NUM_LIT:2>:<NUM_LIT:0>, <NUM_LIT:3>:<NUM_LIT:1>, <NUM_LIT:4>:<NUM_LIT:2>}.get(copyLen, <NUM_LIT:3>)<EOL>ct = self.cmaps[D][bt<<<NUM_LIT:2>|cm]<EOL>index, offset = self.verboseRead(<EOL>self.prefixCodes[D][ct],<EOL>context='<STR_LIT>'.format(bt,cm))<EOL>distance = self.lastDistances[-index]+offset if index else offset<EOL>if index==<NUM_LIT:1> and offset==<NUM_LIT:0>:<EOL><INDENT>dist0Flag = True<EOL><DEDENT><DEDENT>if distance<=maxDistance:<EOL><INDENT>for i in range(<EOL>maxDistance-distance,<EOL>maxDistance-distance+copyLen):<EOL><INDENT>self.output.append(self.output[i])<EOL><DEDENT>if not dist0Flag: self.lastDistances.append(distance)<EOL>comment = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>newWord = wordList.word(copyLen, distance-maxDistance-<NUM_LIT:1>)<EOL>self.output.extend(newWord)<EOL>copyLen = len(newWord)<EOL>comment = '<STR_LIT>'<EOL><DEDENT>blockLen += copyLen<EOL>print('<STR_LIT:U+0020>'*<NUM_LIT>,<EOL>comment,<EOL>'<STR_LIT>',<EOL>outputFormatter(self.output[-copyLen:]),<EOL>'<STR_LIT:">',<EOL>sep='<STR_LIT>')<EOL>self.contextMode.add(self.output[-<NUM_LIT:2>])<EOL>self.contextMode.add(self.output[-<NUM_LIT:1>])<EOL><DEDENT>

Process the data.
        Relevant variables of self:
        numberOfBlockTypes[kind]: number of block types
        currentBlockTypes[kind]: current block types (=0)
        literalContextModes: the context modes for the literal block types
        currentBlockCounts[kind]: counters for block types
        blockTypeCodes[kind]: code for block type
        blockCountCodes[kind]: code for block count
        cmaps[kind]: the context maps (not for I)
        prefixCodes[kind][#]: the prefix codes
        lastDistances: the last four distances
        lastChars: the last two chars
        output: the result


version_file_path = os.path.join(CURR_DIR, '<STR_LIT:c>', '<STR_LIT>', '<STR_LIT>')<EOL>version = <NUM_LIT:0><EOL>with open(version_file_path, '<STR_LIT:r>') as f:<EOL><INDENT>for line in f:<EOL><INDENT>m = re.match(r'<STR_LIT>', line)<EOL>if m:<EOL><INDENT>version = int(m.group(<NUM_LIT:1>), <NUM_LIT:16>)<EOL><DEDENT><DEDENT><DEDENT>if version == <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>major = version >> <NUM_LIT><EOL>minor = (version >> <NUM_LIT:12>) & <NUM_LIT><EOL>patch = version & <NUM_LIT><EOL>return '<STR_LIT>'.format(major, minor, patch)<EOL>

Return BROTLI_VERSION string as defined in 'common/version.h' file.


args, _, _, defaults, *rest = inspect.getfullargspec(func)<EOL>return dict(zip(reversed(args), reversed(defaults)))<EOL>

returns a dictionary of arg_name:default_values for the input function


def get_object_key(object_, object_key, object_tuple_key):

if object_tuple_key:<EOL><INDENT>object_ = getattr(object_, object_tuple_key)<EOL><DEDENT>if isinstance(object_key, (list, tuple)):<EOL><INDENT>return_string = '<STR_LIT>'<EOL>for element in object_key:<EOL><INDENT>return_string += str(getattr(object_, element))<EOL>return_string += '<STR_LIT:_>'<EOL><DEDENT>return return_string<EOL><DEDENT>return str(getattr(object_, object_key))<EOL>

:param object_: Object you want to pull a value fron
:param object_key: String or list/tuple of strings containing field names you want to generate a key from
:param object_tuple_key: A temporary shortcut until we allow dot.path traversal for object_key.
Will call getattr(getattr(result, join_table_name), object_key)
:return:


def get_argument_key(kwargs, argument_key, index):

if isinstance(argument_key, (list, tuple)):<EOL><INDENT>return_string = '<STR_LIT>'<EOL>for element in argument_key:<EOL><INDENT>return_val = kwargs.get(element)[index]<EOL>if return_val is None:<EOL><INDENT>return_val = '<STR_LIT>'<EOL><DEDENT>return_string += str(return_val)<EOL>if len(argument_key) > <NUM_LIT:1>:<EOL><INDENT>return_string += '<STR_LIT:_>'<EOL><DEDENT><DEDENT>return return_string<EOL><DEDENT>return str(kwargs.get(argument_key)[index] or '<STR_LIT>')<EOL>

:param kwargs: keyword arguments used to kickoff the multiget
:param argument_key: Name or set of names of keyword arguments to read from
:param index: which set of arguments are considered
:return:


def map_arguments_to_objects(kwargs, objects, object_key, object_tuple_key, argument_key, result_value, default_result):

<EOL>map_ = map_objects_to_result(objects, object_key, object_tuple_key, result_value, default_result)<EOL>element_count = get_request_count(kwargs)<EOL>return [map_[get_argument_key(kwargs, argument_key, index)] for index in range(<NUM_LIT:0>, element_count)]<EOL>

:param kwargs: kwargs used to call the multiget function
:param objects: objects returned from the inner function
:param object_key: field or set of fields that map to the kwargs provided
:param object_tuple_key: A temporary shortcut until we allow dot.path traversal for object_key.
Will call getattr(getattr(result, join_table_name), object_key)
:param argument_key: field or set of fields that map to the objects provided
:param result_value: Limit the fields returned to this field or set of fields (none = whole object)
:param default_result: If the inner function returned none for a set of parameters, default to this
:return:


@abstractmethod<EOL><INDENT>def get(self, k, d=None):<DEDENT>

pass<EOL>

D.get(k[,d]) -> D[k] if k in D, else d.  d defaults to None.


pass<EOL>


pass<EOL>

D.items() -> a set-like object providing a view on D's items


@abstractmethod<EOL><INDENT>def __setitem__(self, *args, **kwargs):<DEDENT>

pass<EOL>


pass<EOL>

D.clear() -> None.  Remove all items from D.


@abstractmethod<EOL><INDENT>def __delitem__(self, *args, **kwargs):  <DEDENT>

pass<EOL>


@abstractmethod<EOL><INDENT>def __contains__(self, *args, **kwargs):<DEDENT>

pass<EOL>


key = self.get_cache_key(*args, **kwargs)<EOL>cache = get_cache()<EOL>if key in cache:<EOL><INDENT>return cache[key]<EOL><DEDENT>else:<EOL><INDENT>key = self.get_cache_key(*args, **kwargs)<EOL>result = self.inner_f(*args, **kwargs)<EOL>cache[key] = result<EOL>return result<EOL><DEDENT>

Pull the return value from the request cache if possible. If not,
        pull the return value from memcache if possible. If not, pull the
        value from calling inner_f, then set the value in request cache
        and memcache where needed.


cache = get_cache()<EOL>key = self.get_cache_key(*args)<EOL>if key in cache:<EOL><INDENT>del cache[key]<EOL><DEDENT>

Remove the key from the request cache and from memcache.


def multiget_cached(object_key, argument_key=None, default_result=None,<EOL>result_fields=None, join_table_name=None, coerce_args_to_strings=False):

def create_wrapper(inner_f):<EOL><INDENT>return MultigetCacheWrapper(<EOL>inner_f, object_key, argument_key, default_result, result_fields,<EOL>join_table_name, coerce_args_to_strings=coerce_args_to_strings<EOL>)<EOL><DEDENT>return create_wrapper<EOL>

:param object_key: the names of the attributes on the result object that are meant to match the function parameters
:param argument_key: the function parameter names you wish to match with the `object_key`s.
By default, this will be all of your wrapped function's arguments, in order.
So, you'd really only use this when you want to ignore a given function argument.
:param default_result: The result to put into the cache if nothing is matched.
:param result_fields: The attribute on your result object you wish to return the value of.
By default, the whole object is returned.
:param join_table_name: A temporary shortcut until we allow dot.path traversal for object_key.
Will call getattr(getattr(result, join_table_name), object_key)
:param coerce_args_to_strings: force coerce all arguments to the inner function to strings.
Useful for SQL where mixes of ints and strings in `WHERE x IN (list)` clauses causes poor performance.
:return: A wrapper that allows you to queue many O(1) calls and flush the queue all at once,
rather than executing the inner function body N times.


key = self.get_cache_key(*args, **kwargs)<EOL>cache = get_cache()<EOL>if key in cache:<EOL><INDENT>return cache[key]<EOL><DEDENT>else:<EOL><INDENT>self.prime(*args, **kwargs)<EOL>self._issue_gets_for_primes()<EOL>return cache[key]<EOL><DEDENT>

Pull the return value from the request cache if possible. If not,
pull the value from calling inner_f, then set the value in request cache


global primary_cache<EOL>primary_cache = cache<EOL>

Either an object conforming to the CacheInterface, or a function which returns one


return [<EOL>(tag.name, "<STR_LIT>" if tag.name in tags else "<STR_LIT>")<EOL>for tag in self.model.objects.all()<EOL>]<EOL>

Generates a list of tags identifying those previously selected.

Returns a list of tuples of the form (<tag name>, <CSS class name>).

Uses the string names rather than the tags themselves in order to work
with tag lists built from forms not fully submitted.


obj_dict = obj.to_dict()<EOL>for key, val in obj_dict.iteritems():<EOL><INDENT>if isinstance(val, types.StringType):<EOL><INDENT>try:<EOL><INDENT>unicode(val)<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>obj_dict[key] = base64.b64encode(val)<EOL><DEDENT><DEDENT><DEDENT>return obj_dict<EOL>

Encode objects like ndb.Model which have a `.to_dict()` method.


return list(obj)<EOL>

Encode generator-like objects, such as ndb.Query.


<EOL>return obj.get_async()<EOL>

Get the Entity from the ndb.Key for further encoding.


return obj.pairs()<EOL>

Get the ndb.Key as a tuple of (kind, id) pairs.


return obj.urlsafe()<EOL>

Get the ndb.Key as URL-safe base64-encoded string.


return obj.get_result()<EOL>


<EOL>zone = '<STR_LIT>' if getattr(obj, '<STR_LIT>', True) else '<STR_LIT>'<EOL>return obj.isoformat() + zone<EOL>

Encode a datetime.datetime or datetime.date object as an ISO 8601 format string.


return [obj.real, obj.imag]<EOL>

Convert a complex number object into a list containing the real and imaginary values.


return obj.b_val<EOL>

Retrieve the actual value from a ndb.model._BaseValue.

    This is a convenience function to assist with the following issue:
    https://code.google.com/p/appengine-ndb-experiment/issues/detail?id=208


return NdbEncoder(**kwargs).encode(ndb_model)<EOL>

Custom json dumps using the custom encoder above.


for chunk in NdbEncoder(**kwargs).iterencode(ndb_model):<EOL><INDENT>fp.write(chunk)<EOL><DEDENT>

Custom json dump using the custom encoder above.


return NdbDecoder(**kwargs).decode(json_str)<EOL>

Custom json loads function that converts datetime strings.


json.JSONDecoder.__init__(self, object_hook=self.object_hook_handler, **kwargs)<EOL>

Override the default __init__ in order to specify our own parameters.


return {k: self.decode_date(v) for k, v in val.iteritems()}<EOL>

Handles decoding of nested date strings.


if isinstance(val, basestring) and val.count('<STR_LIT:->') == <NUM_LIT:2> and len(val) > <NUM_LIT:9>:<EOL><INDENT>try:<EOL><INDENT>dt = dateutil.parser.parse(val)<EOL>if val.endswith(('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>dt = dt.replace(tzinfo=None)<EOL><DEDENT>return dt<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return val<EOL>

Tries to decode strings that look like dates into datetime objects.


<EOL>new_val = self.decode_date(val)<EOL>if val != new_val:<EOL><INDENT>return new_val<EOL><DEDENT>return json.JSONDecoder.decode(self, val)<EOL>

Override of the default decode method that also uses decode_date.


obj_type = type(obj)<EOL>if obj_type not in self._ndb_type_encoding:<EOL><INDENT>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj_type = obj.__metaclass__<EOL><DEDENT>else:<EOL><INDENT>for ndb_type in NDB_TYPES:<EOL><INDENT>if isinstance(obj, ndb_type):<EOL><INDENT>obj_type = ndb_type<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>fn = self._ndb_type_encoding.get(obj_type)<EOL>if fn:<EOL><INDENT>return fn(obj)<EOL><DEDENT>return json.JSONEncoder.default(self, obj)<EOL>

Overriding the default JSONEncoder.default for NDB support.


try:<EOL><INDENT>return modules.get_current_version_name()<EOL><DEDENT>except KeyError:<EOL><INDENT>return None<EOL><DEDENT>

Returns the current version of the app, or None if there is no current version found.


try:<EOL><INDENT>return modules.get_current_module_name()<EOL><DEDENT>except KeyError:<EOL><INDENT>return None<EOL><DEDENT>

Returns the current module of the app, or None if there is no current module found..


return os.environ.get('<STR_LIT>', '<STR_LIT>').startswith('<STR_LIT>') or get_hostname().endswith('<STR_LIT>')<EOL>

True if the app is being hosted from Google App Engine servers.


version = version or get_current_version_name()<EOL>return version == get_default_version()<EOL>

True if the current or specified app version is the default.


version = version or get_current_version_name_safe()<EOL>return version == get_default_version()<EOL>

True if the current or specified app version is the default.
Returns False when there is no version found.


return os.environ.get('<STR_LIT>', '<STR_LIT>').startswith('<STR_LIT>')<EOL>

True if the dev_appserver is running (localhost or local development server).


return is_host_google() and not is_default_version(version)<EOL>

True if the app is hosted by Google (appspot.com) but the version is not the default.


is_default_version = is_default_version_safe()<EOL>if is_default_version is None:<EOL><INDENT>return False<EOL><DEDENT>return is_host_google() and not is_default_version<EOL>


return is_host_google() and is_default_version(version)<EOL>

True if the app is being hosted by Google and the default version.


is_default_version = is_default_version_safe(version)<EOL>if is_default_version is None:<EOL><INDENT>return False<EOL><DEDENT>return is_host_google() and is_default_version<EOL>


version = version or get_current_version_name()<EOL>module = module or get_current_module_name()<EOL>return '<STR_LIT>'.join((version, module))<EOL>

Returns the current version/module in -dot- notation which is used by `target:` parameters.


version = version or get_current_version_name_safe()<EOL>module = module or get_current_module_name_safe()<EOL>if version and module:<EOL><INDENT>return '<STR_LIT>'.join((version, module))<EOL><DEDENT>return None<EOL>

Returns the current version/module in -dot- notation which is used by `target:` parameters.
If there is no current version or module then None is returned.


return {k: os.environ.get(k, _UNDEFINED) for k in keys}<EOL>

Return a dictionary of key/values from os.environ.


return {k: getattr(app_identity, k)() for k in keys}<EOL>

Return a dictionary of key/values from the app_identity module functions.


return {k: getattr(modules, k)() for k in keys}<EOL>

Return a dictionary of key/values from the modules module functions.


return {k: getattr(namespace_manager, k)() for k in keys}<EOL>

Return a dictionary of key/values from the namespace_manager module functions.


Return a dictionary of all environment keys/values.


self.workers = workers<EOL>self.stopper = stopper<EOL>

Creates a SignalHandler associated with all the worker threads and a stopper Event object.

:param list workers: List of all the worker threads from the Worker class
:param threading.Event stopper: Event that signals that the thread should stop execution


log.info('<STR_LIT>')<EOL>self.stopper.set()<EOL>for worker in self.workers:<EOL><INDENT>worker.join(<NUM_LIT:1>)<EOL><DEDENT>sys.exit(<NUM_LIT:0>)<EOL>

Gets called by Python's signal module when the threads should be stopped.

:param signal signum: Signal number that must be handled
:param Frame frame: Current stack frame


def use_music_service(self, service_name, api_key):

try:<EOL><INDENT>self.current_music = self.music_services[service_name]<EOL><DEDENT>except KeyError:<EOL><INDENT>if service_name == '<STR_LIT>':<EOL><INDENT>self.music_services['<STR_LIT>'] = Youtube()<EOL>self.current_music = self.music_services['<STR_LIT>']<EOL><DEDENT>elif service_name == '<STR_LIT>':<EOL><INDENT>self.music_services['<STR_LIT>'] = Soundcloud(api_key=api_key)<EOL>self.current_music = self.music_services['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT><DEDENT>

Sets the current music service to service_name.

:param str service_name: Name of the music service
:param str api_key: Optional API key if necessary


def use_storage_service(self, service_name, custom_path):

try:<EOL><INDENT>self.current_storage = self.storage_services[service_name]<EOL><DEDENT>except KeyError:<EOL><INDENT>if service_name == '<STR_LIT>':<EOL><INDENT>self.storage_services['<STR_LIT>'] = GoogleDrive()<EOL>self.current_storage = self.storage_services['<STR_LIT>']<EOL>self.current_storage.connect()<EOL><DEDENT>elif service_name == '<STR_LIT>':<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT>elif service_name == '<STR_LIT>':<EOL><INDENT>self.storage_services['<STR_LIT>'] = LocalStorage(custom_path=custom_path)<EOL>self.current_storage = self.storage_services['<STR_LIT>']<EOL>self.current_storage.connect()<EOL><DEDENT>else:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT><DEDENT>

Sets the current storage service to service_name and runs the connect method on the service.

:param str service_name: Name of the storage service
:param str custom_path: Custom path where to download tracks for local storage (optional, and must already exist, use absolute paths only)


self.queues = {<EOL>'<STR_LIT>': Queue(),<EOL>'<STR_LIT>': Queue(),<EOL>'<STR_LIT>': Queue(),<EOL>'<STR_LIT>': Queue(),<EOL>'<STR_LIT>': Queue(),<EOL>}<EOL>self.connection_handler = ConnectionHandler()<EOL>self.workers = []<EOL>self.stopper = Event()<EOL>self.signal_handler = None<EOL>

Initializes all the queues and sets default values for services and workers.


if self.connection_handler.current_music is None:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT>elif self.connection_handler.current_storage is None:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>self.queues['<STR_LIT>'].put(url)<EOL><DEDENT>

Adds an URL to the download queue.

:param str url: URL to the music service track


def use_music_service(self, service_name, api_key=None):

self.connection_handler.use_music_service(service_name, api_key=api_key)<EOL>


def use_storage_service(self, service_name, custom_path=None):

self.connection_handler.use_storage_service(service_name, custom_path=custom_path)<EOL>

Sets the current storage service to service_name and attempts to connect to it.

:param str service_name: Name of the storage service
:param str custom_path: Custom path where to download tracks for local storage (optional, and must already exist, use absolute paths only)


if not self.workers:<EOL><INDENT>for _ in range(workers_per_task):<EOL><INDENT>self.workers.append(Worker(self._download, self.queues['<STR_LIT>'], self.queues['<STR_LIT>'], self.stopper))<EOL>self.workers.append(Worker(self._convert, self.queues['<STR_LIT>'], self.queues['<STR_LIT>'], self.stopper))<EOL>self.workers.append(Worker(self._upload, self.queues['<STR_LIT>'], self.queues['<STR_LIT>'], self.stopper))<EOL>self.workers.append(Worker(self._delete, self.queues['<STR_LIT>'], self.queues['<STR_LIT>'], self.stopper))<EOL><DEDENT>self.signal_handler = SignalHandler(self.workers, self.stopper)<EOL>signal.signal(signal.SIGINT, self.signal_handler)<EOL>for worker in self.workers:<EOL><INDENT>worker.start()<EOL><DEDENT><DEDENT>

Creates and starts the workers, as well as attaching a handler to terminate them gracefully when a SIGINT signal is received.

:param int workers_per_task: Number of workers to create for each task in the pipeline


return self.connection_handler.current_music.download(url)<EOL>

Downloads the file associated with the given URL.

:param str url: URL of the video/audio file
:return str: Filename of the file in local storage


return convert_to_mp3(file_name, self.queues['<STR_LIT>'])<EOL>

Converts the file associated with the file_name passed into a MP3 file.

:param str file_name: Filename and extension of the file to be converted
:return str: Filename of the new file in local storage


return self.connection_handler.current_storage.upload(file_name)<EOL>

Uploads the file associated with the file_name passed to the storage service.

:param str file_name: Filename of the file to be uploaded
:return str: Original filename passed as an argument


return delete_local_file(file_name)<EOL>

Deletes the file associated with the file_name passed from local storage.

:param str file_name: Filename of the file to be deleted
:return str: Filename of the file that was just deleted


pass<EOL>

TODO: Notify that the MP3 has been uploaded through a messaging service.


file = os.path.splitext(file_name)<EOL>if file[<NUM_LIT:1>] == '<STR_LIT>':<EOL><INDENT>log.info(f"<STR_LIT>")<EOL>return file_name<EOL><DEDENT>new_file_name = file[<NUM_LIT:0>] + '<STR_LIT>'<EOL>ff = FFmpeg(<EOL>inputs={file_name: None},<EOL>outputs={new_file_name: None}<EOL>)<EOL>log.info(f"<STR_LIT>")<EOL>start_time = time()<EOL>try:<EOL><INDENT>ff.run(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)<EOL><DEDENT>except FFRuntimeError:<EOL><INDENT>os.remove(new_file_name)<EOL>ff.run(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)<EOL><DEDENT>end_time = time()<EOL>log.info(f"<STR_LIT>")<EOL>delete_queue.put(file_name)<EOL>return new_file_name<EOL>

Converts the file associated with the file_name passed into a MP3 file.

:param str file_name: Filename of the original file in local storage
:param Queue delete_queue: Delete queue to add the original file to after conversion is done
:return str: Filename of the new file in local storage


try:<EOL><INDENT>os.remove(file_name)<EOL>log.info(f"<STR_LIT>")<EOL>return file_name<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>


def __init__(self, func, in_queue, out_queue, stopper):

super().__init__()<EOL>self.func = func<EOL>self.in_queue = in_queue<EOL>self.out_queue = out_queue<EOL>self.stopper = stopper<EOL>

Builds a worker by taking a function that does work on items from the input queue to be put in the output queue.

:param function func: Function that does work on items from the input queue
:param Queue in_queue: Input queue
:param Queue out_queue: Output queue
:param threading.Event stopper: Event that signals that the thread should stop execution


while not self.stopper.is_set():<EOL><INDENT>try:<EOL><INDENT>item = self.in_queue.get(timeout=<NUM_LIT:5>)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>result = self.func(item)<EOL><DEDENT>except TypeError:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>self.out_queue.put(result)<EOL><DEDENT><DEDENT>

Method that gets run when the Worker thread is started.

When there's an item in in_queue, it takes it out, passes it to func as an argument, and puts the result in out_queue.


Downloads a song file from the music service.


Connects to the storage service, sets connection attribute.


try:<EOL><INDENT>yt = YouTube(url)<EOL><DEDENT>except RegexMatchError:<EOL><INDENT>log.error(f"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>stream = yt.streams.first()<EOL>log.info(f"<STR_LIT>")<EOL>start_time = time()<EOL>stream.download()<EOL>end_time = time()<EOL>log.info(f"<STR_LIT>")<EOL>return stream.default_filename<EOL><DEDENT>

Downloads a MP4 or WebM file that is associated with the video at the URL passed.

:param str url: URL of the video to be downloaded
:return str: Filename of the file in local storage


try:<EOL><INDENT>track = self.client.get('<STR_LIT>', url=url)<EOL><DEDENT>except HTTPError:<EOL><INDENT>log.error(f"<STR_LIT>")<EOL>return<EOL><DEDENT>r = requests.get(self.client.get(track.stream_url, allow_redirects=False).location, stream=True)<EOL>total_size = int(r.headers['<STR_LIT>'])<EOL>chunk_size = <NUM_LIT><EOL>file_name = track.title + '<STR_LIT>'<EOL>with open(file_name, '<STR_LIT:wb>') as f:<EOL><INDENT>for data in tqdm(r.iter_content(chunk_size), desc=track.title, total=total_size / chunk_size, unit='<STR_LIT>', file=sys.stdout):<EOL><INDENT>f.write(data)<EOL><DEDENT><DEDENT>return file_name<EOL>

Downloads a MP3 file that is associated with the track at the URL passed.

:param str url: URL of the track to be downloaded


SCOPES = '<STR_LIT>'<EOL>store = file.Storage('<STR_LIT>')<EOL>creds = store.get()<EOL>if not creds or creds.invalid:<EOL><INDENT>try:<EOL><INDENT>flow = client.flow_from_clientsecrets('<STR_LIT>', SCOPES)<EOL><DEDENT>except InvalidClientSecretsError:<EOL><INDENT>log.error('<STR_LIT>')<EOL>return<EOL><DEDENT>creds = tools.run_flow(flow, store)<EOL><DEDENT>self.connection = build('<STR_LIT>', '<STR_LIT>', http=creds.authorize(Http()))<EOL>response = self.connection.files().list(q="<STR_LIT>").execute()<EOL>try:<EOL><INDENT>folder_id = response.get('<STR_LIT>', [])[<NUM_LIT:0>]['<STR_LIT:id>']<EOL><DEDENT>except IndexError:<EOL><INDENT>log.warning('<STR_LIT>')<EOL>folder_metadata = {'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>'}<EOL>folder = self.connection.files().create(body=folder_metadata, fields='<STR_LIT:id>').execute()<EOL><DEDENT>

Creates connection to the Google Drive API, sets the connection attribute to make requests, and creates the Music folder if it doesn't exist.


response = self.connection.files().list(q="<STR_LIT>").execute()<EOL>folder_id = response.get('<STR_LIT>', [])[<NUM_LIT:0>]['<STR_LIT:id>']<EOL>file_metadata = {'<STR_LIT:name>': file_name, '<STR_LIT>': [folder_id]}<EOL>media = MediaFileUpload(file_name, mimetype='<STR_LIT>')<EOL>log.info(f"<STR_LIT>")<EOL>start_time = time()<EOL>self.connection.files().create(body=file_metadata, media_body=media, fields='<STR_LIT:id>').execute()<EOL>end_time = time()<EOL>log.info(f"<STR_LIT>")<EOL>return file_name<EOL>

Uploads the file associated with the file_name passed to Google Drive in the Music folder.

:param str file_name: Filename of the file to be uploaded
:return str: Original filename passed as an argument (in order for the worker to send it to the delete queue)


if self.music_folder is None:<EOL><INDENT>music_folder = os.path.join(os.path.expanduser('<STR_LIT>'), '<STR_LIT>')<EOL>if not os.path.exists(music_folder):<EOL><INDENT>os.makedirs(music_folder)<EOL><DEDENT>self.music_folder = music_folder<EOL><DEDENT>

Initializes the connection attribute with the path to the user home folder's Music folder, and creates it if it doesn't exist.


log.info(f"<STR_LIT>")<EOL>start_time = time()<EOL>os.rename(file_name, os.path.join(self.music_folder, file_name))<EOL>end_time = time()<EOL>log.info(f"<STR_LIT>")<EOL>

Moves the file associated with the file_name passed to the Music folder in the local storage.

:param str file_name: Filename of the file to be uploaded


def _convolution_integrand(t_val, f, g,<EOL>inverse_time=None, return_log=False):

if inverse_time is None:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if inverse_time:<EOL><INDENT>tau_min = max(t_val - f.xmax, g.xmin)<EOL>tau_max = min(t_val - f.xmin, g.xmax)<EOL><DEDENT>else:<EOL><INDENT>tau_min = max(f.xmin-t_val, g.xmin)<EOL>tau_max = min(f.xmax-t_val, g.xmax)<EOL><DEDENT>if tau_max <= tau_min:<EOL><INDENT>if return_log:<EOL><INDENT>return ttconf.BIG_NUMBER<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0.0> <EOL><DEDENT><DEDENT>else:<EOL><INDENT>if inverse_time:<EOL><INDENT>tau = np.unique(np.concatenate((g.x, t_val-f.x,[tau_min,tau_max])))<EOL><DEDENT>else:<EOL><INDENT>tau = np.unique(np.concatenate((g.x, f.x-t_val,[tau_min,tau_max])))<EOL><DEDENT>tau = tau[(tau>tau_min-ttconf.TINY_NUMBER)&(tau<tau_max+ttconf.TINY_NUMBER)]<EOL>if len(tau)<<NUM_LIT:10>:<EOL><INDENT>tau = np.linspace(tau_min, tau_max, <NUM_LIT:10>)<EOL><DEDENT>if inverse_time: <EOL><INDENT>tnode = t_val - tau<EOL>fg = f(tnode) + g(tau, tnode=tnode)<EOL><DEDENT>else:<EOL><INDENT>fg = f(t_val + tau) + g(tau, tnode=t_val)<EOL><DEDENT>FG = Distribution(tau, fg, is_log=True, min_width = np.max([f.min_width, g.min_width]),<EOL>kind='<STR_LIT>', assume_sorted=True)<EOL>return FG<EOL><DEDENT>

Evaluates int_tau f(t+tau)*g(tau) or int_tau f(t-tau)g(tau) if inverse time is TRUE

Parameters
-----------

 t_val : double
    Time point

 f : Interpolation object
    First multiplier in convolution

 g : Interpolation object
    Second multiplier in convolution

 inverse_time : bool, None
    time direction. If True, then the f(t-tau)*g(tau) is calculated, otherwise,
    f(t+tau)*g(tau)

 return_log : bool
    If True, the logarithm will be returned

Returns
-------

 FG : Distribution
    The function to be integrated as Distribution object (interpolator)


def _max_of_integrand(t_val, f, g, inverse_time=None, return_log=False):

<EOL>FG = _convolution_integrand(t_val, f, g, inverse_time, return_log=True)<EOL>if FG == ttconf.BIG_NUMBER:<EOL><INDENT>res = ttconf.BIG_NUMBER, <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>X = FG.x[FG.y.argmin()]<EOL>Y = FG.y.min()<EOL>res =  Y, X<EOL><DEDENT>if not return_log:<EOL><INDENT>res[<NUM_LIT:0>] = np.log(res[<NUM_LIT:0>])<EOL><DEDENT>return res<EOL>

Evaluates max_tau f(t+tau)*g(tau) or max_tau f(t-tau)g(tau) if inverse time is TRUE

Parameters
-----------

 t_val : double
    Time point

 f : Interpolation object
    First multiplier in convolution

 g : Interpolation object
    Second multiplier in convolution

 inverse_time : bool, None
    time direction. If True, then the f(t-tau)*g(tau) is calculated, otherwise,
    f(t+tau)*g(tau)

 return_log : bool
    If True, the logarithm will be returned

Returns
-------

 FG : Distribution
    The function to be integrated as Distribution object (interpolator)


def _evaluate_convolution(t_val, f, g,  n_integral = <NUM_LIT:100>, inverse_time=None, return_log=False):

FG = _convolution_integrand(t_val, f, g, inverse_time, return_log)<EOL>if (return_log and FG == ttconf.BIG_NUMBER) or(not return_log and FG == <NUM_LIT:0.0>): <EOL><INDENT>res = ttconf.BIG_NUMBER <EOL><DEDENT>else:<EOL><INDENT>res = -FG.integrate(a=FG.xmin, b=FG.xmax, n=n_integral, return_log=True)<EOL><DEDENT>if return_log:<EOL><INDENT>return res, -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return np.exp(-res), -<NUM_LIT:1><EOL><DEDENT>

Calculate convolution F(t) = int { f(tau)g(t-tau) } dtau


@classmethod<EOL><INDENT>def convolve(cls, node_interp, branch_interp, max_or_integral='<STR_LIT>',<EOL>n_grid_points = ttconf.NODE_GRID_SIZE, n_integral=ttconf.N_INTEGRAL,<EOL>inverse_time=True, rel_tol=<NUM_LIT>, yc=<NUM_LIT:10>):<DEDENT>

if max_or_integral not in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>raise Exception("<STR_LIT>"<EOL>+ str(max_or_integral)  + "<STR_LIT>")<EOL><DEDENT>def conv_in_point(time_point):<EOL><INDENT>if max_or_integral == '<STR_LIT>': <EOL><INDENT>return _evaluate_convolution(time_point, node_interp, branch_interp,<EOL>n_integral=n_integral, return_log=True,<EOL>inverse_time = inverse_time)<EOL><DEDENT>else: <EOL><INDENT>return _max_of_integrand(time_point, node_interp, branch_interp,<EOL>return_log=True, inverse_time = inverse_time)<EOL><DEDENT><DEDENT>joint_fwhm  = (node_interp.fwhm + branch_interp.fwhm)<EOL>min_fwhm  = min(node_interp.fwhm, branch_interp.fwhm)<EOL>if inverse_time:<EOL><INDENT>new_peak_pos = node_interp.peak_pos + branch_interp.peak_pos<EOL>tmin = node_interp.xmin+branch_interp.xmin<EOL>tmax = node_interp.xmax+branch_interp.xmax<EOL><DEDENT>else:<EOL><INDENT>new_peak_pos = node_interp.peak_pos - branch_interp.peak_pos<EOL>tmin = node_interp.xmin - branch_interp.xmax<EOL>tmax = node_interp.xmax - branch_interp.xmin<EOL><DEDENT>n = n_grid_points/<NUM_LIT:3><EOL>center_width = <NUM_LIT:3>*joint_fwhm<EOL>grid_center = new_peak_pos + np.linspace(-<NUM_LIT:1>, <NUM_LIT:1>, n)*center_width<EOL>right_range = (tmax - grid_center[-<NUM_LIT:1>])<EOL>if right_range><NUM_LIT:4>*center_width:<EOL><INDENT>grid_right = grid_center[-<NUM_LIT:1>] + right_range*(np.linspace(<NUM_LIT:0>, <NUM_LIT:1>, n)**<NUM_LIT>)<EOL><DEDENT>elif right_range><NUM_LIT:0>: <EOL><INDENT>grid_right = grid_center[-<NUM_LIT:1>] + right_range*np.linspace(<NUM_LIT:0>,<NUM_LIT:1>, int(min(n,<NUM_LIT:1>+<NUM_LIT:0.5>*n*right_range/center_width)))<EOL><DEDENT>else:<EOL><INDENT>grid_right =[]<EOL><DEDENT>left_range = grid_center[<NUM_LIT:0>]-tmin<EOL>if left_range><NUM_LIT:4>*center_width:<EOL><INDENT>grid_left = tmin + left_range*(np.linspace(<NUM_LIT:0>, <NUM_LIT:1>, n)**<NUM_LIT>)<EOL><DEDENT>elif left_range><NUM_LIT:0>:<EOL><INDENT>grid_left = tmin + left_range*np.linspace(<NUM_LIT:0>,<NUM_LIT:1>, int(min(n,<NUM_LIT:1>+<NUM_LIT:0.5>*n*left_range/center_width)))<EOL><DEDENT>else:<EOL><INDENT>grid_left =[]<EOL><DEDENT>if tmin>-<NUM_LIT:1>:<EOL><INDENT>grid_zero_left = tmin + (tmax-tmin)*np.linspace(<NUM_LIT:0>,<NUM_LIT>,<NUM_LIT:11>)**<NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>grid_zero_left = [tmin]<EOL><DEDENT>if tmax<<NUM_LIT:1>:<EOL><INDENT>grid_zero_right = tmax - (tmax-tmin)*np.linspace(<NUM_LIT:0>,<NUM_LIT>,<NUM_LIT:11>)**<NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>grid_zero_right = [tmax]<EOL><DEDENT>t_grid_0 = np.unique(np.concatenate([grid_zero_left, grid_left[:-<NUM_LIT:1>], grid_center, grid_right[<NUM_LIT:1>:], grid_zero_right]))<EOL>t_grid_0 = t_grid_0[(t_grid_0 > tmin-ttconf.TINY_NUMBER) & (t_grid_0 < tmax+ttconf.TINY_NUMBER)]<EOL>res_0, t_0 = np.array([conv_in_point(t_val) for t_val in t_grid_0]).T<EOL>interp_error = np.abs(res_0[<NUM_LIT:3>:-<NUM_LIT:1>]+res_0[<NUM_LIT:1>:-<NUM_LIT:3>]-<NUM_LIT:2>*res_0[<NUM_LIT:2>:-<NUM_LIT:2>])<EOL>dy = (res_0[<NUM_LIT:2>:-<NUM_LIT:2>]-res_0.min())<EOL>dx = np.diff(t_grid_0)<EOL>refine_factor = np.minimum(np.minimum(np.array(np.floor(np.sqrt(interp_error/(rel_tol*(<NUM_LIT:1>+(dy/yc)**<NUM_LIT:4>)))), dtype=int),<EOL>np.array(<NUM_LIT:100>*(dx[<NUM_LIT:1>:-<NUM_LIT:2>]+dx[<NUM_LIT:2>:-<NUM_LIT:1>])/min_fwhm, dtype=int)), <NUM_LIT:10>)<EOL>insert_point_idx = np.zeros(interp_error.shape[<NUM_LIT:0>]+<NUM_LIT:1>, dtype=int)<EOL>insert_point_idx[<NUM_LIT:1>:] = refine_factor<EOL>insert_point_idx[:-<NUM_LIT:1>] += refine_factor<EOL>if np.sum(insert_point_idx):<EOL><INDENT>add_x = np.concatenate([np.linspace(t1,t2,n+<NUM_LIT:2>)[<NUM_LIT:1>:-<NUM_LIT:1>] for t1,t2,n in<EOL>zip(t_grid_0[<NUM_LIT:1>:-<NUM_LIT:2>], t_grid_0[<NUM_LIT:2>:-<NUM_LIT:1>], insert_point_idx) if n><NUM_LIT:0>])<EOL>add_y, add_t = np.array([conv_in_point(t_val) for t_val in add_x]).T<EOL>t_grid_0 = np.concatenate((t_grid_0, add_x))<EOL>res_0 = np.concatenate ((res_0, add_y))<EOL>t_0 = np.concatenate ((t_0, add_t))<EOL><DEDENT>res_y = cls(t_grid_0, res_0, is_log=True, kind='<STR_LIT>')<EOL>res_t = Distribution(t_grid_0, t_0, is_log=True,<EOL>min_width=node_interp.min_width, kind='<STR_LIT>')<EOL>return res_y, res_t<EOL>

calculate H(t) = \int_tau f(t-tau)g(tau) if inverse_time=True
          H(t) = \int_tau f(t+tau)g(tau) if inverse_time=False

This function determines the time points of the grid of the result to
ensure an accurate approximation.


if isinstance(Tc, Iterable):<EOL><INDENT>if len(Tc)==len(T):<EOL><INDENT>x = np.concatenate(([-ttconf.BIG_NUMBER], T, [ttconf.BIG_NUMBER]))<EOL>y = np.concatenate(([Tc[<NUM_LIT:0>]], Tc, [Tc[-<NUM_LIT:1>]]))<EOL>self.Tc = interp1d(x,y)<EOL><DEDENT>else:<EOL><INDENT>self.logger("<STR_LIT>",<NUM_LIT:2>,warn=True)<EOL>self.Tc = interp1d([-ttconf.BIG_NUMBER, ttconf.BIG_NUMBER], [<NUM_LIT>, <NUM_LIT>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.Tc = interp1d([-ttconf.BIG_NUMBER, ttconf.BIG_NUMBER],<EOL>[Tc+ttconf.TINY_NUMBER, Tc+ttconf.TINY_NUMBER])<EOL><DEDENT>self.calc_integral_merger_rate()<EOL>

initialize the merger model with a coalescent time

Args:
    - Tc:   a float or an iterable, if iterable another argument T of same shape is required
    - T:    an array like of same shape as Tc that specifies the time pivots corresponding to Tc
Returns:
    - None


<EOL>self.tree_events = np.array(sorted([(n.time_before_present, len(n.clades)-<NUM_LIT:1>)<EOL>for n in self.tree.find_clades() if not n.bad_branch],<EOL>key=lambda x:-x[<NUM_LIT:0>]))<EOL>from collections import defaultdict<EOL>dn_branch = defaultdict(int)<EOL>for (t, dn) in self.tree_events:<EOL><INDENT>dn_branch[t]+=dn<EOL><DEDENT>unique_mergers = np.array(sorted(dn_branch.items(), key = lambda x:-x[<NUM_LIT:0>]))<EOL>nbranches = [[ttconf.BIG_NUMBER, <NUM_LIT:1>], [unique_mergers[<NUM_LIT:0>,<NUM_LIT:0>]+ttconf.TINY_NUMBER, <NUM_LIT:1>]]<EOL>for ti, (t, dn) in enumerate(unique_mergers[:-<NUM_LIT:1>]):<EOL><INDENT>new_n = nbranches[-<NUM_LIT:1>][<NUM_LIT:1>]+dn<EOL>next_t = unique_mergers[ti+<NUM_LIT:1>,<NUM_LIT:0>]+ttconf.TINY_NUMBER<EOL>nbranches.append([t, new_n])<EOL>nbranches.append([next_t, new_n])<EOL><DEDENT>new_n += unique_mergers[-<NUM_LIT:1>,<NUM_LIT:1>]<EOL>nbranches.append([next_t, new_n])<EOL>nbranches.append([-ttconf.BIG_NUMBER, new_n])<EOL>nbranches=np.array(nbranches)<EOL>self.nbranches = interp1d(nbranches[:,<NUM_LIT:0>], nbranches[:,<NUM_LIT:1>], kind='<STR_LIT>')<EOL>

calculates an interpolation object that maps time to the number of
concurrent branches in the tree. The result is stored in self.nbranches


<EOL>tvals = np.unique(self.nbranches.x[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>rate = self.branch_merger_rate(tvals)<EOL>avg_rate = <NUM_LIT:0.5>*(rate[<NUM_LIT:1>:] + rate[:-<NUM_LIT:1>])<EOL>cost = np.concatenate(([<NUM_LIT:0>],np.cumsum(np.diff(tvals)*avg_rate)))<EOL>self.integral_merger_rate = interp1d(np.concatenate(([-ttconf.BIG_NUMBER], tvals,[ttconf.BIG_NUMBER])),<EOL>np.concatenate(([cost[<NUM_LIT:0>]], cost,[cost[-<NUM_LIT:1>]])), kind='<STR_LIT>')<EOL>

calculates the integral int_0^t (k(t')-1)/2Tc(t') dt' and stores it as
self.integral_merger_rate. This differences of this quantity evaluated at
different times points are the cost of a branch.


def cost(self, t_node, branch_length, multiplicity=<NUM_LIT>):

merger_time = t_node+branch_length<EOL>return self.integral_merger_rate(merger_time) - self.integral_merger_rate(t_node)- np.log(self.total_merger_rate(merger_time))*(multiplicity-<NUM_LIT:1.0>)/multiplicity<EOL>

returns the cost associated with a branch starting at t_node
t_node is time before present, the branch goes back in time

Args:
    - t_node:           time of the node
    - branch_length:    branch length, determines when this branch merges with sister
    - multiplicity:     2 if merger is binary, higher if this is a polytomy


for clade in self.tree.find_clades():<EOL><INDENT>if clade.up is not None:<EOL><INDENT>clade.branch_length_interpolator.merger_cost = self.cost<EOL><DEDENT><DEDENT>

attaches the the merger cost to each branch length interpolator in the tree.


from scipy.optimize import minimize_scalar<EOL>initial_Tc = self.Tc<EOL>def cost(Tc):<EOL><INDENT>self.set_Tc(Tc)<EOL>return -self.total_LH()<EOL><DEDENT>sol = minimize_scalar(cost, bounds=[ttconf.TINY_NUMBER,<NUM_LIT>])<EOL>if "<STR_LIT:success>" in sol and sol["<STR_LIT:success>"]:<EOL><INDENT>self.set_Tc(sol['<STR_LIT:x>'])<EOL><DEDENT>else:<EOL><INDENT>self.logger("<STR_LIT>" + str(sol), <NUM_LIT:0>, warn=True)<EOL>self.set_Tc(initial_Tc.y, T=initial_Tc.x)<EOL><DEDENT>

determines the coalescent time scale that optimizes the coalescent likelihood of the tree


def optimize_skyline(self, n_points=<NUM_LIT:20>, stiffness=<NUM_LIT>, method = '<STR_LIT>',<EOL>tol=<NUM_LIT>, regularization=<NUM_LIT>, **kwarks):

self.logger("<STR_LIT>"%self.total_LH(),<NUM_LIT:2>)<EOL>from scipy.optimize import minimize<EOL>initial_Tc = self.Tc<EOL>tvals = np.linspace(self.tree_events[<NUM_LIT:0>,<NUM_LIT:0>], self.tree_events[-<NUM_LIT:1>,<NUM_LIT:0>], n_points)<EOL>def cost(logTc):<EOL><INDENT>self.set_Tc(np.exp(np.maximum(-<NUM_LIT:200>,np.minimum(<NUM_LIT:100>,logTc))), tvals)<EOL>neglogLH = -self.total_LH() + stiffness*np.sum(np.diff(logTc)**<NUM_LIT:2>)+ np.sum((logTc><NUM_LIT:0>)*logTc*regularization)- np.sum((logTc<-<NUM_LIT:100>)*logTc*regularization)<EOL>return neglogLH<EOL><DEDENT>sol = minimize(cost, np.ones_like(tvals)*np.log(self.Tc.y.mean()), method=method, tol=tol)<EOL>if "<STR_LIT:success>" in sol and sol["<STR_LIT:success>"]:<EOL><INDENT>dlogTc = <NUM_LIT:0.1><EOL>opt_logTc = sol['<STR_LIT:x>']<EOL>dcost = []<EOL>for ii in range(len(opt_logTc)):<EOL><INDENT>tmp = opt_logTc.copy()<EOL>tmp[ii]+=dlogTc<EOL>cost_plus = cost(tmp)<EOL>tmp[ii]-=<NUM_LIT:2>*dlogTc<EOL>cost_minus = cost(tmp)<EOL>dcost.append([cost_minus, cost_plus])<EOL><DEDENT>dcost = np.array(dcost)<EOL>optimal_cost = cost(opt_logTc)<EOL>self.confidence = -dlogTc/(<NUM_LIT:2>*optimal_cost - dcost[:,<NUM_LIT:0>] - dcost[:,<NUM_LIT:1>])<EOL>self.logger("<STR_LIT>"%self.total_LH(),<NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>self.set_Tc(initial_Tc.y, T=initial_Tc.x)<EOL>self.logger("<STR_LIT>"+str(sol),<NUM_LIT:0>, warn=True)<EOL><DEDENT>

optimize the trajectory of the merger rate 1./T_c to maximize the
coalescent likelihood.
parameters:
    n_points    --  number of pivots of the Tc interpolation object
    stiffness   --  penalty for rapid changes in log(Tc)
    methods     --  method used to optimize
    tol         --  optimization tolerance
    regularization --  cost of moving logTc outsize of the range [-100,0]
                       merger rate is measured in branch length units, no
                       plausible rates should never be outside this window


def skyline_empirical(self, gen=<NUM_LIT:1.0>, n_points = <NUM_LIT:20>):

mergers = self.tree_events[:,<NUM_LIT:1>]><NUM_LIT:0><EOL>merger_tvals = self.tree_events[mergers,<NUM_LIT:0>]<EOL>nlineages = self.nbranches(merger_tvals-ttconf.TINY_NUMBER)<EOL>expected_merger_density = nlineages*(nlineages-<NUM_LIT:1>)*<NUM_LIT:0.5><EOL>nmergers = len(mergers)<EOL>et = merger_tvals<EOL>ev = <NUM_LIT:1.0>/expected_merger_density<EOL>if <NUM_LIT:2>*n_points>len(expected_merger_density):<EOL><INDENT>n_points = len(ev)//<NUM_LIT:4><EOL><DEDENT>avg = np.sum(ev)/np.abs(et[<NUM_LIT:0>]-et[-<NUM_LIT:1>])<EOL>dt = et[<NUM_LIT:0>]-et[-<NUM_LIT:1>]<EOL>mid_points = np.concatenate(([et[<NUM_LIT:0>]-<NUM_LIT:0.5>*(et[<NUM_LIT:1>]-et[<NUM_LIT:0>])],<EOL><NUM_LIT:0.5>*(et[<NUM_LIT:1>:] + et[:-<NUM_LIT:1>]),<EOL>[et[-<NUM_LIT:1>]+<NUM_LIT:0.5>*(et[-<NUM_LIT:1>]-et[-<NUM_LIT:2>])]))<EOL>self.Tc_inv = interp1d(mid_points[n_points:-n_points],<EOL>[np.sum(ev[(et>=l)&(et<u)])/(u-l+dt/nmergers)<EOL>for u,l in zip(mid_points[:-<NUM_LIT:2>*n_points],mid_points[<NUM_LIT:2>*n_points:])])<EOL>return interp1d(self.date2dist.to_numdate(self.Tc_inv.x), gen/self.date2dist.clock_rate/self.Tc_inv.y)<EOL>

returns the skyline, i.e., an estimate of the inverse rate of coalesence.
Here, the skyline is estimated from a sliding window average of the observed
mergers, i.e., without reference to the coalescence likelihood.
parameters:
    gen -- number of generations per year.


def skyline_inferred(self, gen=<NUM_LIT:1.0>, confidence=False):

if len(self.Tc.x)<=<NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>")<EOL>return gen/self.date2dist.clock_rate*self.Tc.y[-<NUM_LIT:1>]<EOL><DEDENT>skyline = interp1d(self.date2dist.to_numdate(self.Tc.x[<NUM_LIT:1>:-<NUM_LIT:1>]), gen/self.date2dist.clock_rate*self.Tc.y[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>if confidence and hasattr(self, '<STR_LIT>'):<EOL><INDENT>conf = [skyline.y*np.exp(-confidence*self.confidence), skyline.y*np.exp(confidence*self.confidence)]<EOL>return skyline, conf<EOL><DEDENT>else:<EOL><INDENT>return skyline<EOL><DEDENT>

return the skyline, i.e., an estimate of the inverse rate of coalesence.
This function merely returns the merger rate self.Tc that was set or
estimated by other means. If it was determined using self.optimize_skyline,
the returned skyline will maximize the coalescent likelihood.
parameters:
    gen -- number of generations per year. Unit of time is branch length,
           hence this needs to be the inverse substitution rate per generation
    confidence -- False, or number of standard deviations of confidence intervals


if params.tree is None:<EOL><INDENT>params.tree = os.path.basename(params.aln)+'<STR_LIT>'<EOL>print("<STR_LIT>")<EOL>utils.tree_inference(params.aln, params.tree, tmp_dir = tmp_dir)<EOL><DEDENT>if os.path.isdir(tmp_dir):<EOL><INDENT>shutil.rmtree(tmp_dir)<EOL><DEDENT>try:<EOL><INDENT>tt = TreeAnc(params.tree)<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>

Function that attempts to load a tree and build it from the alignment
if no tree is provided.


model = params.gtr<EOL>gtr_params = params.gtr_params<EOL>if model == '<STR_LIT>':<EOL><INDENT>gtr = GTR.standard('<STR_LIT>', alphabet='<STR_LIT>' if params.aa else '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>kwargs = {}<EOL>if gtr_params is not None:<EOL><INDENT>for param in gtr_params:<EOL><INDENT>keyval = param.split('<STR_LIT:=>')<EOL>if len(keyval)!=<NUM_LIT:2>: continue<EOL>if keyval[<NUM_LIT:0>] in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>keyval[<NUM_LIT:0>] = '<STR_LIT>'<EOL>keyval[<NUM_LIT:1>] = list(map(float, keyval[<NUM_LIT:1>].split('<STR_LIT:U+002C>')))<EOL><DEDENT>elif keyval[<NUM_LIT:0>] not in ['<STR_LIT>']:<EOL><INDENT>keyval[<NUM_LIT:1>] = float(keyval[<NUM_LIT:1>])<EOL><DEDENT>kwargs[keyval[<NUM_LIT:0>]] = keyval[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print ("<STR_LIT>")<EOL><DEDENT>gtr = GTR.standard(model, **kwargs)<EOL>infer_gtr = False<EOL><DEDENT>except:<EOL><INDENT>print ("<STR_LIT>")<EOL>gtr = GTR.standard('<STR_LIT>', alphabet='<STR_LIT>' if params.aa else '<STR_LIT>')<EOL>infer_gtr = False<EOL><DEDENT><DEDENT>return gtr<EOL>

parse the arguments referring to the GTR model and return a GTR structure


ref = None<EOL>aln = params.aln<EOL>fixed_pi = None<EOL>if hasattr(params, '<STR_LIT>') and params.aln is not None:<EOL><INDENT>if any([params.aln.lower().endswith(x) for x in ['<STR_LIT>', '<STR_LIT>']]):<EOL><INDENT>if not params.vcf_reference:<EOL><INDENT>print("<STR_LIT>")<EOL>return -<NUM_LIT:1><EOL><DEDENT>compress_seq = read_vcf(params.aln, params.vcf_reference)<EOL>sequences = compress_seq['<STR_LIT>']<EOL>ref = compress_seq['<STR_LIT>']<EOL>aln = sequences<EOL>if not hasattr(params, '<STR_LIT>') or params.gtr=="<STR_LIT>": <EOL><INDENT>alpha = alphabets['<STR_LIT>'] if params.aa else alphabets['<STR_LIT>']<EOL>fixed_pi = [ref.count(base)/len(ref) for base in alpha]<EOL>if fixed_pi[-<NUM_LIT:1>] == <NUM_LIT:0>:<EOL><INDENT>fixed_pi[-<NUM_LIT:1>] = <NUM_LIT><EOL>fixed_pi = [v-<NUM_LIT> for v in fixed_pi]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return aln, ref, fixed_pi<EOL>

Checks if input is VCF and reads in appropriately if it is


if assure_tree(params, tmp_dir='<STR_LIT>'):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>gtr = create_gtr(params)<EOL>aln, ref, fixed_pi = read_if_vcf(params)<EOL>is_vcf = True if ref is not None else False<EOL>treeanc = TreeAnc(params.tree, aln=aln, ref=ref, gtr=gtr, verbose=<NUM_LIT:1>,<EOL>fill_overhangs=True)<EOL>if treeanc.aln is None: <EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if is_vcf:<EOL><INDENT>L = len(ref) + params.const<EOL><DEDENT>else:<EOL><INDENT>L = treeanc.aln.get_alignment_length() + params.const<EOL><DEDENT>N_seq = len(treeanc.aln)<EOL>N_tree = treeanc.tree.count_terminals()<EOL>if params.rescale!=<NUM_LIT:1.0>:<EOL><INDENT>for n in treeanc.tree.find_clades():<EOL><INDENT>n.branch_length *= params.rescale<EOL>n.mutation_length = n.branch_length<EOL><DEDENT><DEDENT>print("<STR_LIT>"%(params.aln,N_seq,L))<EOL>print("<STR_LIT>"%(params.tree,N_tree))<EOL>print("<STR_LIT>")<EOL>ndiff = treeanc.infer_ancestral_sequences('<STR_LIT>', infer_gtr=params.gtr=='<STR_LIT>',<EOL>marginal=False, fixed_pi=fixed_pi)<EOL>print("<STR_LIT>")<EOL>if ndiff==ttconf.ERROR: <EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>if is_vcf:<EOL><INDENT>treeanc.recover_var_ambigs()<EOL><DEDENT>from collections import defaultdict<EOL>from scipy.stats import poisson<EOL>offset = <NUM_LIT:0> if params.zero_based else <NUM_LIT:1><EOL>if params.drms:<EOL><INDENT>DRM_info = read_in_DRMs(params.drms, offset)<EOL>drms = DRM_info['<STR_LIT>']<EOL><DEDENT>mutations = defaultdict(list)<EOL>positions = defaultdict(list)<EOL>terminal_mutations = defaultdict(list)<EOL>for n in treeanc.tree.find_clades():<EOL><INDENT>if n.up is None:<EOL><INDENT>continue<EOL><DEDENT>if len(n.mutations):<EOL><INDENT>for (a,pos, d) in n.mutations:<EOL><INDENT>if '<STR_LIT:->' not in [a,d] and '<STR_LIT:N>' not in [a,d]:<EOL><INDENT>mutations[(a,pos+offset,d)].append(n)<EOL>positions[pos+offset].append(n)<EOL><DEDENT><DEDENT>if n.is_terminal():<EOL><INDENT>for (a,pos, d) in n.mutations:<EOL><INDENT>if '<STR_LIT:->' not in [a,d] and '<STR_LIT:N>' not in [a,d]:<EOL><INDENT>terminal_mutations[(a,pos+offset,d)].append(n)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>mutation_by_strain = defaultdict(list)<EOL>for n in treeanc.tree.get_terminals():<EOL><INDENT>for a,pos,d in n.mutations:<EOL><INDENT>if pos+offset in positions and len(positions[pos+offset])><NUM_LIT:1>:<EOL><INDENT>if '<STR_LIT:->' not in [a,d] and '<STR_LIT:N>' not in [a,d]:<EOL><INDENT>mutation_by_strain[n.name].append([(a,pos+offset,d), len(positions[pos])])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>total_branch_length = treeanc.tree.total_branch_length()<EOL>corrected_branch_length = np.sum([np.exp(-x.branch_length)*np.sinh(x.branch_length)<EOL>for x in treeanc.tree.find_clades()])<EOL>corrected_terminal_branch_length = np.sum([np.exp(-x.branch_length)*np.sinh(x.branch_length)<EOL>for x in treeanc.tree.get_terminals()])<EOL>expected_mutations = L*corrected_branch_length<EOL>expected_terminal_mutations = L*corrected_terminal_branch_length<EOL>multiplicities = np.bincount([len(x) for x in mutations.values()])<EOL>total_mutations = np.sum([len(x) for x in mutations.values()])<EOL>multiplicities_terminal = np.bincount([len(x) for x in terminal_mutations.values()])<EOL>terminal_mutation_count = np.sum([len(x) for x in terminal_mutations.values()])<EOL>multiplicities_positions = np.bincount([len(x) for x in positions.values()])<EOL>multiplicities_positions[<NUM_LIT:0>] = L - np.sum(multiplicities_positions)<EOL>print("<STR_LIT>"<EOL>%(total_branch_length,total_mutations))<EOL>print("<STR_LIT>"%total_mutations<EOL>+"<STR_LIT>".join(['<STR_LIT>'%(n,mi)<EOL>for mi,n in enumerate(multiplicities) if n]))<EOL>if params.detailed:<EOL><INDENT>print("<STR_LIT>"<EOL>%(corrected_terminal_branch_length,terminal_mutation_count))<EOL>print("<STR_LIT>"%terminal_mutation_count<EOL>+"<STR_LIT>".join(['<STR_LIT>'%(n,mi)<EOL>for mi,n in enumerate(multiplicities_terminal) if n]))<EOL><DEDENT>print("<STR_LIT>"%L<EOL>+"<STR_LIT>".join(['<STR_LIT>'%(n,mi,L*poisson.pmf(mi,<NUM_LIT:1.0>*total_mutations/L))<EOL>for mi,n in enumerate(multiplicities_positions) if n]))<EOL>p = poisson.pmf(np.arange(<NUM_LIT:10>*multiplicities_positions.max()),<NUM_LIT:1.0>*total_mutations/L)<EOL>print("<STR_LIT>"%(<EOL>- L*np.sum(p*np.log(p+<NUM_LIT>))<EOL>+ np.sum(multiplicities_positions*np.log(p[:len(multiplicities_positions)]+<NUM_LIT>))))<EOL>if params.drms:<EOL><INDENT>print("<STR_LIT>")<EOL>mutations_sorted = sorted(mutations.items(), key=lambda x:len(x[<NUM_LIT:1>])-<NUM_LIT:0.1>*x[<NUM_LIT:0>][<NUM_LIT:1>]/L, reverse=True)<EOL>for mut, val in mutations_sorted[:params.n]:<EOL><INDENT>if len(val)><NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>"%(mut[<NUM_LIT:0>], mut[<NUM_LIT:1>], mut[<NUM_LIT:2>], len(val),<EOL>"<STR_LIT:U+0020>".join([drms[mut[<NUM_LIT:1>]]['<STR_LIT>'], drms[mut[<NUM_LIT:1>]]['<STR_LIT>'], drms[mut[<NUM_LIT:1>]]['<STR_LIT>'][mut[<NUM_LIT:2>]]]) if mut[<NUM_LIT:1>] in drms else "<STR_LIT>"))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>mutations_sorted = sorted(mutations.items(), key=lambda x:len(x[<NUM_LIT:1>])-<NUM_LIT:0.1>*x[<NUM_LIT:0>][<NUM_LIT:1>]/L, reverse=True)<EOL>for mut, val in mutations_sorted[:params.n]:<EOL><INDENT>if len(val)><NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>"%(mut[<NUM_LIT:0>], mut[<NUM_LIT:1>], mut[<NUM_LIT:2>], len(val)))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>if params.detailed:<EOL><INDENT>if params.drms:<EOL><INDENT>print("<STR_LIT>")<EOL>terminal_mutations_sorted = sorted(terminal_mutations.items(), key=lambda x:len(x[<NUM_LIT:1>])-<NUM_LIT:0.1>*x[<NUM_LIT:0>][<NUM_LIT:1>]/L, reverse=True)<EOL>for mut, val in terminal_mutations_sorted[:params.n]:<EOL><INDENT>if len(val)><NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>"%(mut[<NUM_LIT:0>], mut[<NUM_LIT:1>], mut[<NUM_LIT:2>], len(val),<EOL>"<STR_LIT:U+0020>".join([drms[mut[<NUM_LIT:1>]]['<STR_LIT>'], drms[mut[<NUM_LIT:1>]]['<STR_LIT>'], drms[mut[<NUM_LIT:1>]]['<STR_LIT>'][mut[<NUM_LIT:2>]]]) if mut[<NUM_LIT:1>] in drms else "<STR_LIT>"))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>terminal_mutations_sorted = sorted(terminal_mutations.items(), key=lambda x:len(x[<NUM_LIT:1>])-<NUM_LIT:0.1>*x[<NUM_LIT:0>][<NUM_LIT:1>]/L, reverse=True)<EOL>for mut, val in terminal_mutations_sorted[:params.n]:<EOL><INDENT>if len(val)><NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>"%(mut[<NUM_LIT:0>], mut[<NUM_LIT:1>], mut[<NUM_LIT:2>], len(val)))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if params.detailed:<EOL><INDENT>if params.drms:<EOL><INDENT>print("<STR_LIT>")<EOL>mutation_by_strain_sorted = sorted(mutation_by_strain.items(), key=lambda x:len(x[<NUM_LIT:1>]), reverse=True)<EOL>for name, val in mutation_by_strain_sorted[:params.n]:<EOL><INDENT>if len(val):<EOL><INDENT>print("<STR_LIT>"%(name, len(val),<EOL>len([mut for mut,l in val if mut[<NUM_LIT:1>] in drms])))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>mutation_by_strain_sorted = sorted(mutation_by_strain.items(), key=lambda x:len(x[<NUM_LIT:1>]), reverse=True)<EOL>for name, val in mutation_by_strain_sorted[:params.n]:<EOL><INDENT>if len(val):<EOL><INDENT>print("<STR_LIT>"%(name, len(val)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return <NUM_LIT:0><EOL>

the function implementing treetime homoplasies


if params.relax is None:<EOL><INDENT>relaxed_clock_params = None<EOL><DEDENT>elif params.relax==[]:<EOL><INDENT>relaxed_clock_params=True<EOL><DEDENT>elif len(params.relax)==<NUM_LIT:2>:<EOL><INDENT>relaxed_clock_params={'<STR_LIT>':params.relax[<NUM_LIT:0>], '<STR_LIT>':params.relax[<NUM_LIT:1>]}<EOL><DEDENT>dates = utils.parse_dates(params.dates)<EOL>if len(dates)==<NUM_LIT:0>:<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>if assure_tree(params, tmp_dir='<STR_LIT>'):<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>outdir = get_outdir(params, '<STR_LIT>')<EOL>gtr = create_gtr(params)<EOL>infer_gtr = params.gtr=='<STR_LIT>'<EOL>aln, ref, fixed_pi = read_if_vcf(params)<EOL>is_vcf = True if ref is not None else False<EOL>branch_length_mode = params.branch_length_mode<EOL>if is_vcf or (params.aln and params.sequence_length):<EOL><INDENT>if branch_length_mode == '<STR_LIT>':<EOL><INDENT>branch_length_mode = '<STR_LIT>'<EOL><DEDENT><DEDENT>if params.aln is None and params.sequence_length is None:<EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT>myTree = TreeTime(dates=dates, tree=params.tree, ref=ref,<EOL>aln=aln, gtr=gtr, seq_len=params.sequence_length,<EOL>verbose=params.verbose)<EOL>myTree.tip_slack=params.tip_slack<EOL>if not myTree.one_mutation:<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>try:<EOL><INDENT>coalescent = float(params.coalescent)<EOL>if coalescent<<NUM_LIT:10>*myTree.one_mutation:<EOL><INDENT>coalescent = None<EOL><DEDENT><DEDENT>except:<EOL><INDENT>if params.coalescent in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>coalescent = params.coalescent<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT><DEDENT>calc_confidence = params.confidence<EOL>if params.clock_std_dev:<EOL><INDENT>vary_rate = params.clock_std_dev if calc_confidence else False<EOL><DEDENT>elif params.confidence and params.covariation:<EOL><INDENT>vary_rate = True<EOL><DEDENT>elif params.confidence:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>vary_rate = False<EOL>calc_confidence = False<EOL><DEDENT>else:<EOL><INDENT>vary_rate = False<EOL><DEDENT>root = None if params.keep_root else params.reroot<EOL>success = myTree.run(root=root, relaxed_clock=relaxed_clock_params,<EOL>resolve_polytomies=(not params.keep_polytomies),<EOL>Tc=coalescent, max_iter=params.max_iter,<EOL>fixed_clock_rate=params.clock_rate,<EOL>n_iqd=params.clock_filter,<EOL>time_marginal="<STR_LIT>" if calc_confidence else False,<EOL>vary_rate = vary_rate,<EOL>branch_length_mode = branch_length_mode,<EOL>fixed_pi=fixed_pi,<EOL>use_covariation = params.covariation)<EOL>if success==ttconf.ERROR: <EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>if infer_gtr:<EOL><INDENT>print('<STR_LIT>')<EOL>print(myTree.gtr)<EOL><DEDENT>print(myTree.date2dist)<EOL>basename = get_basename(params, outdir)<EOL>if coalescent in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>print("<STR_LIT>")<EOL>if coalescent=='<STR_LIT>':<EOL><INDENT>print_save_plot_skyline(myTree, plot=basename+'<STR_LIT>', save=basename+'<STR_LIT>', screen=True)<EOL><DEDENT>else:<EOL><INDENT>Tc = myTree.merger_model.Tc.y[<NUM_LIT:0>]<EOL>print("<STR_LIT>"%Tc)<EOL>print("<STR_LIT>"%(Tc/myTree.date2dist.clock_rate))<EOL>print("<STR_LIT>"%(Tc/myTree.date2dist.clock_rate*<NUM_LIT:50>))<EOL><DEDENT><DEDENT>import matplotlib.pyplot as plt<EOL>from .treetime import plot_vs_years<EOL>leaf_count = myTree.tree.count_terminals()<EOL>label_func = lambda x: (x.name if x.is_terminal() and ((leaf_count<<NUM_LIT:30><EOL>and (not params.no_tip_labels))<EOL>or params.tip_labels) else '<STR_LIT>')<EOL>plot_vs_years(myTree, show_confidence=False, label_func=label_func,<EOL>confidence=<NUM_LIT> if params.confidence else None)<EOL>tree_fname = (outdir + params.plot_tree)<EOL>plt.savefig(tree_fname)<EOL>print("<STR_LIT>"%tree_fname)<EOL>plot_rtt(myTree, outdir + params.plot_rtt)<EOL>if params.relax:<EOL><INDENT>fname = outdir+'<STR_LIT>'<EOL>print("<STR_LIT>"%fname)<EOL>with open(fname, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write("<STR_LIT>")<EOL>for n in myTree.tree.find_clades(order="<STR_LIT>"):<EOL><INDENT>if n==myTree.tree.root:<EOL><INDENT>continue<EOL><DEDENT>g = n.branch_length_interpolator.gamma<EOL>fh.write("<STR_LIT>"%(n.name, n.clock_length, n.mutation_length, myTree.date2dist.clock_rate*g, g))<EOL><DEDENT><DEDENT><DEDENT>export_sequences_and_tree(myTree, basename, is_vcf, params.zero_based,<EOL>timetree=True, confidence=calc_confidence)<EOL>return <NUM_LIT:0><EOL>


<EOL>if assure_tree(params, tmp_dir='<STR_LIT>'):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>outdir = get_outdir(params, '<STR_LIT>')<EOL>basename = get_basename(params, outdir)<EOL>gtr = create_gtr(params)<EOL>aln, ref, fixed_pi = read_if_vcf(params)<EOL>is_vcf = True if ref is not None else False<EOL>treeanc = TreeAnc(params.tree, aln=aln, ref=ref, gtr=gtr, verbose=<NUM_LIT:1>,<EOL>fill_overhangs=not params.keep_overhangs)<EOL>ndiff =treeanc.infer_ancestral_sequences('<STR_LIT>', infer_gtr=params.gtr=='<STR_LIT>',<EOL>marginal=params.marginal, fixed_pi=fixed_pi)<EOL>if ndiff==ttconf.ERROR: <EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if params.gtr=="<STR_LIT>":<EOL><INDENT>print('<STR_LIT>')<EOL>print(treeanc.gtr)<EOL><DEDENT>export_sequences_and_tree(treeanc, basename, is_vcf, params.zero_based,<EOL>report_ambiguous=params.report_ambiguous)<EOL>return <NUM_LIT:0><EOL>


<EOL>if os.path.isfile(params.states):<EOL><INDENT>states = pd.read_csv(params.states, sep='<STR_LIT:\t>' if params.states[-<NUM_LIT:3>:]=='<STR_LIT>' else '<STR_LIT:U+002C>',<EOL>skipinitialspace=True)<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>outdir = get_outdir(params, '<STR_LIT>')<EOL>taxon_name = '<STR_LIT:name>' if '<STR_LIT:name>' in states.columns else states.columns[<NUM_LIT:0>]<EOL>if params.attribute:<EOL><INDENT>if params.attribute in states.columns:<EOL><INDENT>attr = params.attribute<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>"+params.states, file=sys.stderr)<EOL>print("<STR_LIT>"+"<STR_LIT:U+002CU+0020>".join(states.columns), file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>attr = states.columns[<NUM_LIT:1>]<EOL>print("<STR_LIT>"+attr, file=sys.stderr)<EOL><DEDENT>leaf_to_attr = {x[taxon_name]:x[attr] for xi, x in states.iterrows()<EOL>if x[attr]!=params.missing_data}<EOL>unique_states = sorted(set(leaf_to_attr.values()))<EOL>nc = len(unique_states)<EOL>if nc><NUM_LIT>:<EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT>elif nc<<NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT>alphabet = [chr(<NUM_LIT>+i) for i,state in enumerate(unique_states)]<EOL>missing_char = chr(<NUM_LIT>+nc)<EOL>letter_to_state = {a:unique_states[i] for i,a in enumerate(alphabet)}<EOL>letter_to_state[missing_char]=params.missing_data<EOL>reverse_alphabet = {v:k for k,v in letter_to_state.items()}<EOL>if params.weights:<EOL><INDENT>params.infer_gtr = True<EOL>tmp_weights = pd.read_csv(params.weights, sep='<STR_LIT:\t>' if params.states[-<NUM_LIT:3>:]=='<STR_LIT>' else '<STR_LIT:U+002C>',<EOL>skipinitialspace=True)<EOL>weights = {row[<NUM_LIT:0>]:row[<NUM_LIT:1>] for ri,row in tmp_weights.iterrows()}<EOL>mean_weight = np.mean(list(weights.values()))<EOL>weights = np.array([weights[c] if c in weights else mean_weight for c in unique_states], dtype=float)<EOL>weights/=weights.sum()<EOL><DEDENT>else:<EOL><INDENT>weights = np.ones(nc, dtype=float)/nc<EOL><DEDENT>W = np.ones((nc,nc), dtype=float)<EOL>mugration_GTR = GTR.custom(pi = weights, W=W, alphabet = np.array(alphabet))<EOL>mugration_GTR.profile_map[missing_char] = np.ones(nc)<EOL>mugration_GTR.ambiguous=missing_char<EOL>treeanc = TreeAnc(params.tree, gtr=mugration_GTR, verbose=params.verbose,<EOL>convert_upper=False, one_mutation=<NUM_LIT>)<EOL>pseudo_seqs = [SeqRecord(id=n.name,name=n.name,<EOL>seq=Seq(reverse_alphabet[leaf_to_attr[n.name]]<EOL>if n.name in leaf_to_attr else missing_char))<EOL>for n in treeanc.tree.get_terminals()]<EOL>treeanc.aln = MultipleSeqAlignment(pseudo_seqs)<EOL>ndiff = treeanc.infer_ancestral_sequences(method='<STR_LIT>', infer_gtr=True,<EOL>store_compressed=False, pc=params.pc, marginal=True, normalized_rate=False,<EOL>fixed_pi=weights if params.weights else None)<EOL>if ndiff==ttconf.ERROR: <EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>print("<STR_LIT>"%attr,params.tree)<EOL>basename = get_basename(params, outdir)<EOL>gtr_name = basename + '<STR_LIT>'<EOL>with open(gtr_name, '<STR_LIT:w>') as ofile:<EOL><INDENT>ofile.write('<STR_LIT>')<EOL>for state in unique_states:<EOL><INDENT>ofile.write('<STR_LIT>'%(reverse_alphabet[state], state))<EOL><DEDENT>ofile.write('<STR_LIT>'+str(treeanc.gtr)+'<STR_LIT:\n>')<EOL>print("<STR_LIT>", gtr_name)<EOL><DEDENT>terminal_count = <NUM_LIT:0><EOL>for n in treeanc.tree.find_clades():<EOL><INDENT>if n.up is None:<EOL><INDENT>continue<EOL><DEDENT>n.confidence=None<EOL>if n.is_terminal() and len(n.name)><NUM_LIT> and bioversion<"<STR_LIT>":<EOL><INDENT>n.name = n.name[:<NUM_LIT>]+'<STR_LIT>'%terminal_count<EOL>terminal_count+=<NUM_LIT:1><EOL><DEDENT>n.comment= '<STR_LIT>'%attr + letter_to_state[n.sequence[<NUM_LIT:0>]] +'<STR_LIT:">'<EOL><DEDENT>if params.confidence:<EOL><INDENT>conf_name = basename+'<STR_LIT>'<EOL>with open(conf_name, '<STR_LIT:w>') as ofile:<EOL><INDENT>ofile.write('<STR_LIT>'+'<STR_LIT:U+002CU+0020>'.join(unique_states)+'<STR_LIT:\n>')<EOL>for n in treeanc.tree.find_clades():<EOL><INDENT>ofile.write(n.name + '<STR_LIT:U+002CU+0020>'+'<STR_LIT:U+002CU+0020>'.join([str(x) for x in n.marginal_profile[<NUM_LIT:0>]])+'<STR_LIT:\n>')<EOL><DEDENT><DEDENT>print("<STR_LIT>", conf_name)<EOL><DEDENT>outtree_name = basename+'<STR_LIT>'<EOL>Phylo.write(treeanc.tree, outtree_name, '<STR_LIT>')<EOL>print("<STR_LIT>",outtree_name)<EOL>return <NUM_LIT:0><EOL>


if assure_tree(params, tmp_dir='<STR_LIT>'):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>dates = utils.parse_dates(params.dates)<EOL>if len(dates)==<NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>outdir = get_outdir(params, '<STR_LIT>')<EOL>aln, ref, fixed_pi = read_if_vcf(params)<EOL>is_vcf = True if ref is not None else False<EOL>if params.aln is None and params.sequence_length is None:<EOL><INDENT>print("<STR_LIT>", file=sys.stderr)<EOL>return <NUM_LIT:1><EOL><DEDENT>basename = get_basename(params, outdir)<EOL>myTree = TreeTime(dates=dates, tree=params.tree, aln=aln, gtr='<STR_LIT>',<EOL>verbose=params.verbose, seq_len=params.sequence_length,<EOL>ref=ref)<EOL>myTree.tip_slack=params.tip_slack<EOL>if myTree.tree is None:<EOL><INDENT>print("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>if params.clock_filter:<EOL><INDENT>n_bad = [n.name for n in myTree.tree.get_terminals() if n.bad_branch]<EOL>myTree.clock_filter(n_iqd=params.clock_filter, reroot=params.reroot or '<STR_LIT>')<EOL>n_bad_after = [n.name for n in myTree.tree.get_terminals() if n.bad_branch]<EOL>if len(n_bad_after)>len(n_bad):<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>+"<STR_LIT>".join(set(n_bad_after).difference(n_bad)))<EOL><DEDENT><DEDENT>if not params.keep_root:<EOL><INDENT>if params.covariation: <EOL><INDENT>myTree.run(root="<STR_LIT>", max_iter=<NUM_LIT:0>,<EOL>use_covariation=params.covariation)<EOL><DEDENT>res = myTree.reroot(params.reroot,<EOL>force_positive=not params.allow_negative_rate)<EOL>myTree.get_clock_model(covariation=params.covariation)<EOL>if res==ttconf.ERROR:<EOL><INDENT>print("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>myTree.get_clock_model(covariation=params.covariation)<EOL><DEDENT>d2d = utils.DateConversion.from_regression(myTree.clock_model)<EOL>print('<STR_LIT:\n>',d2d)<EOL>print('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>print('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>if params.covariation:<EOL><INDENT>reg = myTree.clock_model<EOL>dp = np.array([reg['<STR_LIT>']/reg['<STR_LIT>']**<NUM_LIT:2>,-<NUM_LIT:1.>/reg['<STR_LIT>']])<EOL>droot = np.sqrt(reg['<STR_LIT>'][:<NUM_LIT:2>,:<NUM_LIT:2>].dot(dp).dot(dp))<EOL>print('<STR_LIT>'%(-d2d.intercept/d2d.clock_rate, droot))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>'%(-d2d.intercept/d2d.clock_rate))<EOL><DEDENT>if not params.keep_root:<EOL><INDENT>outtree_name = basename+'<STR_LIT>'<EOL>Phylo.write(myTree.tree, outtree_name, '<STR_LIT>')<EOL>print("<STR_LIT>"%outtree_name)<EOL><DEDENT>table_fname = basename+'<STR_LIT>'<EOL>with open(table_fname, '<STR_LIT:w>') as ofile:<EOL><INDENT>ofile.write("<STR_LIT>")<EOL>ofile.write("<STR_LIT>")<EOL>for n in myTree.tree.get_terminals():<EOL><INDENT>if hasattr(n, "<STR_LIT>") and (n.raw_date_constraint is not None):<EOL><INDENT>if np.isscalar(n.raw_date_constraint):<EOL><INDENT>tmp_str = str(n.raw_date_constraint)<EOL><DEDENT>elif len(n.raw_date_constraint):<EOL><INDENT>tmp_str = str(n.raw_date_constraint[<NUM_LIT:0>])+'<STR_LIT:->'+str(n.raw_date_constraint[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>tmp_str = '<STR_LIT>'<EOL><DEDENT>ofile.write("<STR_LIT>"%(n.name, tmp_str, n.dist2root))<EOL><DEDENT>else:<EOL><INDENT>ofile.write("<STR_LIT>"%(n.name, d2d.numdate_from_dist2root(n.dist2root), n.dist2root))<EOL><DEDENT><DEDENT>for n in myTree.tree.get_nonterminals(order='<STR_LIT>'):<EOL><INDENT>ofile.write("<STR_LIT>"%(n.name, d2d.numdate_from_dist2root(n.dist2root), n.dist2root))<EOL><DEDENT>print("<STR_LIT>"%table_fname)<EOL><DEDENT>plot_rtt(myTree, outdir+params.plot_rtt)<EOL>return <NUM_LIT:0><EOL>


tmp = np.einsum('<STR_LIT>', self.Pi, self.W)<EOL>diag_vals = np.sum(tmp, axis=<NUM_LIT:0>)<EOL>for x in range(tmp.shape[-<NUM_LIT:1>]):<EOL><INDENT>np.fill_diagonal(tmp[:,:,x], -diag_vals[:,x])<EOL><DEDENT>return tmp<EOL>

function that return the product of the transition matrix
           and the equilibrium frequencies to obtain the rate matrix
           of the GTR model


def assign_rates(self, mu=<NUM_LIT:1.0>, pi=None, W=None):

n = len(self.alphabet)<EOL>self.mu = np.copy(mu)<EOL>if pi is not None and pi.shape[<NUM_LIT:0>]==n:<EOL><INDENT>self.seq_len = pi.shape[-<NUM_LIT:1>]<EOL>Pi = np.copy(pi)<EOL><DEDENT>else:<EOL><INDENT>if pi is not None and len(pi)!=n:<EOL><INDENT>self.logger("<STR_LIT>", <NUM_LIT:4>, warn=True)<EOL>self.logger("<STR_LIT>", <NUM_LIT:4>, warn=True)<EOL><DEDENT>Pi = np.ones(shape=(n,self.seq_len))<EOL><DEDENT>self.Pi = Pi/np.sum(Pi, axis=<NUM_LIT:0>)<EOL>if W is None or W.shape!=(n,n):<EOL><INDENT>if (W is not None) and W.shape!=(n,n):<EOL><INDENT>self.logger("<STR_LIT>", <NUM_LIT:4>, warn=True)<EOL>self.logger("<STR_LIT>", <NUM_LIT:4>, warn=True)<EOL><DEDENT>W = np.ones((n,n))<EOL><DEDENT>else:<EOL><INDENT>W=<NUM_LIT:0.5>*(np.copy(W)+np.copy(W).T)<EOL><DEDENT>np.fill_diagonal(W,<NUM_LIT:0>)<EOL>avg_pi = self.Pi.mean(axis=-<NUM_LIT:1>)<EOL>average_rate = W.dot(avg_pi).dot(avg_pi)<EOL>self.W = W/average_rate<EOL>self.mu *=average_rate<EOL>self._eig()<EOL>

Overwrite the GTR model given the provided data

Parameters
----------

 mu : float
    Substitution rate

 W : nxn matrix
    Substitution matrix

 pi : n vector
    Equilibrium frequencies


@classmethod<EOL><INDENT>def random(cls, L=<NUM_LIT:1>, avg_mu=<NUM_LIT:1.0>, alphabet='<STR_LIT>', pi_dirichlet_alpha=<NUM_LIT:1>,<EOL>W_dirichlet_alpha=<NUM_LIT>, mu_gamma_alpha=<NUM_LIT>):<DEDENT>

from scipy.stats import gamma<EOL>alphabet=alphabets[alphabet]<EOL>gtr = cls(alphabet=alphabet, seq_len=L)<EOL>n = gtr.alphabet.shape[<NUM_LIT:0>]<EOL>if pi_dirichlet_alpha:<EOL><INDENT>pi = <NUM_LIT:1.0>*gamma.rvs(pi_dirichlet_alpha, size=(n,L))<EOL><DEDENT>else:<EOL><INDENT>pi = np.ones((n,L))<EOL><DEDENT>pi /= pi.sum(axis=<NUM_LIT:0>)<EOL>if W_dirichlet_alpha:<EOL><INDENT>tmp = <NUM_LIT:1.0>*gamma.rvs(W_dirichlet_alpha, size=(n,n))<EOL><DEDENT>else:<EOL><INDENT>tmp = np.ones((n,n))<EOL><DEDENT>tmp = np.tril(tmp,k=-<NUM_LIT:1>)<EOL>W = tmp + tmp.T<EOL>if mu_gamma_alpha:<EOL><INDENT>mu = gamma.rvs(mu_gamma_alpha, size=(L,))<EOL><DEDENT>else:<EOL><INDENT>mu = np.ones(L)<EOL><DEDENT>gtr.assign_rates(mu=mu, pi=pi, W=W)<EOL>gtr.mu *= avg_mu/np.mean(gtr.mu)<EOL>return gtr<EOL>

Creates a random GTR model

Parameters
----------

 mu : float
    Substitution rate

 alphabet : str
    Alphabet name (should be standard: 'nuc', 'nuc_gap', 'aa', 'aa_gap')


@classmethod<EOL><INDENT>def custom(cls, mu=<NUM_LIT:1.0>, pi=None, W=None, **kwargs):<DEDENT>

gtr = cls(**kwargs)<EOL>gtr.assign_rates(mu=mu, pi=pi, W=W)<EOL>return gtr<EOL>

Create a GTR model by specifying the matrix explicitly

Parameters
----------

 mu : float
    Substitution rate

 W : nxn matrix
    Substitution matrix

 pi : n vector
    Equilibrium frequencies

 **kwargs:
    Key word arguments to be passed

Keyword Args
------------

 alphabet : str
    Specify alphabet when applicable. If the alphabet specification is
    required, but no alphabet is specified, the nucleotide alphabet will be used as
    default.


@classmethod<EOL><INDENT>def infer(cls, sub_ija, T_ia, root_state, pc=<NUM_LIT>,<EOL>gap_limit=<NUM_LIT>, Nit=<NUM_LIT:30>, dp=<NUM_LIT>, **kwargs):<DEDENT>

from scipy import linalg as LA<EOL>gtr = cls(**kwargs)<EOL>gtr.logger("<STR_LIT>",<NUM_LIT:1>)<EOL>q = len(gtr.alphabet)<EOL>L = sub_ija.shape[-<NUM_LIT:1>]<EOL>n_iter = <NUM_LIT:0><EOL>n_ija = np.copy(sub_ija)<EOL>n_ija[range(q),range(q),:] = <NUM_LIT:0><EOL>n_ij = n_ija.sum(axis=-<NUM_LIT:1>)<EOL>m_ia = np.sum(n_ija,axis=<NUM_LIT:1>) + root_state + pc<EOL>n_a = n_ija.sum(axis=<NUM_LIT:1>).sum(axis=<NUM_LIT:0>) + pc<EOL>Lambda = np.sum(root_state,axis=<NUM_LIT:0>) + q*pc<EOL>p_ia_old=np.zeros((q,L))<EOL>p_ia = np.ones((q,L))/q<EOL>mu_a = np.ones(L)<EOL>W_ij = np.ones((q,q)) - np.eye(q)<EOL>while (LA.norm(p_ia_old-p_ia)>dp) and n_iter<Nit:<EOL><INDENT>n_iter += <NUM_LIT:1><EOL>p_ia_old = np.copy(p_ia)<EOL>S_ij = np.einsum('<STR_LIT>',mu_a, p_ia, T_ia)<EOL>W_ij = (n_ij + n_ij.T + pc)/(S_ij + S_ij.T + pc)<EOL>avg_pi = p_ia.mean(axis=-<NUM_LIT:1>)<EOL>average_rate = W_ij.dot(avg_pi).dot(avg_pi)<EOL>W_ij = W_ij/average_rate<EOL>mu_a *=average_rate<EOL>p_ia = m_ia/(mu_a*np.dot(W_ij,T_ia)+Lambda)<EOL>p_ia = p_ia/p_ia.sum(axis=<NUM_LIT:0>)<EOL>mu_a = n_a/(pc+np.einsum('<STR_LIT>', p_ia, W_ij, T_ia))<EOL><DEDENT>if n_iter >= Nit:<EOL><INDENT>gtr.logger('<STR_LIT>',<NUM_LIT:3>, warn=True)<EOL>if LA.norm(p_ia_old-p_ia) > dp:<EOL><INDENT>gtr.logger('<STR_LIT>',<NUM_LIT:3>,warn=True)<EOL><DEDENT><DEDENT>if gtr.gap_index is not None:<EOL><INDENT>for p in range(p_ia.shape[-<NUM_LIT:1>]):<EOL><INDENT>if p_ia[gtr.gap_index,p]<gap_limit:<EOL><INDENT>gtr.logger('<STR_LIT>'%p_ia[gtr.gap_index]+<EOL>'<STR_LIT>'+<EOL>'<STR_LIT>'%gap_limit,<NUM_LIT:2>,warn=True)<EOL><DEDENT>p_ia[gtr.gap_index,p] = gap_limit<EOL>p_ia[:,p] /= p_ia[:,p].sum()<EOL><DEDENT><DEDENT>gtr.assign_rates(mu=mu_a, W=W_ij, pi=p_ia)<EOL>return gtr<EOL>

Infer a GTR model by specifying the number of transitions and time spent in each
character. The basic equation that is being solved is

:math:`n_{ij} = pi_i W_{ij} T_j`

where :math:`n_{ij}` are the transitions, :math:`pi_i` are the equilibrium
state frequencies, :math:`W_{ij}` is the "substitution attempt matrix",
while :math:`T_i` is the time on the tree spent in character state
:math:`i`. To regularize the process, we add pseudocounts and also need
to account for the fact that the root of the tree is in a particular
state. the modified equation is

:math:`n_{ij} + pc = pi_i W_{ij} (T_j+pc+root\_state)`

Parameters
----------

 nija : nxn matrix
    The number of times a change in character state is observed
    between state j and i at position a

 Tia :n vector
    The time spent in each character state at position a

 root_state : n vector
    The number of characters in state i in the sequence
    of the root node.

 pc : float
    Pseudocounts, this determines the lower cutoff on the rate when
    no substitutions are observed

 **kwargs:
    Key word arguments to be passed

Keyword Args
------------

 alphabet : str
    Specify alphabet when applicable. If the alphabet specification
    is required, but no alphabet is specified, the nucleotide alphabet will be used as default.


def propagate_profile(self, profile, t, return_log=False):

Qt = self.expQt(t)<EOL>res = np.einsum('<STR_LIT>', profile, Qt)<EOL>return np.log(res) if return_log else res<EOL>

Compute the probability of the sequence state of the parent
at time (t+t0, backwards), given the sequence state of the
child (profile) at time t0.

Parameters
----------

 profile : numpy.array
    Sequence profile. Shape = (L, a),
    where L - sequence length, a - alphabet size.

 t : double
    Time to propagate

 return_log: bool
    If True, return log-probability

Returns
-------

 res : np.array
    Profile of the sequence after time t in the past.
    Shape = (L, a), where L - sequence length, a - alphabet size.


def evolve(self, profile, t, return_log=False):

Compute the probability of the sequence state of the child
at time t later, given the parent profile.

Parameters
----------

 profile : numpy.array
    Sequence profile. Shape = (L, a),
    where L - sequence length, a - alphabet size.

 t : double
    Time to propagate

 return_log: bool
    If True, return log-probability

Returns
-------

 res : np.array
    Profile of the sequence after time t in the future.
    Shape = (L, a), where L - sequence length, a - alphabet size.


def prob_t(self, seq_p, seq_ch, t, pattern_multiplicity = None,<EOL>return_log=False, ignore_gaps=True):

if t<<NUM_LIT:0>:<EOL><INDENT>logP = -ttconf.BIG_NUMBER<EOL><DEDENT>else:<EOL><INDENT>tmp_eQT = self.expQt(t)<EOL>bad_indices=(tmp_eQT==<NUM_LIT:0>)<EOL>logQt = np.log(tmp_eQT + ttconf.TINY_NUMBER*(bad_indices))<EOL>logQt[np.isnan(logQt) | np.isinf(logQt) | bad_indices] = -ttconf.BIG_NUMBER<EOL>seq_indices_c = np.zeros(len(seq_ch), dtype=int)<EOL>seq_indices_p = np.zeros(len(seq_p), dtype=int)<EOL>for ai, a in self.alphabet:<EOL><INDENT>seq_indices_p[seq_p==a] = ai<EOL>seq_indices_c[seq_ch==a] = ai<EOL><DEDENT>if len(logQt.shape)==<NUM_LIT:2>:<EOL><INDENT>logP = np.sum(logQt[seq_indices_p, seq_indices_c]*pattern_multiplicity)<EOL><DEDENT>else:<EOL><INDENT>logP = np.sum(logQt[seq_indices_p, seq_indices_c, np.arange(len(seq_c))]*pattern_multiplicity)<EOL><DEDENT><DEDENT>return logP if return_log else np.exp(logP)<EOL>

Compute the probability to observe seq_ch (child sequence) after time t starting from seq_p
(parent sequence).

Parameters
----------

 seq_p : character array
    Parent sequence

 seq_c : character array
    Child sequence

 t : double
    Time (branch len) separating the profiles.

 pattern_multiplicity : numpy array
    If sequences are reduced by combining identical alignment patterns,
    these multplicities need to be accounted for when counting the number
    of mutations across a branch. If None, all pattern are assumed to
    occur exactly once.

 return_log : bool
    It True, return log-probability.

Returns
-------
 prob : np.array
    Resulting probability


def seq2array(seq, fill_overhangs=True, ambiguous_character='<STR_LIT:N>'):

try:<EOL><INDENT>sequence = '<STR_LIT>'.join(seq)<EOL><DEDENT>except TypeError:<EOL><INDENT>sequence = seq<EOL><DEDENT>sequence = np.array(list(sequence))<EOL>if fill_overhangs:<EOL><INDENT>sequence [:np.where(sequence != '<STR_LIT:->')[<NUM_LIT:0>][<NUM_LIT:0>]] = ambiguous_character<EOL>sequence [np.where(sequence != '<STR_LIT:->')[<NUM_LIT:0>][-<NUM_LIT:1>]+<NUM_LIT:1>:] = ambiguous_character<EOL><DEDENT>return sequence<EOL>

Take the raw sequence, substitute the "overhanging" gaps with 'N' (missequenced),
and convert the sequence to the numpy array of chars.

Parameters
----------
 seq : Biopython.SeqRecord, str, iterable
    Sequence as an object of SeqRecord, string or iterable

 fill_overhangs : bool
    If True, substitute the "overhanging" gaps with ambiguous character symbol

 ambiguous_character : char
    Specify the character for ambiguous state ('N' default for nucleotide)

Returns
-------
 sequence : np.array
    Sequence as 1D numpy array of chars


return np.array([profile_map[k] for k in seq])<EOL>

Convert the given character sequence into the profile according to the
alphabet specified.

Parameters
----------

 seq : numpy.array
    Sequence to be converted to the profile

 profile_map : dic
    Mapping valid characters to profiles

Returns
-------

 idx : numpy.array
    Profile for the character. Zero array if the character not found


def prof2seq(profile, gtr, sample_from_prof=False, normalize=True):

<EOL>if normalize:<EOL><INDENT>tmp_profile, pre=normalize_profile(profile, return_offset=False)<EOL><DEDENT>else:<EOL><INDENT>tmp_profile = profile<EOL><DEDENT>if sample_from_prof:<EOL><INDENT>cumdis = tmp_profile.cumsum(axis=<NUM_LIT:1>).T<EOL>randnum = np.random.random(size=cumdis.shape[<NUM_LIT:1>])<EOL>idx = np.argmax(cumdis>=randnum, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>idx = tmp_profile.argmax(axis=<NUM_LIT:1>)<EOL><DEDENT>seq = gtr.alphabet[idx]  <EOL>prof_values = tmp_profile[np.arange(tmp_profile.shape[<NUM_LIT:0>]), idx]<EOL>return seq, prof_values, idx<EOL>

Convert profile to sequence and normalize profile across sites.

Parameters
----------

 profile : numpy 2D array
    Profile. Shape of the profile should be (L x a), where L - sequence
    length, a - alphabet size.

 gtr : gtr.GTR
    Instance of the GTR class to supply the sequence alphabet

 collapse_prof : bool
    Whether to convert the profile to the delta-function

Returns
-------
 seq : numpy.array
    Sequence as numpy array of length L

 prof_values :  numpy.array
    Values of the profile for the chosen sequence characters (length L)

 idx : numpy.array
    Indices chosen from profile as array of length L


def normalize_profile(in_profile, log=False, return_offset = True):

if log:<EOL><INDENT>tmp_prefactor = in_profile.max(axis=<NUM_LIT:1>)<EOL>tmp_prof = np.exp(in_profile.T - tmp_prefactor).T<EOL><DEDENT>else:<EOL><INDENT>tmp_prefactor = <NUM_LIT:0.0><EOL>tmp_prof = in_profile<EOL><DEDENT>norm_vector = tmp_prof.sum(axis=<NUM_LIT:1>)<EOL>return (np.copy(np.einsum('<STR_LIT>',tmp_prof,<NUM_LIT:1.0>/norm_vector)),<EOL>(np.log(norm_vector) + tmp_prefactor) if return_offset else None)<EOL>

return a normalized version of a profile matrix

    Parameters
    ----------
    in_profile : np.array
        shape Lxq, will be normalized to one across each row
    log : bool, optional
        treat the input as log probabilities
    return_offset : bool, optional
        return the log of the scale factor for each row

    Returns
    -------
    tuple
        normalized profile (fresh np object) and offset (if return_offset==True)


if isinstance(distribution, interp1d):<EOL><INDENT>if is_neg_log:<EOL><INDENT>ymin = distribution.y.min()<EOL>log_prob = distribution.y-ymin<EOL><DEDENT>else:<EOL><INDENT>log_prob = -np.log(distribution.y)<EOL>log_prob -= log_prob.min()<EOL><DEDENT>xvals = distribution.x<EOL><DEDENT>elif isinstance(distribution, Distribution):<EOL><INDENT>xvals = distribution._func.x<EOL>log_prob = distribution._func.y<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>");<EOL><DEDENT>L = xvals.shape[<NUM_LIT:0>]<EOL>tmp = np.where(log_prob < <NUM_LIT>)[<NUM_LIT:0>]<EOL>x_l, x_u = tmp[<NUM_LIT:0>], tmp[-<NUM_LIT:1>]<EOL>if L < <NUM_LIT:2>:<EOL><INDENT>print ("<STR_LIT>")<EOL>return min(TINY_NUMBER, distribution.xmax - distribution.xmin)<EOL><DEDENT>else:<EOL><INDENT>return max(TINY_NUMBER, xvals[min(x_u+<NUM_LIT:1>,L-<NUM_LIT:1>)] - xvals[max(<NUM_LIT:0>,x_l-<NUM_LIT:1>)])<EOL><DEDENT>

Assess the width of the probability distribution. This returns
full-width-half-max


@classmethod<EOL><INDENT>def delta_function(cls, x_pos, weight=<NUM_LIT:1.>, min_width=MIN_INTEGRATION_PEAK):<DEDENT>

distribution = cls(x_pos,<NUM_LIT:0.>,is_log=True, min_width=min_width)<EOL>distribution.weight  = weight<EOL>return distribution<EOL>


if  not all([isinstance(k, Distribution) for k in dists]):<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>n_delta = np.sum([k.is_delta for k in dists])<EOL>min_width = np.max([k.min_width for k in dists])<EOL>if n_delta><NUM_LIT:1>:<EOL><INDENT>raise ArithmeticError("<STR_LIT>")<EOL><DEDENT>elif n_delta==<NUM_LIT:1>:<EOL><INDENT>delta_dist_ii = np.where([k.is_delta for k in dists])[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>delta_dist = dists[delta_dist_ii]<EOL>new_xpos = delta_dist.peak_pos<EOL>new_weight  = np.prod([k.prob(new_xpos) for k in dists if k!=delta_dist_ii]) * delta_dist.weight<EOL>res = Distribution.delta_function(new_xpos, weight = new_weight,min_width=min_width)<EOL><DEDENT>else:<EOL><INDENT>new_xmin = np.max([k.xmin for k in dists])<EOL>new_xmax = np.min([k.xmax for k in dists])<EOL>x_vals = np.unique(np.concatenate([k.x for k in dists]))<EOL>x_vals = x_vals[(x_vals>new_xmin-TINY_NUMBER)&(x_vals<new_xmax+TINY_NUMBER)]<EOL>y_vals = np.sum([k.__call__(x_vals) for k in dists], axis=<NUM_LIT:0>)<EOL>peak = y_vals.min()<EOL>ind = (y_vals-peak)<BIG_NUMBER/<NUM_LIT:1000><EOL>n_points = ind.sum()<EOL>if n_points == <NUM_LIT:0>:<EOL><INDENT>print ("<STR_LIT>")<EOL>x_vals = [<NUM_LIT:0>,<NUM_LIT:1>]<EOL>y_vals = [BIG_NUMBER,BIG_NUMBER]<EOL>res = Distribution(x_vals, y_vals, is_log=True,<EOL>min_width=min_width, kind='<STR_LIT>')<EOL><DEDENT>elif n_points == <NUM_LIT:1>:<EOL><INDENT>res = Distribution.delta_function(x_vals[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>res = Distribution(x_vals[ind], y_vals[ind], is_log=True,<EOL>min_width=min_width, kind='<STR_LIT>', assume_sorted=True)<EOL><DEDENT><DEDENT>return res<EOL>

multiplies a list of Distribution objects


def __init__(self, x, y, is_log=True, min_width = MIN_INTEGRATION_PEAK,<EOL>kind='<STR_LIT>', assume_sorted=False):

self.min_width = min_width<EOL>if isinstance(x, Iterable) and isinstance (y, Iterable):<EOL><INDENT>self._delta = False <EOL>if assume_sorted:<EOL><INDENT>xvals, yvals = x,y<EOL><DEDENT>else:<EOL><INDENT>xvals, yvals = np.array(sorted(zip(x,y))).T<EOL><DEDENT>if not is_log:<EOL><INDENT>yvals = -np.log(yvals)<EOL><DEDENT>yvals[np.isnan(yvals)] = BIG_NUMBER<EOL>self._kind=kind<EOL>self._xmin, self._xmax = xvals[<NUM_LIT:0>], xvals[-<NUM_LIT:1>]<EOL>self._support = self._xmax - self._xmin<EOL>self._peak_idx = yvals.argmin()<EOL>self._peak_val = yvals.min()<EOL>self._peak_pos = xvals[self._peak_idx]<EOL>yvals -= self._peak_val<EOL>self._ymax = yvals.max()<EOL>self._func= interp1d(xvals, yvals, kind=kind, fill_value=BIG_NUMBER,<EOL>bounds_error=False, assume_sorted=True)<EOL>self._fwhm = Distribution.calc_fwhm(self)<EOL><DEDENT>elif np.isscalar(x):<EOL><INDENT>assert (np.isscalar(y) or y is None)<EOL>self._delta = True<EOL>self._peak_pos = x<EOL>self._fwhm = <NUM_LIT:0><EOL>if y is None:<EOL><INDENT>self._peak_val = np.inf<EOL><DEDENT>else:<EOL><INDENT>self._peak_val = y<EOL><DEDENT>self._xmin, self._xmax = x, x<EOL>self._support = <NUM_LIT:0.><EOL>self._func = lambda x : (x==self.peak_pos)*self.peak_val<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>