<EOL>project_dir = ip.core.assembly._expander(parsedict['<STR_LIT>'])<EOL>assembly_name = parsedict['<STR_LIT>']<EOL>assembly_file = os.path.join(project_dir, assembly_name)<EOL>if not os.path.exists(project_dir):<EOL><INDENT>os.mkdir(project_dir)<EOL><DEDENT>try:<EOL><INDENT>if ('<STR_LIT:1>' in args.steps) and args.force:<EOL><INDENT>data = ip.Assembly(assembly_name, cli=True)<EOL><DEDENT>else:<EOL><INDENT>data = ip.load_json(assembly_file, cli=True)<EOL>data._cli = True<EOL><DEDENT><DEDENT>except IPyradWarningExit as _:<EOL><INDENT>if '<STR_LIT:1>' not in args.steps:<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>".format(assembly_file))<EOL><DEDENT>else:<EOL><INDENT>data = ip.Assembly(assembly_name, cli=True)<EOL><DEDENT><DEDENT>for param in parsedict:<EOL><INDENT>if param == "<STR_LIT>":<EOL><INDENT>if parsedict[param] != data.name:<EOL><INDENT>data.set_params(param, parsedict[param])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>data.set_params(param, parsedict[param])<EOL><DEDENT>except IndexError as _:<EOL><INDENT>print("<STR_LIT>".format(args.params))<EOL>print("<STR_LIT>".format(param, parsedict[param]))<EOL>sys.exit(-<NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT>return data<EOL>

loads assembly or creates a new one and set its params from
parsedict. Does not launch ipcluster.


import urllib2<EOL>from distutils.version import LooseVersion<EOL>header ="<STR_LIT>"+"<STR_LIT>".format(ip.__version__)+"<STR_LIT>"+"<STR_LIT>"<EOL>try:<EOL><INDENT>htmldat = urllib2.urlopen("<STR_LIT>").readlines()<EOL>curversion = next((x for x in htmldat if "<STR_LIT>" in x), None).split("<STR_LIT:>>")[<NUM_LIT:1>].split("<STR_LIT:<>")[<NUM_LIT:0>]<EOL>if LooseVersion(ip.__version__) < LooseVersion(curversion):<EOL><INDENT>msg = """<STR_LIT>""".format(curversion)<EOL>print(header + "<STR_LIT:\n>" + msg)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>except Exception as inst:<EOL><INDENT>pass<EOL><DEDENT>

Test if there's a newer version and nag the user to upgrade.


<EOL>parser = argparse.ArgumentParser(<EOL>formatter_class=argparse.RawDescriptionHelpFormatter,<EOL>epilog="""<STR_LIT>""")<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', action='<STR_LIT:version>', <EOL>version=str(pkg_resources.get_distribution('<STR_LIT>')))<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>", type=str, <EOL>default=None, <EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>type=str, default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>type=str, default=None, nargs="<STR_LIT:*>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>default=None, nargs="<STR_LIT:*>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT:-c>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=int, default=<NUM_LIT:0>,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=int, default=<NUM_LIT:2>,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, nargs="<STR_LIT:?>", const="<STR_LIT:default>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, nargs="<STR_LIT:*>", default=None, <EOL>help="<STR_LIT>")<EOL>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>args = parser.parse_args()<EOL>if not any(x in ["<STR_LIT>", "<STR_LIT>"] for x in vars(args).keys()):<EOL><INDENT>print("<STR_LIT>"+"<STR_LIT>")<EOL>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return args<EOL>


<EOL>ip.__interactive__ = <NUM_LIT:0><EOL>_check_version()<EOL>args = parse_command_line()<EOL>if os.path.exists(ip.__debugflag__):<EOL><INDENT>os.remove(ip.__debugflag__)<EOL><DEDENT>if args.debug:<EOL><INDENT>print("<STR_LIT>")<EOL>ip._debug_on()<EOL>atexit.register(ip._debug_off)        <EOL><DEDENT>if args.new:<EOL><INDENT>try:<EOL><INDENT>tmpassembly = ip.Assembly(args.new, quiet=True, cli=True)<EOL>tmpassembly.write_params("<STR_LIT>".format(args.new), <EOL>force=args.force)<EOL><DEDENT>except Exception as inst:<EOL><INDENT>print(inst)<EOL>sys.exit(<NUM_LIT:2>)<EOL><DEDENT>print("<STR_LIT>".format(args.new, os.path.realpath(os.path.curdir)))<EOL>sys.exit(<NUM_LIT:2>)<EOL><DEDENT>if args.params:<EOL><INDENT>if not any([args.branch, args.results, args.steps]):<EOL><INDENT>print("""<STR_LIT>""")<EOL>sys.exit(<NUM_LIT:2>)<EOL><DEDENT><DEDENT>if not args.params:<EOL><INDENT>if any([args.branch, args.results, args.steps]):<EOL><INDENT>print("""<STR_LIT>""")<EOL><DEDENT><DEDENT>if any([args.branch, args.merge]):        <EOL><INDENT>args.steps = "<STR_LIT>"    <EOL>print("<STR_LIT>")    <EOL><DEDENT>header ="<STR_LIT>"+"<STR_LIT>".format(ip.__version__)+"<STR_LIT>"+"<STR_LIT>"<EOL>with open(ip.__debugfile__, '<STR_LIT:a>') as logfile:<EOL><INDENT>logfile.write(header)<EOL>logfile.write("<STR_LIT>".format(time.strftime("<STR_LIT>")))<EOL>logfile.write("<STR_LIT>".format(vars(args)))<EOL>logfile.write("<STR_LIT>".format(os.uname()))<EOL><DEDENT>if args.merge:<EOL><INDENT>print(header)<EOL>merge_assemblies(args)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if args.download:<EOL><INDENT>if len(args.download) == <NUM_LIT:1>:<EOL><INDENT>downloaddir = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>downloaddir = args.download[<NUM_LIT:1>]<EOL><DEDENT>sratools_download(args.download[<NUM_LIT:0>], workdir=downloaddir, force=args.force)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>elif args.params:<EOL><INDENT>parsedict = parse_params(args)<EOL>if args.branch:<EOL><INDENT>branch_assembly(args, parsedict)<EOL><DEDENT>elif args.steps:<EOL><INDENT>print(header)<EOL>if os.path.exists(ip.__debugfile__):<EOL><INDENT>if os.path.getsize(ip.__debugfile__) > <NUM_LIT>:<EOL><INDENT>with open(ip.__debugfile__, '<STR_LIT:w>') as clear:<EOL><INDENT>clear.write("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>data = getassembly(args, parsedict)<EOL>data._ipcluster["<STR_LIT>"] = args.threads<EOL>if args.ipcluster:<EOL><INDENT>ipyclient = ipp.Client(profile=args.ipcluster)<EOL>data._ipcluster["<STR_LIT>"] = len(ipyclient)<EOL><DEDENT>else:<EOL><INDENT>ipyclient = None<EOL>data._ipcluster["<STR_LIT>"] = args.cores if args.cores else detect_cpus()<EOL>data._ipcluster["<STR_LIT>"] = "<STR_LIT>"<EOL>if args.MPI:<EOL><INDENT>data._ipcluster["<STR_LIT>"] = "<STR_LIT>"<EOL>if not args.cores:<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>")<EOL><DEDENT><DEDENT>data = register_ipcluster(data)<EOL><DEDENT>data._headers = <NUM_LIT:1><EOL>steps = list(args.steps)<EOL>data.run(<EOL>steps=steps, <EOL>force=args.force, <EOL>show_cluster=<NUM_LIT:1>, <EOL>ipyclient=ipyclient)<EOL><DEDENT>if args.results:<EOL><INDENT>showstats(parsedict)<EOL><DEDENT><DEDENT>


<EOL>hosts = []<EOL>for eid in ipyclient.ids:<EOL><INDENT>engine = ipyclient[eid]<EOL>if not engine.outstanding:<EOL><INDENT>hosts.append(engine.apply(_socket.gethostname))<EOL><DEDENT><DEDENT>hosts = [i.get() for i in hosts]<EOL>result = []<EOL>for hostname in set(hosts):<EOL><INDENT>result.append("<STR_LIT>".format(spacer, hosts.count(hostname), hostname))<EOL><DEDENT>print("<STR_LIT:\n>".join(result))<EOL>

reports host and engine info for an ipyclient


<EOL>with open(__debugflag__, '<STR_LIT:w>') as dfile:<EOL><INDENT>dfile.write("<STR_LIT>")<EOL><DEDENT>__loglevel__ = "<STR_LIT>"<EOL>_LOGGER.info("<STR_LIT>")<EOL>_set_debug_dict(__loglevel__)<EOL>

Turns on debugging by creating hidden tmp file
This is only run by the __main__ engine.


_lconfig.dictConfig({<EOL>'<STR_LIT:version>': <NUM_LIT:1>,<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': "<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"<EOL>},<EOL>},<EOL>'<STR_LIT>': {<EOL>__name__: {<EOL>'<STR_LIT>':__loglevel__,<EOL>'<STR_LIT:class>':'<STR_LIT>',<EOL>'<STR_LIT:filename>':__debugfile__,<EOL>'<STR_LIT>':"<STR_LIT>",<EOL>'<STR_LIT>':'<STR_LIT>'<EOL>}<EOL>},<EOL>'<STR_LIT>':{<EOL>__name__: {<EOL>'<STR_LIT>': [__name__],<EOL>'<STR_LIT>': __loglevel__,<EOL>'<STR_LIT>': True<EOL>}<EOL>}<EOL>})<EOL>


if _os.path.exists(__debugflag__):<EOL><INDENT>_os.remove(__debugflag__)<EOL><DEDENT>__loglevel__ = "<STR_LIT>"<EOL>_LOGGER.info("<STR_LIT>")<EOL>_set_debug_dict(__loglevel__)<EOL>

turns off debugging by removing hidden tmp file


return _subprocess.call("<STR_LIT>" + cmd,<EOL>shell=True,<EOL>stdout=_subprocess.PIPE,<EOL>stderr=_subprocess.PIPE) == <NUM_LIT:0><EOL>


<EOL>if not _sys.maxsize > <NUM_LIT:2>**<NUM_LIT:32>:<EOL><INDENT>_sys.exit("<STR_LIT>")<EOL><DEDENT>_platform = _sys.platform<EOL>if '<STR_LIT>' in _os.environ:<EOL><INDENT>ipyrad_path = _os.environ['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>path = _os.path.abspath(_os.path.dirname(__file__))<EOL>ipyrad_path = _os.path.dirname(path)<EOL><DEDENT>ipyrad_path = _os.path.dirname(path)<EOL>bin_path = _os.path.join(ipyrad_path, "<STR_LIT>")<EOL>if '<STR_LIT>' in _platform:<EOL><INDENT>vsearch = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>muscle = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>smalt = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>bwa = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>samtools = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>bedtools = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>qmc = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>vsearch = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>muscle = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>smalt = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>bwa = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>samtools = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>bedtools = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL>qmc = _os.path.join(<EOL>_os.path.abspath(bin_path),<EOL>"<STR_LIT>")<EOL><DEDENT>assert _cmd_exists(muscle), "<STR_LIT>"+muscle<EOL>assert _cmd_exists(vsearch), "<STR_LIT>"+vsearch<EOL>assert _cmd_exists(smalt), "<STR_LIT>"+smalt<EOL>assert _cmd_exists(bwa), "<STR_LIT>"+bwa<EOL>assert _cmd_exists(samtools), "<STR_LIT>"+samtools<EOL>assert _cmd_exists(bedtools), "<STR_LIT>"+bedtools<EOL>return vsearch, muscle, smalt, bwa, samtools, bedtools, qmc<EOL>

gets the right version of vsearch, muscle, and smalt
    depending on linux vs osx


print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>new_assembly = ip.Assembly("<STR_LIT>", quiet=True)<EOL>data._hackersonly = deepcopy(new_assembly._hackersonly)<EOL>new_params = set(new_assembly.paramsdict.keys())<EOL>my_params = set(data.paramsdict.keys())<EOL>removed_params = my_params.difference(new_params)<EOL>for i in removed_params:<EOL><INDENT>print("<STR_LIT>".format(i, data.paramsdict[i]))<EOL><DEDENT>added_params = new_params.difference(my_params)<EOL>for i in added_params:<EOL><INDENT>print("<STR_LIT>".format(i, new_assembly.paramsdict[i]))<EOL><DEDENT>print("<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>")<EOL>time.sleep(<NUM_LIT:5>)<EOL>print("<STR_LIT>")<EOL>if added_params:<EOL><INDENT>for i in data.paramsdict:<EOL><INDENT>if i not in removed_params:<EOL><INDENT>new_assembly.paramsdict[i] = data.paramsdict[i]<EOL><DEDENT><DEDENT>data.paramsdict = deepcopy(new_assembly.paramsdict)<EOL><DEDENT>data.save()<EOL>return data<EOL>

Create a new Assembly() and convert as many of our old params to the new
version as we can. Also report out any parameters that are removed
and what their values are.


<EOL>datadict = OrderedDict([<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", dict(data.__dict__["<STR_LIT>"])),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"])<EOL>])<EOL>


<EOL>datadict = OrderedDict([<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT:name>", data.__dict__["<STR_LIT:name>"]), <EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"].keys()),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),        <EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>])<EOL>sampledict = OrderedDict([])<EOL>for key, sample in data.samples.iteritems():<EOL><INDENT>sampledict[key] = sample._to_fulldict()<EOL><DEDENT>fulldumps = json.dumps({<EOL>"<STR_LIT>": datadict,<EOL>"<STR_LIT>": sampledict<EOL>},<EOL>cls=Encoder,<EOL>sort_keys=False, indent=<NUM_LIT:4>, separators=("<STR_LIT:U+002C>", "<STR_LIT::>"),<EOL>)<EOL>assemblypath = os.path.join(data.dirs.project, data.name+"<STR_LIT>")<EOL>if not os.path.exists(data.dirs.project):<EOL><INDENT>os.mkdir(data.dirs.project)<EOL><DEDENT>done = <NUM_LIT:0><EOL>while not done:<EOL><INDENT>try:<EOL><INDENT>with open(assemblypath, '<STR_LIT:w>') as jout:<EOL><INDENT>jout.write(fulldumps)<EOL><DEDENT>done = <NUM_LIT:1><EOL><DEDENT>except (KeyboardInterrupt, SystemExit): <EOL><INDENT>print('<STR_LIT:.>')<EOL>continue<EOL><DEDENT><DEDENT>


def load_json(path, quiet=False, cli=False):

<EOL>checkfor = [path+"<STR_LIT>", path]<EOL>for inpath in checkfor:<EOL><INDENT>inpath = inpath.replace("<STR_LIT>", os.path.expanduser("<STR_LIT>"))<EOL>try:<EOL><INDENT>with open(inpath, '<STR_LIT:r>') as infile:<EOL><INDENT>fullj = json.loads(infile.read(), object_hook=_tup_and_byte)<EOL><DEDENT><DEDENT>except IOError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>oldname = fullj["<STR_LIT>"].pop("<STR_LIT:name>")<EOL>olddir = fullj["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"]<EOL>oldpath = os.path.join(olddir, os.path.splitext(oldname)[<NUM_LIT:0>]+"<STR_LIT>")<EOL>null = ip.Assembly(oldname, quiet=True, cli=cli)<EOL><DEDENT>except (UnboundLocalError, AttributeError) as inst:<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(inpath))<EOL><DEDENT>if not quiet:<EOL><INDENT>oldpath = oldpath.replace(os.path.expanduser("<STR_LIT>"), "<STR_LIT>")<EOL>print("<STR_LIT>".format(null._spacer, oldname))<EOL>print("<STR_LIT>".format(null._spacer, oldpath))<EOL><DEDENT>samplekeys = fullj["<STR_LIT>"].pop("<STR_LIT>")<EOL>null.samples = {name: "<STR_LIT>" for name in samplekeys}<EOL>oldparams = fullj["<STR_LIT>"].pop("<STR_LIT>")<EOL>for param, val in oldparams.iteritems():<EOL><INDENT>if param not in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>try:<EOL><INDENT>null.set_params(param, val)<EOL><DEDENT>except IPyradWarningExit as inst:<EOL><INDENT>LOGGER.warning("""<STR_LIT>""".format(inst))<EOL><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>oldhackersonly = fullj["<STR_LIT>"].pop("<STR_LIT>")<EOL>for param, val in oldhackersonly.iteritems():<EOL><INDENT>if val == None:<EOL><INDENT>null._hackersonly[param] = None<EOL><DEDENT>else:<EOL><INDENT>null._hackersonly[param] = val<EOL><DEDENT><DEDENT><DEDENT>except Exception as inst:<EOL><INDENT>LOGGER.warning("""<STR_LIT>""")<EOL><DEDENT>newkeys = null.__dict__.keys()<EOL>oldkeys = fullj["<STR_LIT>"].keys()<EOL>sharedkeys = set(oldkeys).intersection(set(newkeys))<EOL>lostkeys = set(oldkeys).difference(set(newkeys))<EOL>if lostkeys:<EOL><INDENT>LOGGER.warning("""<STR_LIT>""".format(a=len(lostkeys), <EOL>b=oldname,<EOL>c=fullj["<STR_LIT>"]["<STR_LIT>"],<EOL>d=lostkeys,<EOL>e=null._version))<EOL><DEDENT>for key in sharedkeys:<EOL><INDENT>null.__setattr__(key, fullj["<STR_LIT>"][key])<EOL><DEDENT>try:<EOL><INDENT>if fullj["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>null.__setattr__("<STR_LIT>", fullj["<STR_LIT>"]["<STR_LIT>"])<EOL>null.svd = ObjDict(null.svd)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>LOGGER.debug("<STR_LIT>")<EOL><DEDENT>sample_names = fullj["<STR_LIT>"].keys()<EOL>if not sample_names:<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(inpath))<EOL><DEDENT>sample_keys = fullj["<STR_LIT>"][sample_names[<NUM_LIT:0>]].keys()<EOL>stats_keys = fullj["<STR_LIT>"][sample_names[<NUM_LIT:0>]]["<STR_LIT>"].keys()<EOL>stats_dfs_keys = fullj["<STR_LIT>"][sample_names[<NUM_LIT:0>]]["<STR_LIT>"].keys()<EOL>ind_statkeys =[fullj["<STR_LIT>"][sample_names[<NUM_LIT:0>]]["<STR_LIT>"][i].keys()for i in stats_dfs_keys]<EOL>ind_statkeys = list(itertools.chain(*ind_statkeys))<EOL>nsamp = ip.Sample()<EOL>newkeys = nsamp.__dict__.keys()<EOL>newstats = nsamp.__dict__["<STR_LIT>"].keys()<EOL>newstatdfs = nsamp.__dict__["<STR_LIT>"].keys()<EOL>newindstats = [nsamp.__dict__["<STR_LIT>"][i].keys() for i in newstatdfs]<EOL>newindstats = list(itertools.chain(*[i.values for i in newindstats]))<EOL>diffattr = set(sample_keys).difference(newkeys)<EOL>diffstats = set(stats_keys).difference(newstats)<EOL>diffindstats = set(ind_statkeys).difference(newindstats)<EOL>alldiffs = diffattr.union(diffstats).union(diffindstats)<EOL>if any(alldiffs):<EOL><INDENT>LOGGER.warning("""<STR_LIT>""".format(a=len(alldiffs), <EOL>b=oldname,<EOL>c=fullj["<STR_LIT>"]["<STR_LIT>"],<EOL>d=alldiffs,<EOL>e=null._version))<EOL><DEDENT>for sample in null.samples:<EOL><INDENT>null.samples[sample] = ip.Sample()<EOL>sdat = fullj["<STR_LIT>"][sample]['<STR_LIT>']<EOL>newstats = [x for x in newstats if x in stats_keys]<EOL>null.samples[sample].stats = pd.Series(sdat).reindex(newstats)<EOL>for statskey in stats_dfs_keys:<EOL><INDENT>null.samples[sample].stats_dfs[statskey] =pd.Series(fullj["<STR_LIT>"][sample]["<STR_LIT>"][statskey]).reindex(nsamp.__dict__["<STR_LIT>"][statskey].keys())<EOL><DEDENT>for filehandle in fullj["<STR_LIT>"][sample]["<STR_LIT>"].keys():<EOL><INDENT>null.samples[sample].files[filehandle] =fullj["<STR_LIT>"][sample]["<STR_LIT>"][filehandle]<EOL><DEDENT><DEDENT>for statskey in stats_dfs_keys:<EOL><INDENT>indstat = null._build_stat(statskey)<EOL>if not indstat.empty:<EOL><INDENT>null.stats_dfs[statskey] = indstat<EOL><DEDENT><DEDENT>shared_keys = set(sample_keys).intersection(newkeys)<EOL>shared_keys.discard("<STR_LIT>")<EOL>shared_keys.discard("<STR_LIT>")    <EOL>shared_keys.discard("<STR_LIT>")<EOL>shared_keys.discard("<STR_LIT>")<EOL>for sample in null.samples:<EOL><INDENT>for key in shared_keys:<EOL><INDENT>null.samples[sample].__setattr__(key, fullj["<STR_LIT>"][sample][key])<EOL><DEDENT><DEDENT>null.dirs = ObjDict(null.dirs)<EOL>null.stats_files = ObjDict(null.stats_files)<EOL>null.stats_dfs = ObjDict(null.stats_dfs)    <EOL>null.populations = ObjDict(null.populations)<EOL>null.outfiles = ObjDict(null.outfiles)<EOL>return null<EOL>

Load a json serialized object and ensure it matches to the current
Assembly object format


<EOL>if isinstance(obj, unicode):<EOL><INDENT>return obj.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if isinstance(obj, list):<EOL><INDENT>return [_tup_and_byte(item) for item in obj]<EOL><DEDENT>if isinstance(obj, dict):<EOL><INDENT>if "<STR_LIT>" in obj:<EOL><INDENT>return tuple(_tup_and_byte(item) for item in obj["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>return {<EOL>_tup_and_byte(key): _tup_and_byte(val) forkey, val in obj.iteritems()<EOL>}<EOL><DEDENT><DEDENT>return obj<EOL>


def hint_tuples(item):<EOL><INDENT>"""<STR_LIT>"""<EOL>if isinstance(item, tuple):<EOL><INDENT>return {'<STR_LIT>': True, '<STR_LIT>': item}<EOL><DEDENT>if isinstance(item, list):<EOL><INDENT>return [hint_tuples(e) for e in item]<EOL><DEDENT>if isinstance(item, dict):<EOL><INDENT>return {<EOL>key: hint_tuples(val) for key, val in item.iteritems()<EOL>}<EOL><DEDENT>else:<EOL><INDENT>return item<EOL><DEDENT><DEDENT>return super(Encoder, self).encode(hint_tuples(obj))<EOL>


outfile = open(os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>"), '<STR_LIT:w>')<EOL>inloci = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>names = [i.name for i in samples]<EOL>names.sort()<EOL>version = "<STR_LIT>"<EOL>mindepth = data.paramsdict["<STR_LIT>"] <EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>"+time.strftime("<STR_LIT>"), file=outfile)<EOL>print("<STR_LIT>"+version, file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT>", file=outfile)<EOL>print("<STR_LIT:\t>".join(["<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>","<STR_LIT>"]+list(names)), file=outfile)<EOL>loci = open(inloci).read().split("<STR_LIT:|>")[:-<NUM_LIT:1>]<EOL>snps = <NUM_LIT:0><EOL>vcflist = []<EOL>for locusnumber in range(len(loci)):<EOL><INDENT>samps = [i.split()[<NUM_LIT:0>][<NUM_LIT:1>:] for i in loci[locusnumber].strip().split("<STR_LIT:\n>") if "<STR_LIT:>>" in i]<EOL>loc = np.array([tuple(i.split()[-<NUM_LIT:1>]) for i in loci[locusnumber].strip().split("<STR_LIT:\n>") if "<STR_LIT:>>" in i])<EOL>NS = str(len(loc))<EOL>DP = str(mindepth)<EOL>for base in range(len(loc.T)):<EOL><INDENT>col = []<EOL>site = list(loc.T[base])<EOL>site = list("<STR_LIT>".join(site).replace("<STR_LIT:->","<STR_LIT>").replace("<STR_LIT:N>","<STR_LIT>"))<EOL>if site:<EOL><INDENT>for bb in site:<EOL><INDENT>if bb in list("<STR_LIT>"):<EOL><INDENT>col += unstruct(bb)[<NUM_LIT:0>]<EOL>col += unstruct(bb)[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>col += bb<EOL><DEDENT><DEDENT>REF = most_common([i for i in col if i not in list("<STR_LIT>")])<EOL>ALT = set([i for i in col if (i in list("<STR_LIT>")) and (i!=REF)])<EOL>if ALT:<EOL><INDENT>snps += <NUM_LIT:1><EOL>GENO = [REF]+list(ALT)<EOL>GENOS = []<EOL>for samp in names:<EOL><INDENT>if samp in samps:<EOL><INDENT>idx = samps.index(samp)<EOL>f = unstruct(loc.T[base][idx])<EOL>if ('<STR_LIT:->' in f) or ('<STR_LIT:N>' in f):<EOL><INDENT>GENOS.append("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>GENOS.append(str(GENO.index(f[<NUM_LIT:0>]))+"<STR_LIT:|>"+str(GENO.index(f[<NUM_LIT:1>])))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>GENOS.append("<STR_LIT>")<EOL><DEDENT><DEDENT>vcflist.append("<STR_LIT:\t>".join([repr(locusnumber+<NUM_LIT:1>), repr(base+<NUM_LIT:1>), '<STR_LIT:.>', REF, "<STR_LIT:U+002C>".join(ALT), "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT:;>".join(["<STR_LIT>"+NS, "<STR_LIT>"+DP]), "<STR_LIT>"]+GENOS))<EOL><DEDENT><DEDENT><DEDENT>if not locusnumber % <NUM_LIT:1000>:<EOL><INDENT>outfile.write( "<STR_LIT:\n>".join(vcflist)+"<STR_LIT:\n>" )<EOL>vcflist = []<EOL><DEDENT><DEDENT>outfile.write( "<STR_LIT:\n>".join(vcflist) )<EOL>outfile.close()<EOL>

build a vcf file from the supercatg array and the cat.clust.gz output


def loci2migrate(name, locifile, popdict, mindict=<NUM_LIT:1>):

<EOL>outfile = open(name+"<STR_LIT>", '<STR_LIT:w>')<EOL>infile = open(locifile, '<STR_LIT:r>')<EOL>if isinstance(mindict, int):<EOL><INDENT>mindict = {pop: mindict for pop in popdict}<EOL><DEDENT>else:<EOL><INDENT>mindict = mindict<EOL><DEDENT>keep = []<EOL>MINS = zip(taxa.keys(), minhits)<EOL>loci  = infile.read().strip().split("<STR_LIT:|>")[:-<NUM_LIT:1>]<EOL>for loc in loci:<EOL><INDENT>samps = [i.split()[<NUM_LIT:0>].replace("<STR_LIT:>>","<STR_LIT>") for i in loc.split("<STR_LIT:\n>") if "<STR_LIT:>>" in i]<EOL>GG = []<EOL>for group,mins in MINS:<EOL><INDENT>GG.append( sum([i in samps for i in taxa[group]]) >= int(mins) )<EOL><DEDENT>if all(GG):<EOL><INDENT>keep.append(loc)<EOL><DEDENT><DEDENT>print >>outfile, len(taxa), len(keep), "<STR_LIT>", data.name+"<STR_LIT>","<STR_LIT:)>"<EOL>done = <NUM_LIT:0><EOL>for group in taxa:<EOL><INDENT>if not done:<EOL><INDENT>loclens = [len(loc.split("<STR_LIT:\n>")[<NUM_LIT:1>].split()[-<NUM_LIT:1>].replace("<STR_LIT:x>","<STR_LIT:n>").replace("<STR_LIT:n>","<STR_LIT>")) for loc in keep]<EOL>print >>outfile, "<STR_LIT:U+0020>".join(map(str,loclens))<EOL>done += <NUM_LIT:1><EOL><DEDENT>indslist = []<EOL>for loc in keep:<EOL><INDENT>samps = [i.split()[<NUM_LIT:0>].replace("<STR_LIT:>>","<STR_LIT>") for i in loc.split("<STR_LIT:\n>") if "<STR_LIT:>>" in i]<EOL>inds = sum([i in samps for i in taxa[group]])<EOL>indslist.append(inds)<EOL><DEDENT>print >>outfile, "<STR_LIT:U+0020>".join(map(str,indslist)), group<EOL>for loc in range(len(keep)):<EOL><INDENT>seqs = [i.split()[-<NUM_LIT:1>] for i in keep[loc].split("<STR_LIT:\n>") ifi.split()[<NUM_LIT:0>].replace("<STR_LIT:>>","<STR_LIT>") in taxa[group]]<EOL>for i in range(len(seqs)):<EOL><INDENT>print >>outfile, group[<NUM_LIT:0>:<NUM_LIT:8>]+"<STR_LIT:_>"+str(i)+("<STR_LIT:U+0020>"*(<NUM_LIT:10>-len(group[<NUM_LIT:0>:<NUM_LIT:8>]+"<STR_LIT:_>"+str(i))))+seqs[i].replace("<STR_LIT:x>","<STR_LIT:n>").replace("<STR_LIT:n>","<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>outfile.close()<EOL>

A function to build an input file for the program migrate from an ipyrad
.loci file, and a dictionary grouping Samples into populations.

Parameters:
-----------
name: (str)
   The name prefix for the migrate formatted output file.
locifile: (str)
   The path to the .loci file produced by ipyrad.
popdict: (dict)
   A Python dictionary grouping Samples into Populations.

Examples:
---------
You can create the population dictionary by hand, and pass in the path
to your .loci file as a string.
   >> popdict = {'A': ['a', 'b', 'c'], 'B': ['d', 'e', 'f']}
   >> loci2migrate("outfile.migrate", "./mydata.loci", popdict)

Or, if you load your ipyrad.Assembly object from it's JSON file, you can
access the loci file path and population information from there directly.
   >> data = ip.load_json("mydata.json")
   >> loci2migrate("outfile.migrate", data.outfiles.loci, data.populations)


<EOL>outfile = open(os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>"), '<STR_LIT:w>')<EOL>lines = open(os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>"), '<STR_LIT:r>')<EOL>longname = max(len(x) for x in data.samples.keys())<EOL>name_padding = <NUM_LIT:5><EOL>writing = []<EOL>loc = <NUM_LIT:0><EOL>for line in lines:<EOL><INDENT>if "<STR_LIT:>>" in line:<EOL><INDENT>name, seq = line.split("<STR_LIT:U+0020>")[<NUM_LIT:0>], line.split("<STR_LIT:U+0020>")[-<NUM_LIT:1>]<EOL>allele1, allele2 = splitalleles(seq.strip())<EOL>writing.append(name+"<STR_LIT>"+"<STR_LIT:U+0020>"*(longname-len(name)-<NUM_LIT:2>+name_padding)+allele1)<EOL>writing.append(name+"<STR_LIT>"+"<STR_LIT:U+0020>"*(longname-len(name)-<NUM_LIT:2>+name_padding)+allele2)<EOL><DEDENT>else:<EOL><INDENT>writing.append(line.strip())<EOL><DEDENT>loc += <NUM_LIT:1><EOL>if not loc % <NUM_LIT>:<EOL><INDENT>outfile.write("<STR_LIT:\n>".join(writing)+"<STR_LIT:\n>")<EOL>writing = []<EOL><DEDENT><DEDENT>outfile.write("<STR_LIT:\n>".join(writing))<EOL>outfile.close()<EOL>

reads in .loci and builds alleles from case characters


invcffile   =  os.path.join( data.dirs.consens, data.name+"<STR_LIT>" )<EOL>outlocifile  =  os.path.join( data.dirs.outfiles, data.name+"<STR_LIT>" )<EOL>importvcf( invcffile, outlocifile )<EOL>

Convert vcf from step6 to .loci format to facilitate downstream format conversion


try:<EOL><INDENT>with open( invcffile, '<STR_LIT:r>' ) as invcf:<EOL><INDENT>for line in invcf:<EOL><INDENT>if line.split()[<NUM_LIT:0>] == "<STR_LIT>":<EOL><INDENT>names_col = line.split().index( "<STR_LIT>" ) + <NUM_LIT:1><EOL>names = line.split()[ names_col:]<EOL>LOGGER.debug( "<STR_LIT>", names )<EOL>break<EOL><DEDENT><DEDENT>print( "<STR_LIT>" )<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>print( "<STR_LIT>" )<EOL><DEDENT>

Function for importing a vcf file into loci format. Arguments
        are the input vcffile and the loci file to write out.


<EOL>if not imap:<EOL><INDENT>raise IPyradWarningExit(IMAP_REQUIRED)<EOL><DEDENT>if minmap:<EOL><INDENT>if minmap.keys() != imap.keys():<EOL><INDENT>raise IPyradWarningExit(KEYS_DIFFER)<EOL><DEDENT><DEDENT>if wdir:<EOL><INDENT>wdir = os.path.abspath(wdir)<EOL>if not os.path.exists(wdir):<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wdir = os.path.curdir<EOL><DEDENT>prog = '<STR_LIT>'<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>prog = '<STR_LIT>'<EOL><DEDENT>outfile = OPJ(wdir, "<STR_LIT>".format(name, prog))<EOL>mapfile = OPJ(wdir, "<STR_LIT>".format(name, prog))<EOL>fout = open(outfile, '<STR_LIT:w>')<EOL>fmap = open(mapfile, '<STR_LIT:w>')<EOL>with open(locifile, '<STR_LIT:r>') as infile:<EOL><INDENT>loci = infile.read().strip().split("<STR_LIT>")<EOL>nloci = len(loci)<EOL><DEDENT>samples = list(itertools.chain(*imap.values()))<EOL>nkept = <NUM_LIT:0><EOL>for iloc in xrange(nloci):<EOL><INDENT>lines = loci[iloc].split("<STR_LIT>")[<NUM_LIT:0>].split()<EOL>names = lines[::<NUM_LIT:2>]<EOL>names = ["<STR_LIT>"+i for i in names]<EOL>seqs = [list(i) for i in lines[<NUM_LIT:1>::<NUM_LIT:2>]]<EOL>seqlen = len(seqs[<NUM_LIT:0>])<EOL>skip = <NUM_LIT:0><EOL>if minmap:<EOL><INDENT>covd = {}<EOL>for group, vals in imap.items():<EOL><INDENT>covd[group] = sum(["<STR_LIT>"+i in names for i in vals])<EOL><DEDENT>if not all([covd[group] >= minmap[group] for group in minmap]):<EOL><INDENT>skip = <NUM_LIT:1><EOL><DEDENT><DEDENT>if maxloci:<EOL><INDENT>if nkept >= maxloci:<EOL><INDENT>skip = <NUM_LIT:1><EOL><DEDENT><DEDENT>if not skip:<EOL><INDENT>data = ["<STR_LIT>".format(i, "<STR_LIT>".join(k).replace("<STR_LIT:->", "<STR_LIT:N>")) for(i, k) in zip(names, seqs) if i[<NUM_LIT:1>:] in samples]<EOL>if data:<EOL><INDENT>fout.write("<STR_LIT>".format(len(data), seqlen, "<STR_LIT:\n>".join(data)))<EOL>nkept += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>fout.close()<EOL>data = ["<STR_LIT>".format(val, key) for keyin sorted(imap) for val in imap[key]]<EOL>fmap.write("<STR_LIT:\n>".join(data))<EOL>fmap.close()<EOL>write_ctl(name, imap, guidetree, nkept,<EOL>infer_sptree, infer_delimit, delimit_alg,<EOL>seed, burnin, nsample, sampfreq,<EOL>thetaprior, tauprior, traits_df, nu, kappa,<EOL>cleandata, useseqdata, usetraitdata, wdir,<EOL>finetune, verbose)<EOL>sys.stderr.write("<STR_LIT>".format(nkept, len(imap.keys()),<EOL>sum([len(i) for i in imap.values()])))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL><DEDENT>return os.path.abspath(<EOL>"<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>

Converts loci file format to bpp file format, i.e., concatenated phylip-like
format, and produces imap and ctl input files for bpp.

Parameters:
-----------
name:
    A prefix name for output files that will be produced
locifile:
    A .loci file produced by ipyrad.
imap:
    A Python dictionary with 'species' names as keys, and lists of sample
    names for the values. Any sample that is not included in the imap
    dictionary will be filtered out of the data when converting the .loci
    file into the bpp formatted sequence file. Each species in the imap
    dictionary must also be present in the input 'guidetree'.
guidetree:
    A newick string species tree hypothesis [e.g., (((a,b),(c,d)),e);]
    All species in the imap dictionary must also be present in the guidetree

Optional parameters:
--------------------
infer_sptree:
    Default=0, only infer parameters on a fixed species tree. If 1, then the
    input tree is treated as a guidetree and tree search is employed to find
    the best tree. The results will include support values for the inferred
    topology.
infer_delimit:
    Default=0, no delimitation. If 1 then splits in the tree that separate
    'species' will be collapsed to test whether fewer species are a better
    fit to the data than the number in the input guidetree.
delimit_alg:
    Species delimitation algorithm. This is a tuple. The first value
    is the algorithm (0 or 1) and the following values are arguments
    for the given algorithm. See other ctl files for examples of what the
    delimitation line looks like. This is where you can enter the params
    (e.g., alpha, migration) for the two different algorithms.
    For example, the following args would produce the following ctl lines:
       alg=0, epsilon=5
       > delimit_alg = (0, 5)
       speciesdelimitation = 1 0 5

       alg=1, alpha=2, migration=1
       > delimit_alg = (1, 2, 1)
       speciesdelimitation = 1 1 2 1

       alg=1, alpha=2, migration=1, diagnosis=0, ?=1
       > delimit_alg = (1, 2, 1, 0, 1)
       speciesdelimitation = 1 1 2 1 0 1
seed:
    A random number seed at start of analysis.
burnin:
    Number of burnin generations in mcmc
nsample:
    Number of mcmc generations to run.
sampfreq:
    How often to sample from the mcmc chain.
thetaprior:
    Prior on theta (4Neu), gamma distributed. mean = a/b. e.g., (5, 5)
tauprior
    Prior on root tau, gamma distributed mean = a/b. Last number is
    dirichlet prior for other taus. e.g., (4, 2, 1)
traits_df:
    A pandas DataFrame with trait data properly formatted. This means only
    quantitative traits are included, and missing values are NaN.
    The first column contains sample names, with "Indiv" as the header.
    The following columns have a header row with trait names. This script
    will write a CSV trait file with trait values mean-standardized, with
    NaN replaced by "NA", and with sample not present in IMAP removed.
nu:
    A prior on phenotypic trait variance (0) for iBPP analysis.
kappa:
    A prior on phenotypic trait mean (0) for iBPP analysis.
useseqdata:
    If false inference proceeds without sequence data (can be used to test
    the effect of priors on the tree distributions).
usetraitdata:
    If false inference proceeds without trait data (can be used to test
    the effect of priors on the trait distributions).
cleandata:
    If 1 then sites with missing or hetero characters are removed.
wdir:
    A working directory to write files to.
finetune:
    See bpp documentation.
verbose:
    If verbose=1 the ctl file text will also be written to screen (stderr).


def write_ctl(name, imap, guidetree, nloci,<EOL>infer_sptree, infer_delimit, delimit_alg,<EOL>seed, burnin, nsample, sampfreq,<EOL>thetaprior, tauprior, traits_df, nu0, kappa0,<EOL>cleandata, useseqdata, usetraitdata, wdir,<EOL>finetune, verbose):

<EOL>ctl = []<EOL>if not guidetree.endswith("<STR_LIT:;>"):<EOL><INDENT>guidetree += "<STR_LIT:;>"<EOL><DEDENT>prog = '<STR_LIT>'<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>prog = '<STR_LIT>'<EOL><DEDENT>ctl.append("<STR_LIT>".format(seed))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL><DEDENT>ctl.append("<STR_LIT>".format(nloci))<EOL>ctl.append("<STR_LIT>".format(useseqdata))<EOL>ctl.append("<STR_LIT>".format(cleandata))<EOL>if infer_sptree:<EOL><INDENT>ctl.append("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>ctl.append("<STR_LIT>")<EOL><DEDENT>ctl.append("<STR_LIT>".format(infer_delimit, delimit_alg[<NUM_LIT:0>],<EOL>"<STR_LIT:U+0020>".join([str(i) for i in delimit_alg[<NUM_LIT:1>:]])))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>try:<EOL><INDENT>traits_df.values.astype(float)<EOL><DEDENT>except Exception:<EOL><INDENT>raise IPyradWarningExit(PDREAD_ERROR)<EOL><DEDENT>samples = sorted(list(itertools.chain(*imap.values())))<EOL>didx = [list(traits_df.index).index(i) for i in traits_df.indexif i not in samples]<EOL>dtraits = traits_df.drop(traits_df.index[didx])<EOL>straits = dtraits.apply(lambda x: (x - x.mean()) / (x.std()))<EOL>ftraits = straits.fillna("<STR_LIT>")<EOL>traitdict = ftraits.T.to_dict("<STR_LIT:list>")<EOL>rev = {val:key for key in sorted(imap) for val in imap[key]}<EOL>traitfile = "<STR_LIT>".format(os.path.join(wdir, name), prog)<EOL>with open(traitfile, '<STR_LIT:w>') as tout:<EOL><INDENT>tout.write("<STR_LIT>")<EOL>tout.write("<STR_LIT:\t>".join(<EOL>['<STR_LIT>'] + list(ftraits.columns))+"<STR_LIT:\n>"<EOL>)<EOL>nindT = <NUM_LIT:0><EOL>for ikey in sorted(imap.keys()):<EOL><INDENT>samps = imap[ikey]<EOL>for samp in sorted(samps):<EOL><INDENT>if samp in traitdict:<EOL><INDENT>tout.write("<STR_LIT:\t>".join([samp, rev[samp]] +[str(i) for i in traitdict[samp]])+"<STR_LIT:\n>"<EOL>)<EOL>nindT += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>ctl.append("<STR_LIT>".format(traits_df.shape[<NUM_LIT:1>]))<EOL>ctl.append("<STR_LIT>".format(nindT))  <EOL>ctl.append("<STR_LIT>".format(usetraitdata))<EOL>ctl.append("<STR_LIT>".format(useseqdata))<EOL>ctl.append("<STR_LIT>".format(nu0))<EOL>ctl.append("<STR_LIT>".format(kappa0))<EOL>ctl.remove("<STR_LIT>".format(useseqdata))<EOL>ctl.remove("<STR_LIT>".format(infer_sptree))<EOL><DEDENT>nspecies = str(len(imap))<EOL>species = "<STR_LIT:U+0020>".join(sorted(imap))<EOL>ninds = "<STR_LIT:U+0020>".join([str(len(imap[i])) for i in sorted(imap)])<EOL>ctl.append("""<STR_LIT>""".format(nspecies, species, ninds, guidetree))<EOL>ctl.append("<STR_LIT>".format(*thetaprior))<EOL>ctl.append("<STR_LIT>".format(*tauprior))<EOL>ctl.append("<STR_LIT>".format("<STR_LIT:U+0020>".join([str(i) for i in finetune])))<EOL>ctl.append("<STR_LIT>")<EOL>ctl.append("<STR_LIT>".format(burnin))<EOL>ctl.append("<STR_LIT>".format(sampfreq))<EOL>ctl.append("<STR_LIT>".format(nsample))<EOL>with open("<STR_LIT>".format(OPJ(wdir, name), prog), '<STR_LIT:w>') as out:<EOL><INDENT>out.write("<STR_LIT:\n>".join(ctl))<EOL><DEDENT>if verbose:<EOL><INDENT>sys.stderr.write("<STR_LIT>"+"<STR_LIT:\n>".join(ctl)+"<STR_LIT>")<EOL><DEDENT>


def loci2cf(name, locifile, popdict, wdir=None, ipyclient=None):

<EOL>if wdir:<EOL><INDENT>wdir = os.path.abspath(wdir)<EOL>if not os.path.exists(wdir):<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wdir = os.path.curdir<EOL><DEDENT>name = name.rsplit("<STR_LIT>")[<NUM_LIT:0>]<EOL>outfile = os.path.join(wdir, "<STR_LIT>".format(name))<EOL>out = open(outfile, '<STR_LIT:w>')<EOL>with open(locifile) as inloc:<EOL><INDENT>loci = inloc.read().strip().split("<STR_LIT>")    <EOL><DEDENT>names = list(itertools.chain(*popdict.values()))<EOL>popkeys = sorted(popdict.keys())<EOL>nsites = sum(len(loc.split("<STR_LIT:\n>")[<NUM_LIT:0>].split()[<NUM_LIT:1>]) for loc in loci[:])<EOL>out.write(HEADER.format(**{"<STR_LIT>": len(popdict),<EOL>"<STR_LIT>": nsites,<EOL>"<STR_LIT>": "<STR_LIT:\t>".join(popkeys)}))<EOL>outstr = "<STR_LIT>"<EOL>for cidx in xrange(len(popkeys)):<EOL><INDENT>outstr += "<STR_LIT>"<EOL><DEDENT>toprint = []<EOL>for idx in xrange(len(loci)):<EOL><INDENT>dat = loci[idx].split("<STR_LIT:\n>")<EOL>seqs = np.array([list(i.split()[<NUM_LIT:1>]) for i in dat[:-<NUM_LIT:1>]])<EOL>names = [i.split()[<NUM_LIT:0>] for i in dat[:-<NUM_LIT:1>]]<EOL>data = np.zeros((seqs.shape[<NUM_LIT:1>], len(popkeys), <NUM_LIT:4>), dtype=np.uint16)<EOL>for sidx in xrange(seqs.shape[<NUM_LIT:1>]):<EOL><INDENT>for cidx in xrange(len(popkeys)):               <EOL><INDENT>for name in popdict[popkeys[cidx]]:<EOL><INDENT>if name in names:<EOL><INDENT>base = seqs[names.index(name), sidx]<EOL>if base in list("<STR_LIT>"):<EOL><INDENT>data[sidx, cidx, BASE2IDX[base]] += <NUM_LIT:2><EOL><DEDENT>elif base in list("<STR_LIT>"):<EOL><INDENT>base1, base2 = AMBIGS[base]<EOL>data[sidx, cidx, BASE2IDX[base1]] += <NUM_LIT:1><EOL>data[sidx, cidx, BASE2IDX[base2]] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>sdat = ["<STR_LIT:U+002C>".join([str(i) for i in i.tolist()]) for i in data[sidx]]<EOL>toprint.append(outstr.format(idx+<NUM_LIT:1>, sidx+<NUM_LIT:1>, *sdat))<EOL><DEDENT>if not idx % <NUM_LIT>:<EOL><INDENT>out.write("<STR_LIT:\n>".join(toprint)+"<STR_LIT:\n>")<EOL>toprint = []<EOL><DEDENT><DEDENT>out.write("<STR_LIT:\n>".join(toprint)+"<STR_LIT:\n>")<EOL>out.close()<EOL>

Convert ipyrad .loci file to an iqtree-pomo 'counts' file

Parameters:
-----------
name:
    A prefix name for output files that will be produced
locifile:
    A .loci file produced by ipyrad.
popdict:
    A python dictionary grouping Clade names to Sample names.
    Example: {"A": ['a', 'b', 'c'], "B": ['d', 'e', 'f']}
ipyclient:
    If you pass it an ipyclient it will distribute work over
    remote engines, otherwise we use multiprocessing (todo).


<EOL>ploidy = data.paramsdict["<STR_LIT>"]<EOL>names = [i.name for i in samples]<EOL>longname = max([len(i) for i in names])<EOL>inloci = open(os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>" ), '<STR_LIT:r>').read()<EOL>snpsout = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>usnpsout = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>structout = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>genoout = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>ugenoout = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>statsfile= os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>formats = data.paramsdict["<STR_LIT>"]<EOL>seed = data._hackersonly["<STR_LIT>"]<EOL>np.random.seed(int(seed))<EOL>S = {}      <EOL>Si = {}     <EOL>for name in list(names):<EOL><INDENT>S[name] = []<EOL>Si[name] = []<EOL><DEDENT>bis = <NUM_LIT:0><EOL>for loc in inloci.strip().split("<STR_LIT:|>")[:-<NUM_LIT:1>]:<EOL><INDENT>pis = "<STR_LIT>"<EOL>ns = []<EOL>ss = []<EOL>cov = {}  <EOL>for line in loc.split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT:>>" in line:<EOL><INDENT>ns.append(line.split()[<NUM_LIT:0>].replace("<STR_LIT:>>","<STR_LIT>"))<EOL>ss.append(line.split()[-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>pis = [i[<NUM_LIT:0>] for i in enumerate(line) if i[<NUM_LIT:1>] in list('<STR_LIT>')]<EOL><DEDENT><DEDENT>for tax in S:<EOL><INDENT>if tax in ns:<EOL><INDENT>if pis:<EOL><INDENT>for snpsite in pis:<EOL><INDENT>snpsite -= (longname+<NUM_LIT:5>)<EOL>S[tax].append(ss[ns.index(tax)][snpsite])<EOL>if snpsite not in cov:<EOL><INDENT>cov[snpsite] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>cov[snpsite] += <NUM_LIT:1><EOL><DEDENT>if ss[ns.index(tax)][snpsite] != '<STR_LIT:->':<EOL><INDENT>cov[snpsite] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if pis:<EOL><INDENT>for snpsite in pis:<EOL><INDENT>S[tax].append("<STR_LIT:N>")<EOL><DEDENT>Si[tax].append("<STR_LIT:N>")<EOL><DEDENT><DEDENT><DEDENT>maxlist = []<EOL>for j,k in cov.items():<EOL><INDENT>if k == max(cov.values()):<EOL><INDENT>maxlist.append(j)<EOL><DEDENT><DEDENT>bisnps = []<EOL>for i in maxlist:<EOL><INDENT>if len(set([ss[ns.index(tax)][i] for tax in S if tax in ns])) < <NUM_LIT:3>:<EOL><INDENT>bisnps.append(i)<EOL><DEDENT><DEDENT>if bisnps:<EOL><INDENT>rando = bisnps[np.random.randint(len(bisnps))]<EOL><DEDENT>elif maxlist:<EOL><INDENT>rando = maxlist[np.random.randint(len(maxlist))]<EOL><DEDENT>tbi = <NUM_LIT:0><EOL>for tax in S:<EOL><INDENT>if tax in ns:<EOL><INDENT>if pis:<EOL><INDENT>if not bisnps:<EOL><INDENT>tbi += <NUM_LIT:1><EOL><DEDENT>Si[tax].append(ss[ns.index(tax)][rando])<EOL><DEDENT><DEDENT>if pis:<EOL><INDENT>S[tax].append("<STR_LIT:U+0020>")<EOL><DEDENT>else:<EOL><INDENT>S[tax].append("<STR_LIT>")<EOL><DEDENT><DEDENT>bis += tbi<EOL><DEDENT>SF = list(S.keys())<EOL>SF.sort()<EOL>if "<STR_LIT>" in formats:<EOL><INDENT>with open(snpsout, '<STR_LIT:w>') as outfile:<EOL><INDENT>print >>outfile, "<STR_LIT>" %(len(S), len("<STR_LIT>".join(S.values()[<NUM_LIT:0>]).split("<STR_LIT:U+0020>"))-<NUM_LIT:1>,<EOL>len("<STR_LIT>".join(S[SF[<NUM_LIT:0>]]).replace("<STR_LIT:U+0020>", "<STR_LIT>")))<EOL>for i in SF:<EOL><INDENT>print >>outfile, i+("<STR_LIT:U+0020>"*(longname-len(i)+<NUM_LIT:3>))+"<STR_LIT>".join(S[i])<EOL><DEDENT><DEDENT><DEDENT>if "<STR_LIT>" in formats:<EOL><INDENT>with open(usnpsout, '<STR_LIT:w>') as outfile:<EOL><INDENT>print >>outfile, len(Si), len("<STR_LIT>".join(Si.values()[<NUM_LIT:0>]))<EOL>for i in SF:<EOL><INDENT>print >>outfile, i+("<STR_LIT:U+0020>"*(longname-len(i)+<NUM_LIT:3>))+"<STR_LIT>".join(Si[i])<EOL><DEDENT><DEDENT>with open(statsfile, '<STR_LIT:a>') as statsout:<EOL><INDENT>print >>statsout, "<STR_LIT>",len(Si.values()[<NUM_LIT:0>])<EOL>print >>statsout, "<STR_LIT>",len(Si.values()[<NUM_LIT:0>])-bis<EOL><DEDENT><DEDENT>if "<STR_LIT:str>" in formats:<EOL><INDENT>with open(structout, '<STR_LIT:w>') as outfile:<EOL><INDENT>B = {'<STR_LIT:A>': '<STR_LIT:0>',<EOL>'<STR_LIT:T>': '<STR_LIT:1>',<EOL>'<STR_LIT>': '<STR_LIT:2>',<EOL>'<STR_LIT:C>': '<STR_LIT:3>',<EOL>'<STR_LIT:N>': '<STR_LIT>',<EOL>'<STR_LIT:->': '<STR_LIT>'}<EOL>if ploidy > <NUM_LIT:1>:<EOL><INDENT>for line in SF:<EOL><INDENT>print >>outfile, line+("<STR_LIT:U+0020>"*(longname-len(line)+<NUM_LIT:3>))+"<STR_LIT:\t>"*<NUM_LIT:6>+"<STR_LIT:\t>".join([B[unstruct(j)[<NUM_LIT:0>]] for j in Si[line]])<EOL>print >>outfile, line+("<STR_LIT:U+0020>"*(longname-len(line)+<NUM_LIT:3>))+"<STR_LIT:\t>"*<NUM_LIT:6>+"<STR_LIT:\t>".join([B[unstruct(j)[<NUM_LIT:1>]] for j in Si[line]])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for line in SF:<EOL><INDENT>print >>outfile, line+("<STR_LIT:U+0020>"*(longname-len(line)+<NUM_LIT:3>))+"<STR_LIT:\t>"*<NUM_LIT:6>+"<STR_LIT:\t>".join([B[unstruct(j)[<NUM_LIT:1>]] for j in Si[line]])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if "<STR_LIT>" in formats:<EOL><INDENT>with open(ugenoout, '<STR_LIT:w>') as outfile:<EOL><INDENT>for i in range(len(Si.values()[<NUM_LIT:0>])):<EOL><INDENT>getref = <NUM_LIT:0><EOL>ref = "<STR_LIT:N>"<EOL>while ref == "<STR_LIT:N>":<EOL><INDENT>ref = unstruct(Si[SF[getref]][i])[<NUM_LIT:0>]<EOL>getref += <NUM_LIT:1><EOL><DEDENT>SNProw = "<STR_LIT>".join(map(str,[unstruct(Si[j][i]).count(ref) if Si[j][i] != "<STR_LIT:N>"else "<STR_LIT>" for j in SF]))<EOL>if len(set(SNProw)) > <NUM_LIT:1>:<EOL><INDENT>print >>outfile, SNProw <EOL><DEDENT><DEDENT><DEDENT>with open(genoout, '<STR_LIT:w>') as outfile:<EOL><INDENT>for i in range(len(S.values()[<NUM_LIT:0>])):<EOL><INDENT>if S[SF[<NUM_LIT:0>]][i].strip("<STR_LIT:_>").strip():<EOL><INDENT>getref = <NUM_LIT:0><EOL>ref = "<STR_LIT:N>"<EOL>while ref == "<STR_LIT:N>":<EOL><INDENT>ref = unstruct(S[SF[getref]][i])[<NUM_LIT:0>]<EOL>getref += <NUM_LIT:1><EOL>SNProw = "<STR_LIT>".join(map(str,[unstruct(S[j][i]).count(ref) ifS[j][i] != "<STR_LIT:N>" else "<STR_LIT>" for j in SF]))<EOL><DEDENT>if len(set(SNProw)) > <NUM_LIT:1>:<EOL><INDENT>print >>outfile, SNProw<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>


data = iter(open(os.path.join(assembly.dirs.outfiles,<EOL>assembly.name+"<STR_LIT>"), '<STR_LIT:r>'))<EOL>ntax, nchar = data.next().strip().split()<EOL>for line in data:<EOL><INDENT>tax, seq = line.strip().split()<EOL>idict[tax] = idict[tax][<NUM_LIT>:]<EOL>idict[tax] += seq[count:count+<NUM_LIT>]<EOL><DEDENT>del line<EOL>return idict<EOL>

updates dictionary with the next .5M reads from the super long string
    phylip file. Makes for faster reading.


<EOL>names = [i.name for i in samples]<EOL>names.sort()<EOL>locifile = os.path.join(data.dirs.outfiles, data.name+"<STR_LIT>")<EOL>locus = iter(open(locifile, '<STR_LIT:rb>'))<EOL>fdict = {name:[] for name in names}<EOL>partitions = []<EOL>initial_pos = <NUM_LIT:1><EOL>done = <NUM_LIT:0><EOL>nloci = <NUM_LIT:0><EOL>nbases = <NUM_LIT:0><EOL>while nloci < <NUM_LIT>: <EOL><INDENT>seqs = []<EOL>anames = []<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>samp = next(locus)<EOL><DEDENT>except StopIteration:<EOL><INDENT>done = <NUM_LIT:1><EOL>break<EOL><DEDENT>if "<STR_LIT>" in samp:<EOL><INDENT>nloci += <NUM_LIT:1><EOL>break<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>name, seq = samp.split()<EOL><DEDENT>except ValueError:<EOL><INDENT>print(samp)<EOL><DEDENT>anames.append(name[<NUM_LIT:1>:])<EOL>seqs.append(seq.strip())<EOL><DEDENT><DEDENT>arrayed = np.array([list(i) for i in seqs])<EOL>if done:<EOL><INDENT>break<EOL><DEDENT>mask = [i for i in arrayed.T if any([j not in list("<STR_LIT>") for j in i])]<EOL>masked = np.dstack(mask)[<NUM_LIT:0>]<EOL>loc_name = "<STR_LIT:p>"+str(nloci)<EOL>loc_range = str(initial_pos) + "<STR_LIT:->" +str(len(masked[<NUM_LIT:0>]) + initial_pos -<NUM_LIT:1>)<EOL>initial_pos += len(masked[<NUM_LIT:0>])<EOL>partitions.append(loc_name+"<STR_LIT:=>"+loc_range)<EOL>for name in names:<EOL><INDENT>if name in anames:<EOL><INDENT>fdict[name].append(masked[anames.index(name),:].tostring())<EOL><DEDENT>else:<EOL><INDENT>fdict[name].append("<STR_LIT:N>"*masked.shape[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>nbases += len(fdict[name][-<NUM_LIT:1>])<EOL>if not nloci % <NUM_LIT>:<EOL><INDENT>for name in fdict:<EOL><INDENT>with open(os.path.join(assembly.dirs.outfiles , "<STR_LIT>", <EOL>"<STR_LIT>".format(name, nloci)), '<STR_LIT:wb>') as wout:<EOL><INDENT>wout.write("<STR_LIT>".join(fdict[name]))<EOL><DEDENT><DEDENT>del fdict<EOL>fdict = {name:[] for name in names}<EOL><DEDENT><DEDENT>superout = open(os.path.join( assembly.dirs.outfiles, assembly.name+"<STR_LIT>" ), '<STR_LIT:wb>')<EOL>print(len(names), nbases, file=superout)<EOL>if nloci < <NUM_LIT>:<EOL><INDENT>for name in names:<EOL><INDENT>print(name+("<STR_LIT:U+0020>"*((longname+<NUM_LIT:3>)-len(name)))+"<STR_LIT>".join(fdict[name]), file=superout)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for name in names:<EOL><INDENT>superout.write("<STR_LIT>".format(<EOL>name,<EOL>"<STR_LIT:U+0020>"*((longname+<NUM_LIT:3>)-len(name)),<EOL>"<STR_LIT>".join(fdict[name])))<EOL>tmpfiles = glob.glob(os.path.join(assembly.dirs.outfiles, "<STR_LIT>", name+"<STR_LIT>"))<EOL>tmpfiles.sort()<EOL>for tmpf in tmpfiles:<EOL><INDENT>with open(tmpf, '<STR_LIT:rb>') as tmpin:<EOL><INDENT>superout.write(tmpin.read())<EOL><DEDENT>os.remove(tmpf)<EOL><DEDENT>superout.write("<STR_LIT:\n>")<EOL><DEDENT><DEDENT>superout.close()<EOL>raxml_part_out = open(os.path.join(assembly.dirs.outfiles, assembly.name+"<STR_LIT>"), '<STR_LIT:w>')<EOL>for partition in partitions:<EOL><INDENT>print("<STR_LIT>" % (partition), file=raxml_part_out)<EOL><DEDENT>raxml_part_out.close()<EOL>return partitions<EOL>

builds phy output. If large files writes 50000 loci at a time to tmp
    files and rebuilds at the end


def makenex(assembly, names, longname, partitions):

<EOL>data = iter(open(os.path.join(assembly.dirs.outfiles, assembly.name+"<STR_LIT>" ), '<STR_LIT:r>' ))<EOL>nexout = open(os.path.join(assembly.dirs.outfiles, assembly.name+"<STR_LIT>" ), '<STR_LIT:wb>' )<EOL>ntax, nchar = data.next().strip().split("<STR_LIT:U+0020>")<EOL>print("<STR_LIT>", file=nexout)<EOL>print("<STR_LIT>", file=nexout)<EOL>print("<STR_LIT>" % (ntax,nchar), file=nexout)<EOL>print("<STR_LIT>", file=nexout)<EOL>print("<STR_LIT>", file=nexout)<EOL>idict = {}<EOL>for line in data:<EOL><INDENT>tax, seq = line.strip().split()<EOL>idict[tax] = seq[<NUM_LIT:0>:<NUM_LIT>]<EOL><DEDENT>del line<EOL>nameorder = list(idict.keys())<EOL>nameorder.sort()<EOL>n=<NUM_LIT:0><EOL>tempn=<NUM_LIT:0><EOL>sz = <NUM_LIT:100><EOL>while n < len(seq):<EOL><INDENT>for tax in nameorder:<EOL><INDENT>print("<STR_LIT:U+0020>"+tax+"<STR_LIT:U+0020>"*((longname-len(tax))+<NUM_LIT:3>)+idict[tax][tempn:tempn+sz], file=nexout)<EOL><DEDENT>n += sz<EOL>tempn += sz<EOL>print("<STR_LIT>", file=nexout)<EOL>if not n % <NUM_LIT>:<EOL><INDENT>idict = update(assembly, idict, n)<EOL>tempn -= <NUM_LIT><EOL><DEDENT><DEDENT>print('<STR_LIT:;>', file=nexout)<EOL>print('<STR_LIT>', file=nexout)<EOL>print("<STR_LIT>", file=nexout)<EOL>for partition in partitions:<EOL><INDENT>print("<STR_LIT>" % (partition), file=nexout)<EOL><DEDENT>print("<STR_LIT>", file=nexout)<EOL>nexout.close()<EOL>


<EOL>longname = max([len(i) for i in list(assembly.samples.keys())])<EOL>names = [i.name for i in samples]<EOL>partitions = makephy(assembly, samples, longname)<EOL>makenex(assembly, names, longname, partitions)<EOL>

Make phylip and nexus formats. This is hackish since I'm recycling the
    code whole-hog from pyrad V3. Probably could be good to go back through
    and clean up the conversion code some time.


<EOL>if outdir:<EOL><INDENT>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>outdir = os.path.curdir<EOL><DEDENT>outdir = os.path.realpath(outdir)<EOL>outdir = os.path.join(outdir, "<STR_LIT>".format(name))<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>else:<EOL><INDENT>ofiles = glob.glob(os.path.join(outdir, "<STR_LIT>"))<EOL>for ofile in ofiles:<EOL><INDENT>os.remove(ofile)<EOL><DEDENT><DEDENT>with open(locifile) as infile:<EOL><INDENT>loci = (i for i in infile.read().strip().split("<STR_LIT>"))<EOL><DEDENT>if not subsamples:<EOL><INDENT>with open(locifile) as infile:<EOL><INDENT>subs = set((i.split()[<NUM_LIT:0>] for i in infile.readlines() if "<STR_LIT>" not in i))<EOL><DEDENT><DEDENT>else:   <EOL><INDENT>subs = set(subsamples)<EOL><DEDENT>lens = len(subs)<EOL>nlocus = <NUM_LIT:0><EOL>for loc in loci:<EOL><INDENT>dat = loc.split("<STR_LIT:\n>")[:-<NUM_LIT:1>]<EOL>names = [i.split()[<NUM_LIT:0>] for i in dat]<EOL>seqs = np.array([list(i.split()[<NUM_LIT:1>]) for i in dat])<EOL>if len(set(names).intersection(set(subs))) == lens:<EOL><INDENT>seqsamp = seqs[[names.index(tax) for tax in subs]]<EOL>seqsamp = _resolveambig(seqsamp)<EOL>pis = _count_PIS(seqsamp, minSNPs)<EOL>if pis:<EOL><INDENT>nlocus += <NUM_LIT:1><EOL>copied = seqsamp.copy()<EOL>copied[copied == "<STR_LIT:->"] = "<STR_LIT:N>"<EOL>rmcol = np.all(copied == "<STR_LIT:N>", axis=<NUM_LIT:0>)<EOL>seqsamp = seqsamp[:, ~rmcol]<EOL>mdict = dict(list(zip(subs, [i.tostring() for i in seqsamp])))<EOL>nexmake(mdict, nlocus, outdir, mcmc_burnin, mcmc_ngen, mcmc_sample_freq)<EOL><DEDENT><DEDENT><DEDENT>print("<STR_LIT>".format(nlocus, outdir))<EOL>

Converts loci file format to multiple nexus formatted files, one for
each locus, and writes a mrbayes block in the nexus information. The
mrbayes block will be set to run 2 replicate chains, for [mcmc_ngen]
generations, skipping [burnin] steps, and sampling every
[mcmc_sample_freq] steps.

Parameters:
-----------
name: (str)
    A prefix name for output files that will be produced
locifile: (str)
    A .loci file produced by ipyrad.
maxloci: (int)
    Limit the number of loci to the first N loci with sufficient sampling
    to be included in the analysis.
minSNPs: (int)
    Only include loci that have at least N parsimony informative SNPs.
seed: (int)
    Random seed used for resolving ambiguities.
burnin: (int)
    mrbayes nexus block burnin parameter used for 'sump burnin' and 'sumt burnin'.
    The number of generations to skip before starting parameter and tree sampling.
mcmc_ngen: (int)
    mrbayes nexus block 'mcmc ngen' and 'mcmc printfreq' parameters. We don't really
    to have any info printed to screen, so these values are set equal. This is the
    length of the chains that will be run.
mcmc_sample_freq: (int)
    mrbayes nexus block 'mcmc samplefreq' parameter. The frequency of sampling from
    the mcmc chain.


def nexmake(mdict, nlocus, dirs, mcmc_burnin, mcmc_ngen, mcmc_sample_freq):

<EOL>max_name_len = max([len(i) for i in mdict])<EOL>namestring = "<STR_LIT>" + str(max_name_len+<NUM_LIT:1>) + "<STR_LIT>"<EOL>matrix = "<STR_LIT>"<EOL>for i in list(mdict.items()):<EOL><INDENT>matrix += namestring.format(i[<NUM_LIT:0>], i[<NUM_LIT:1>])<EOL><DEDENT>handle = os.path.join(dirs, "<STR_LIT>".format(nlocus))<EOL>with open(handle, '<STR_LIT:w>') as outnex:<EOL><INDENT>outnex.write(NEXBLOCK.format(**{<EOL>"<STR_LIT>": len(mdict), <EOL>"<STR_LIT>": len(list(mdict.values())[<NUM_LIT:0>]), <EOL>"<STR_LIT>": matrix,<EOL>"<STR_LIT>": mcmc_ngen, <EOL>"<STR_LIT>": mcmc_sample_freq, <EOL>"<STR_LIT>": mcmc_burnin, <EOL>}))<EOL><DEDENT>

function that takes a dictionary mapping names to
sequences, and a locus number, and writes it as a NEXUS
file with a mrbayes analysis block.


N = []<EOL>for col in subseq:<EOL><INDENT>rand = np.random.binomial(<NUM_LIT:1>, <NUM_LIT:0.5>)<EOL>N.append([_AMBIGS[i][rand] for i in col])<EOL><DEDENT>return np.array(N)<EOL>

Randomly resolves iupac hetero codes. This is a shortcut
for now, we could instead use the phased alleles in RAD loci.


counts = [Counter(col) for col in seqsamp.T if not ("<STR_LIT:->" in col or "<STR_LIT:N>" in col)]<EOL>pis = [i.most_common(<NUM_LIT:2>)[<NUM_LIT:1>][<NUM_LIT:1>] > <NUM_LIT:1> for i in counts if len(i.most_common(<NUM_LIT:2>))><NUM_LIT:1>]<EOL>if sum(pis) >= N:<EOL><INDENT>return sum(pis)<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


proc = subprocess.Popen(<EOL>command_list,<EOL>stderr=subprocess.STDOUT, <EOL>stdout=subprocess.PIPE<EOL>)<EOL>comm = proc.communicate()<EOL>return comm<EOL>


<EOL>cmd = [self.params.binary, <EOL>"<STR_LIT>", OPJ(self.workdir, self.name+"<STR_LIT>"),<EOL>"<STR_LIT>", OPJ(self.workdir, self.name),<EOL>]<EOL>args = []<EOL>for key, val in self.params:<EOL><INDENT>if key not in ["<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>if key == "<STR_LIT:g>":<EOL><INDENT>if val[<NUM_LIT:0>]:<EOL><INDENT>args += ["<STR_LIT:->"+key, str(val[<NUM_LIT:0>]), str(val[<NUM_LIT:1>])]<EOL><DEDENT><DEDENT>elif key == "<STR_LIT>":<EOL><INDENT>if val:<EOL><INDENT>args += ["<STR_LIT:->"+key[:-<NUM_LIT:1>]]<EOL><DEDENT><DEDENT>elif key in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]: <EOL><INDENT>if val:<EOL><INDENT>args += ["<STR_LIT:->"+key]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if val:<EOL><INDENT>args += ["<STR_LIT:->"+key, str(val)]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return cmd+args<EOL>


return "<STR_LIT:U+0020>".join(self._command_list)<EOL>


spans = self.maparr<EOL>samp = np.zeros(spans.shape[<NUM_LIT:0>], dtype=np.uint64)<EOL>for i in xrange(spans.shape[<NUM_LIT:0>]):<EOL><INDENT>samp[i] = np.random.randint(spans[i, <NUM_LIT:0>], spans[i, <NUM_LIT:1>], <NUM_LIT:1>)<EOL><DEDENT>return samp<EOL>

returns a subsample of unlinked snp sites


<EOL>subdict = {i:j for i, j in self.__dict__.iteritems() if i != "<STR_LIT>"}<EOL>newdict = copy.deepcopy(subdict)<EOL>newobj = Treemix(<EOL>data=newdict["<STR_LIT:data>"],<EOL>name=name,<EOL>workdir=newdict["<STR_LIT>"],<EOL>imap={i:j for i, j in newdict["<STR_LIT>"].items()},<EOL>mapfile=newdict['<STR_LIT>'],<EOL>minmap={i:j for i, j in newdict["<STR_LIT>"].items()},<EOL>seed=np.random.randint(<NUM_LIT:0>, int(<NUM_LIT>)),<EOL>)<EOL>for key, val in newobj.params.__dict__.iteritems():<EOL><INDENT>newobj.params.__setattr__(key, self.params.__getattribute__(key))<EOL><DEDENT>return newobj<EOL>

Returns a copy of the treemix object with the same parameter settings
but with the files attributes cleared, and with a new 'name' attribute.

Parameters
----------
name (str):
    A name for the new copied treemix bject that will be used for the
    output files created by the object.


<EOL>tre = toytree.tree(newick=self.results.tree)<EOL>tre.draw(<EOL>axes=axes,<EOL>use_edge_lengths=True,<EOL>tree_style='<STR_LIT:c>',<EOL>tip_labels_align=True,<EOL>edge_align_style={"<STR_LIT>": <NUM_LIT:1>}<EOL>);<EOL>for admix in self.results.admixture:<EOL><INDENT>pidx, pdist, cidx, cdist, weight = admix<EOL>a = _get_admix_point(tre, pidx, pdist)<EOL>b = _get_admix_point(tre, cidx, cdist)<EOL>mark = axes.plot(<EOL>a = (a[<NUM_LIT:0>], b[<NUM_LIT:0>]),<EOL>b = (a[<NUM_LIT:1>], b[<NUM_LIT:1>]),<EOL>style={"<STR_LIT>": <NUM_LIT:10>*weight,<EOL>"<STR_LIT>": <NUM_LIT>, <EOL>"<STR_LIT>": "<STR_LIT>"}<EOL>)<EOL>axes.scatterplot(<EOL>a = (b[<NUM_LIT:0>]),<EOL>b = (b[<NUM_LIT:1>]),<EOL>size=<NUM_LIT:8>,<EOL>title="<STR_LIT>".format(weight),<EOL>)<EOL><DEDENT>axes.y.show=False<EOL>axes.x.ticks.show=True<EOL>axes.x.label.text = "<STR_LIT>"<EOL>return axes<EOL>

Returns a treemix plot on a toyplot.axes object.


<EOL>handle = baba.data<EOL>taxdicts = baba.tests<EOL>mindicts = baba.params.mincov<EOL>nboots = baba.params.nboots<EOL>sims = <NUM_LIT:0><EOL>if isinstance(handle, types.GeneratorType):<EOL><INDENT>handle = list(handle)<EOL>sims = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>handle = os.path.realpath(handle)<EOL><DEDENT>names = []<EOL>if isinstance(taxdicts, dict):<EOL><INDENT>taxdicts = [taxdicts]<EOL><DEDENT>tot = len(taxdicts)<EOL>resarr = np.zeros((tot, <NUM_LIT:7>), dtype=np.float64)<EOL>bootsarr = np.zeros((tot, nboots), dtype=np.float64)<EOL>paneldict = {}<EOL>if not ipyclient:<EOL><INDENT>raise IPyradError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>lbview = ipyclient.load_balanced_view()<EOL><DEDENT>start = time.time()<EOL>asyncs = {}<EOL>idx = <NUM_LIT:0><EOL>if isinstance(handle, str):<EOL><INDENT>with open(handle, '<STR_LIT:r>') as infile:<EOL><INDENT>loci = infile.read().strip().split("<STR_LIT>")<EOL><DEDENT><DEDENT>if isinstance(handle, list):<EOL><INDENT>pass <EOL><DEDENT>itests = iter(taxdicts)<EOL>imdict = itertools.cycle([mindicts])<EOL>for i in xrange(len(ipyclient)):<EOL><INDENT>try:<EOL><INDENT>test = next(itests)<EOL>mindict = next(imdict)<EOL><DEDENT>except StopIteration:<EOL><INDENT>continue<EOL><DEDENT>if sims:<EOL><INDENT>loci = _msp_to_arr(handle, test)<EOL>args = (loci, test, mindict, nboots)<EOL>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>args = [loci, test, mindict, nboots]<EOL>asyncs[idx] = lbview.apply(dstat, *args)<EOL><DEDENT>idx += <NUM_LIT:1><EOL><DEDENT>finished = <NUM_LIT:0><EOL>try:<EOL><INDENT>while <NUM_LIT:1>:<EOL><INDENT>keys = [i for (i, j) in asyncs.items() if j.ready()]<EOL>for job in keys:<EOL><INDENT>if not asyncs[job].successful():<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>".format(job, asyncs[job].exception()))<EOL><DEDENT>else:<EOL><INDENT>_res, _bot = asyncs[job].result()<EOL>if _res.shape[<NUM_LIT:0>] == <NUM_LIT:1>:<EOL><INDENT>resarr[job] = _res.T.as_matrix()[:, <NUM_LIT:0>]<EOL>bootsarr[job] = _bot<EOL><DEDENT>else:   <EOL><INDENT>paneldict[job] = _res.T<EOL><DEDENT>del asyncs[job]<EOL>finished += <NUM_LIT:1><EOL>try:<EOL><INDENT>test = next(itests)<EOL>mindict = next(imdict)<EOL>if sims:<EOL><INDENT>loci = _msp_to_arr(handle, test)<EOL>args = (loci, test, mindict, nboots)<EOL>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>args = [loci, test, mindict, nboots]<EOL>asyncs[idx] = lbview.apply(dstat, *args)<EOL><DEDENT>idx += <NUM_LIT:1><EOL><DEDENT>except StopIteration:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>elap = datetime.timedelta(seconds=int(time.time()-start))<EOL>printstr = "<STR_LIT>"<EOL>progressbar(tot, finished, printstr.format(elap), spacer="<STR_LIT>")<EOL>time.sleep(<NUM_LIT:0.1>)<EOL>if not asyncs:<EOL><INDENT>print("<STR_LIT>")<EOL>break<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt as inst:<EOL><INDENT>try:<EOL><INDENT>ipyclient.abort()<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>raise inst<EOL><DEDENT>if len(test) == <NUM_LIT:4>:<EOL><INDENT>if not names:<EOL><INDENT>names = range(len(taxdicts))<EOL><DEDENT>resarr = pd.DataFrame(resarr, <EOL>index=names,<EOL>columns=["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"])<EOL>resarr = resarr.sort_index()<EOL>order = [list(resarr.index).index(i) for i in names]<EOL>bootsarr = bootsarr[order]<EOL>return resarr, bootsarr<EOL><DEDENT>else:<EOL><INDENT>listres = []<EOL>for key in range(len(paneldict)):<EOL><INDENT>listres.append(paneldict[key])<EOL><DEDENT>ntests = len(paneldict)<EOL>multi_index = [<EOL>np.array([[i] * <NUM_LIT:3> for i in range(ntests)]).flatten(),<EOL>np.array(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'] * ntests),<EOL>]<EOL>resarr = pd.DataFrame(<EOL>data=pd.concat(listres).as_matrix(), <EOL>index=multi_index,<EOL>columns=listres[<NUM_LIT:0>].columns,<EOL>)<EOL>return resarr, None<EOL><DEDENT>


def dstat(inarr, taxdict, mindict=<NUM_LIT:1>, nboots=<NUM_LIT:1000>, name=<NUM_LIT:0>):

<EOL>if isinstance(inarr, list):<EOL><INDENT>arr, _ = _loci_to_arr(inarr, taxdict, mindict)<EOL><DEDENT>if arr.shape[<NUM_LIT:1>] == <NUM_LIT:4>:<EOL><INDENT>res, boots = _get_signif_4(arr, nboots)<EOL>res = pd.DataFrame(res, <EOL>columns=[name],<EOL>index=["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>res, boots = _get_signif_5(arr, nboots)<EOL>res = pd.DataFrame(res,<EOL>index=["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"], <EOL>columns=["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>)<EOL><DEDENT>return res.T, boots<EOL>

private function to perform a single D-stat test


def _loci_to_arr(loci, taxdict, mindict):

<EOL>nloci = len(loci)<EOL>maxlen = np.max(np.array([len(locus.split("<STR_LIT:\n>")[<NUM_LIT:0>]) for locus in loci]))<EOL>keep = np.zeros(nloci, dtype=np.bool_)<EOL>arr = np.zeros((nloci, <NUM_LIT:4>, maxlen), dtype=np.float64)<EOL>if len(taxdict) == <NUM_LIT:5>:<EOL><INDENT>arr = np.zeros((nloci, <NUM_LIT:6>, maxlen), dtype=np.float64)<EOL><DEDENT>if isinstance(mindict, int):<EOL><INDENT>mindict = {i: mindict for i in taxdict}<EOL><DEDENT>elif isinstance(mindict, dict):<EOL><INDENT>mindict = {i: mindict[i] for i in taxdict}<EOL><DEDENT>else:<EOL><INDENT>mindict = {i: <NUM_LIT:1> for i in taxdict}<EOL><DEDENT>allowed_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if any([i not in allowed_names for i in taxdict]):<EOL><INDENT>raise IPyradError("<STR_LIT>")<EOL><DEDENT>keys = sorted([i for i in taxdict.keys() if i[<NUM_LIT:0>] == '<STR_LIT:p>'])<EOL>outg = keys[-<NUM_LIT:1>]<EOL>for loc in xrange(nloci):<EOL><INDENT>lines = loci[loc].split("<STR_LIT:\n>")[:-<NUM_LIT:1>]<EOL>names = [i.split()[<NUM_LIT:0>] for i in lines]<EOL>seqs = np.array([list(i.split()[<NUM_LIT:1>]) for i in lines])<EOL>covs = [sum([j in names for j in taxdict[tax]]) >= mindict[tax]for tax in taxdict]<EOL>if all(covs):<EOL><INDENT>keep[loc] = True<EOL>refidx = np.where([i in taxdict[outg] for i in names])[<NUM_LIT:0>]<EOL>refseq = seqs[refidx].view(np.uint8)<EOL>ancestral = np.array([reftrick(refseq, GETCONS2)[:, <NUM_LIT:0>]])<EOL>iseq = _reffreq2(ancestral, refseq, GETCONS2)<EOL>arr[loc, -<NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq <EOL>if len(taxdict) == <NUM_LIT:4>:<EOL><INDENT>for tidx, key in enumerate(keys[:-<NUM_LIT:1>]):<EOL><INDENT>nidx = np.where([i in taxdict[key] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, tidx, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>iseq = _reffreq2(ancestral, refseq, GETCONS2) <EOL>arr[loc, -<NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq <EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:0>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>nidy = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>sidy = seqs[nidy].view(np.uint8)<EOL>xseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>yseq = _reffreq2(ancestral, sidy, GETCONS2)<EOL>mask3 = xseq != <NUM_LIT:0><EOL>mask4 = yseq != <NUM_LIT:0><EOL>xseq[mask4] = <NUM_LIT:0><EOL>yseq[mask3] = <NUM_LIT:0><EOL>arr[loc, <NUM_LIT:2>, :xseq.shape[<NUM_LIT:1>]] = xseq<EOL>arr[loc, <NUM_LIT:3>, :yseq.shape[<NUM_LIT:1>]] = yseq<EOL>nidx = nidx.tolist() + nidy.tolist()<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:4>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL><DEDENT><DEDENT><DEDENT>arr = arr[keep, :, :]<EOL>arr = masknulls(arr)<EOL>return arr, keep<EOL>

return a frequency array from a loci file for all loci with taxa from
taxdict and min coverage from mindict.


<EOL>boots = np.zeros((nboots,))<EOL>for bidx in xrange(nboots):<EOL><INDENT>lidx = np.random.randint(<NUM_LIT:0>, arr.shape[<NUM_LIT:0>], arr.shape[<NUM_LIT:0>])<EOL>tarr = arr[lidx]<EOL>_, _, dst = _prop_dstat(tarr)<EOL>boots[bidx] = dst<EOL><DEDENT>return boots<EOL>


abba, baba, dst = _prop_dstat(arr)<EOL>boots = _get_boots(arr, nboots)<EOL>estimate, stddev = (boots.mean(), boots.std())<EOL>zscore = <NUM_LIT:0.><EOL>if stddev:<EOL><INDENT>zscore = np.abs(dst) / stddev<EOL><DEDENT>stats = [dst, estimate, stddev, zscore, abba, baba, arr.shape[<NUM_LIT:0>]]<EOL>return np.array(stats), boots<EOL>

returns a list of stats and an array of dstat boots. Stats includes
z-score and two-sided P-value.


statsarr = np.zeros((<NUM_LIT:3>, <NUM_LIT:7>), dtype=np.float64)<EOL>bootsarr = np.zeros((<NUM_LIT:3>, nboots))<EOL>idx = <NUM_LIT:0><EOL>for acol in [<NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:4>]:<EOL><INDENT>rows = np.array([<NUM_LIT:0>, <NUM_LIT:1>, acol, <NUM_LIT:5>])<EOL>tarr = arr[:, rows, :]<EOL>abxa, baxa, dst = _prop_dstat(tarr)<EOL>boots = _get_boots(tarr, nboots)<EOL>estimate, stddev = (boots.mean(), boots.std())<EOL>if stddev:<EOL><INDENT>zscore = np.abs(dst) / stddev<EOL><DEDENT>else:<EOL><INDENT>zscore = np.NaN<EOL><DEDENT>stats = [dst, estimate, stddev, zscore, abxa, baxa, arr.shape[<NUM_LIT:0>]]<EOL>statsarr[idx] = stats<EOL>bootsarr[idx] = boots<EOL>idx += <NUM_LIT:1><EOL><DEDENT>return statsarr, bootsarr<EOL>


def _simulate(self, nreps, admix=None, Ns=<NUM_LIT>, gen=<NUM_LIT:20>):

<EOL>Taus = np.array(list(set(self.verts[:, <NUM_LIT:1>]))) * <NUM_LIT> * gen<EOL>ntips = len(self.tree)<EOL>names = {name: idx for idx, name in enumerate(self.tree.get_leaf_names()[::-<NUM_LIT:1>])}<EOL>pop_config = [<EOL>ms.PopulationConfiguration(sample_size=<NUM_LIT:2>, initial_size=Ns)<EOL>for i in range(ntips)<EOL>]<EOL>migmat = np.zeros((ntips, ntips)).tolist()<EOL>demog = []<EOL>coals = sorted(list(set(self.verts[:, <NUM_LIT:1>])))[<NUM_LIT:1>:]<EOL>for ct in xrange(len(coals)):<EOL><INDENT>nidxs = np.where(self.verts[:, <NUM_LIT:1>] == coals[ct])[<NUM_LIT:0>]<EOL>time = Taus[ct+<NUM_LIT:1>]<EOL>for nidx in nidxs:<EOL><INDENT>node = self.tree.search_nodes(name=str(nidx))[<NUM_LIT:0>]<EOL>dest = sorted(node.get_leaves(), key=lambda x: x.idx)[<NUM_LIT:0>]<EOL>otherchild = [i for i in node.children if not <EOL>i.get_leaves_by_name(dest.name)][<NUM_LIT:0>]<EOL>if otherchild.is_leaf():<EOL><INDENT>source = otherchild<EOL><DEDENT>else:<EOL><INDENT>source = sorted(otherchild.get_leaves(), key=lambda x: x.idx)[<NUM_LIT:0>]<EOL><DEDENT>event = ms.MassMigration(<EOL>time=int(time),<EOL>source=names[source.name], <EOL>destination=names[dest.name], <EOL>proportion=<NUM_LIT:1.0>)<EOL>demog.append(event)<EOL><DEDENT><DEDENT>replicates = ms.simulate(<EOL>population_configurations=pop_config,<EOL>migration_matrix=migmat,<EOL>demographic_events=demog,<EOL>num_replicates=nreps,<EOL>length=<NUM_LIT:100>, <EOL>mutation_rate=<NUM_LIT>)<EOL>return replicates<EOL>

Enter a baba.Tree object in which the 'tree' attribute (newick
derived tree) has edge lengths in units of generations. You can
use the 'gen' parameter to multiply branch lengths by a constant.

Parameters:
-----------

nreps: (int)
    Number of reps (loci) to simulate under the demographic scenario
tree: (baba.Tree object)
    A baba.Tree object initialized by calling baba.Tree(*args).
admix: (list)
    A list of admixture events to occur on the tree. Nodes must be
    reference by their index number, and events must occur in time
    intervals when edges exist. Use the .draw() function of the
    baba.Tree object to see node index numbers and coalescent times.
Ns: (float)
    Fixed effective population size for all lineages (may allow to vary
    in the future).
gen: (int)
    A multiplier applied to branch lengths to scale into units of
    generations. Example, if all edges on a tree were 1 then you might
    enter 50000 to multiply so that edges are 50K generations long.


if self.tests:<EOL><INDENT>keys = sorted(self.tests[<NUM_LIT:0>].keys())<EOL>if isinstance(self.tests, list):<EOL><INDENT>ld = [[(key, i[key]) for key in keys] for i in self.tests]<EOL>dd = [dict(i) for i in ld]<EOL>df = pd.DataFrame(dd)<EOL>return df<EOL><DEDENT>else:<EOL><INDENT>return pd.DataFrame(pd.Series(self.tests)).T<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Returns the .tests list of taxa as a pandas dataframe.
By auto-generating this table from tests it means that
the table itself cannot be modified unless it is returned
and saved.


self.results_table, self.results_boots = batch(self, ipyclient)<EOL>if not isinstance(self.results_table, list):<EOL><INDENT>self.results_table.nloci = np.nan_to_num(self.results_table.nloci).astype(int)<EOL><DEDENT>

Run a batch of dstat tests on a list of tests, where each test is
a dictionary mapping sample names to {p1 - p4} (and sometimes p5).
Parameters modifying the behavior of the run, such as the number
of bootstrap replicates (nboots) or the minimum coverage for
loci (mincov) can be set in {object}.params.

Parameters:
-----------
ipyclient (ipyparallel.Client object):
    An ipyparallel client object to distribute jobs to a cluster.


return copy.deepcopy(self)<EOL>

returns a copy of the baba analysis object


<EOL>outg = taxdicts[<NUM_LIT:0>]["<STR_LIT>"]<EOL>if not all([i["<STR_LIT>"] == outg for i in taxdicts]):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>tre = ete.Tree(tree.write(format=<NUM_LIT:1>)) <EOL>alltax = [i for i in tre.get_leaf_names() if i not in outg]<EOL>alltax += [outg[<NUM_LIT:0>]]<EOL>tre.prune(alltax)<EOL>tre.search_nodes(name=outg[<NUM_LIT:0>])[<NUM_LIT:0>].name = "<STR_LIT>"<EOL>tre.ladderize()<EOL>taxd = copy.deepcopy(taxdicts)<EOL>newtaxdicts = []<EOL>for test in taxd:<EOL><INDENT>test["<STR_LIT>"] = ["<STR_LIT>"]<EOL>newtaxdicts.append(test)<EOL><DEDENT>return tre, newtaxdicts<EOL>

collapse outgroup in ete Tree for easier viewing


def _decompose_tree(ttree, orient='<STR_LIT:right>', use_edge_lengths=True):

<EOL>ttree._orient = orient<EOL>ttree._use_edge_lengths = use_edge_lengths<EOL>ult = use_edge_lengths == False<EOL>names = {}<EOL>idx = <NUM_LIT:0><EOL>for node in ttree.tree.traverse("<STR_LIT>"):<EOL><INDENT>if not node.is_leaf():<EOL><INDENT>if node.name:<EOL><INDENT>names[idx] = node.name<EOL><DEDENT>else:<EOL><INDENT>names[idx] = idx<EOL>node.name = str(idx)<EOL><DEDENT>node.idx = idx<EOL>idx += <NUM_LIT:1><EOL><DEDENT><DEDENT>for node in ttree.tree.get_leaves(): <EOL><INDENT>names[idx] = node.name<EOL>node.idx = idx<EOL>idx += <NUM_LIT:1><EOL><DEDENT>ttree.node_labels = names<EOL>ttree.tip_labels = ttree.tree.get_leaf_names()<EOL>ttree.edges = np.zeros((idx - <NUM_LIT:1>, <NUM_LIT:2>), dtype=int)<EOL>ttree.verts = np.zeros((idx, <NUM_LIT:2>), dtype=float)<EOL>ttree._lines = []        <EOL>ttree._coords = []       <EOL>nidx = <NUM_LIT:0><EOL>tip_num = len(ttree.tree.get_leaves()) - <NUM_LIT:1><EOL>for node in ttree.tree.traverse("<STR_LIT>"):<EOL><INDENT>if node.is_leaf():<EOL><INDENT>node.y = ttree.tree.get_distance(node)<EOL>if ult:<EOL><INDENT>node.y = <NUM_LIT:0.> <EOL><DEDENT>node.x = tip_num<EOL>tip_num -= <NUM_LIT:1><EOL>ttree.verts[node.idx] = [node.x, node.y]<EOL>ttree.edges[nidx] = [node.up.idx, node.idx]<EOL><DEDENT>elif node.is_root():<EOL><INDENT>node.y = ttree.tree.get_distance(node)<EOL>if ult:<EOL><INDENT>node.y = -<NUM_LIT:1> * node.get_farthest_leaf(True)[<NUM_LIT:1>] - <NUM_LIT:1><EOL><DEDENT>node.x = sum(i.x for i in node.children) / float(len(node.children))<EOL>ttree.verts[node.idx] = [node.x, node.y]<EOL><DEDENT>else:<EOL><INDENT>node.y = ttree.tree.get_distance(node)<EOL>if ult:<EOL><INDENT>node.y = -<NUM_LIT:1> * node.get_farthest_leaf(True)[<NUM_LIT:1>] - <NUM_LIT:1><EOL><DEDENT>node.x = sum(i.x for i in node.children) / float(len(node.children))<EOL>ttree.edges[nidx, :] = [node.up.idx, node.idx]<EOL>ttree.verts[node.idx] = [node.x, node.y]<EOL><DEDENT>nidx += <NUM_LIT:1><EOL><DEDENT>cidx = <NUM_LIT:0><EOL>for node in ttree.tree.traverse():<EOL><INDENT>if not node.is_leaf():<EOL><INDENT>ttree._coords += [[node.x, node.y]]<EOL>pidx = cidx<EOL>cidx += <NUM_LIT:1><EOL>for child in node.children:<EOL><INDENT>ttree._coords += [[child.x, node.y], [child.x, child.y]]<EOL>ttree._lines += [[pidx, cidx]]    <EOL>ttree._lines += [[cidx, cidx+<NUM_LIT:1>]]  <EOL>cidx += <NUM_LIT:2><EOL><DEDENT><DEDENT><DEDENT>ttree._coords = np.array(ttree._coords, dtype=float)<EOL>ttree._lines = np.array(ttree._lines, dtype=int)<EOL>if ttree._orient in ['<STR_LIT>', <NUM_LIT:0>]:<EOL><INDENT>pass<EOL><DEDENT>if ttree._orient in ['<STR_LIT:left>', <NUM_LIT:1>]:<EOL><INDENT>ttree.verts[:, <NUM_LIT:1>] = ttree.verts[:, <NUM_LIT:1>] * -<NUM_LIT:1><EOL>ttree.verts = ttree.verts[:, [<NUM_LIT:1>, <NUM_LIT:0>]]<EOL>ttree._coords[:, <NUM_LIT:1>] = ttree._coords[:, <NUM_LIT:1>] * -<NUM_LIT:1><EOL>ttree._coords = ttree._coords[:, [<NUM_LIT:1>, <NUM_LIT:0>]]<EOL><DEDENT>if ttree._orient in ['<STR_LIT>', <NUM_LIT:0>]:<EOL><INDENT>ttree.verts[:, <NUM_LIT:1>] = ttree.verts[:, <NUM_LIT:1>] * -<NUM_LIT:1><EOL>ttree._coords[:, <NUM_LIT:1>] = ttree._coords[:, <NUM_LIT:1>] * -<NUM_LIT:1><EOL><DEDENT>if ttree._orient in ['<STR_LIT:right>', <NUM_LIT:3>]:<EOL><INDENT>ttree.verts = ttree.verts[:, [<NUM_LIT:1>, <NUM_LIT:0>]]<EOL>ttree._coords = ttree._coords[:, [<NUM_LIT:1>, <NUM_LIT:0>]]<EOL><DEDENT>

decomposes tree into component parts for plotting


<EOL>self._decompose_tree(orient=orient, use_edge_lengths=use_edge_lengths)<EOL>dwargs = {}<EOL>dwargs["<STR_LIT>"] = show_tip_labels<EOL>dwargs["<STR_LIT>"] = show_node_support<EOL>dwargs.update(kwargs)<EOL>canvas, axes, panel = tree_panel_plot(self, print_args, **dwargs)<EOL>return canvas, axes, panel<EOL>

plot the tree using toyplot.graph.

Parameters:
-----------
    show_tip_labels: bool
        Show tip names from tree.
    use_edge_lengths: bool
        Use edge lengths from newick tree.
    show_node_support: bool
        Show support values at nodes using a set of default
        options.

    ...


<EOL>hosts = []<EOL>for eid in ipyclient.ids:<EOL><INDENT>engine = ipyclient[eid]<EOL>if not engine.outstanding:<EOL><INDENT>hosts.append(engine.apply(socket.gethostname))<EOL><DEDENT><DEDENT>hosts = [i.get() for i in hosts]<EOL>hostset = set(hosts)<EOL>hostzip = zip(hosts, ipyclient.ids)<EOL>hostdict = {host: [i[<NUM_LIT:1>] for i in hostzip if i[<NUM_LIT:0>] == host] for host in hostset}<EOL>targets = list(itertools.chain(*[hostdict[i][:<NUM_LIT:2>] for i in hostdict]))<EOL>return targets<EOL>

A function to find 2 engines per hostname on the ipyclient.
We'll assume that the CPUs are hyperthreaded, which is why
we grab two. If they are not then no foul. Two multi-threaded
jobs will be run on each of the 2 engines per host.


<EOL>names = self.samples<EOL>if self.params.nboots:<EOL><INDENT>fulltre = ete3.Tree(self.trees.tree, format=<NUM_LIT:0>)<EOL>fulltre.unroot()<EOL>with open(self.trees.boots, '<STR_LIT:r>') as inboots:<EOL><INDENT>bb = [ete3.Tree(i.strip(), format=<NUM_LIT:0>) for i in inboots.readlines()]<EOL>wboots = [fulltre] + bb[-self.params.nboots:]<EOL><DEDENT>wctre, wcounts = consensus_tree(wboots, names=names)<EOL>self.trees.cons = os.path.join(self.dirs, self.name + "<STR_LIT>")<EOL>with open(self.trees.cons, '<STR_LIT:w>') as ocons:<EOL><INDENT>ocons.write(wctre.write(format=<NUM_LIT:0>))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wctre = ete3.Tree(self.trees.tree, format=<NUM_LIT:0>)<EOL>wctre.unroot()<EOL><DEDENT>self.trees.nhx = os.path.join(self.dirs, self.name + "<STR_LIT>")<EOL>with open(self.files.stats, '<STR_LIT:w>') as ostats:<EOL><INDENT>if self.params.nboots:<EOL><INDENT>ostats.write("<STR_LIT>".format(len(wboots)))<EOL>for i, j in enumerate(self.samples):<EOL><INDENT>ostats.write("<STR_LIT>".format(i, j))<EOL><DEDENT>ostats.write("<STR_LIT:\n>")<EOL>for split, freq in wcounts:<EOL><INDENT>if split.count('<STR_LIT:1>') > <NUM_LIT:1>:<EOL><INDENT>ostats.write("<STR_LIT>".format(split, round(freq, <NUM_LIT:2>)))<EOL><DEDENT><DEDENT>ostats.write("<STR_LIT:\n>")<EOL><DEDENT><DEDENT>lbview = ipyclient.load_balanced_view()<EOL>qtots = {}<EOL>qsamp = {}<EOL>tots = sum(<NUM_LIT:1> for i in wctre.iter_leaves())<EOL>totn = set(wctre.get_leaf_names())<EOL>for node in wctre.traverse():<EOL><INDENT>qtots[node] = lbview.apply(_get_total, *(tots, node))<EOL>qsamp[node] = lbview.apply(_get_sampled, *(self, totn, node))<EOL><DEDENT>ipyclient.wait()<EOL>for node in wctre.traverse():<EOL><INDENT>total = qtots[node].result()<EOL>sampled = qsamp[node].result()<EOL>node.add_feature("<STR_LIT>", total)<EOL>node.add_feature("<STR_LIT>", sampled)<EOL><DEDENT>features = ["<STR_LIT>", "<STR_LIT>"]<EOL>with open(self.trees.nhx, '<STR_LIT:w>') as outtre:<EOL><INDENT>outtre.write(wctre.write(format=<NUM_LIT:0>, features=features))<EOL><DEDENT>

compute stats for stats file and NHX tree features


pool = tuple(iterable)<EOL>size = len(pool)<EOL>indices = random.sample(xrange(size), nquartets)<EOL>return tuple(pool[i] for i in indices)<EOL>

Random selection from itertools.combinations(iterable, r).
Use this if not sampling all possible quartets.


pool1 = tuple(iter1)<EOL>pool2 = tuple(iter2)<EOL>ind1 = random.sample(pool1, <NUM_LIT:2>)<EOL>ind2 = random.sample(pool2, <NUM_LIT:2>)<EOL>return tuple(ind1+ind2)<EOL>


return int(reduce(MUL, (Fraction(n-i, i+<NUM_LIT:1>) for i in range(k)), <NUM_LIT:1>))<EOL>

get the number of quartets as n-choose-k. This is used
    in equal splits to decide whether a split should be exhaustively sampled
    or randomly sampled. Edges near tips can be exhaustive while highly nested
    edges probably have too many quartets


<EOL>snps = np.zeros(<NUM_LIT:4>, dtype=np.uint32)<EOL>snps[<NUM_LIT:0>] = np.uint32(mat[<NUM_LIT:0>, <NUM_LIT:5>] + mat[<NUM_LIT:0>, <NUM_LIT:10>] + mat[<NUM_LIT:0>, <NUM_LIT:15>] +mat[<NUM_LIT:5>, <NUM_LIT:0>] + mat[<NUM_LIT:5>, <NUM_LIT:10>] + mat[<NUM_LIT:5>, <NUM_LIT:15>] +mat[<NUM_LIT:10>, <NUM_LIT:0>] + mat[<NUM_LIT:10>, <NUM_LIT:5>] + mat[<NUM_LIT:10>, <NUM_LIT:15>] +mat[<NUM_LIT:15>, <NUM_LIT:0>] + mat[<NUM_LIT:15>, <NUM_LIT:5>] + mat[<NUM_LIT:15>, <NUM_LIT:10>])<EOL>for i in range(<NUM_LIT:16>):<EOL><INDENT>if i % <NUM_LIT:5>:<EOL><INDENT>snps[<NUM_LIT:1>] += mat[i, i]<EOL><DEDENT><DEDENT>snps[<NUM_LIT:2>] = mat[<NUM_LIT:1>, <NUM_LIT:4>] + mat[<NUM_LIT:2>, <NUM_LIT:8>] + mat[<NUM_LIT:3>, <NUM_LIT:12>] +mat[<NUM_LIT:4>, <NUM_LIT:1>] + mat[<NUM_LIT:6>, <NUM_LIT:9>] + mat[<NUM_LIT:7>, <NUM_LIT>] +mat[<NUM_LIT:8>, <NUM_LIT:2>] + mat[<NUM_LIT:9>, <NUM_LIT:6>] + mat[<NUM_LIT:11>, <NUM_LIT>] +mat[<NUM_LIT:12>, <NUM_LIT:3>] + mat[<NUM_LIT>, <NUM_LIT:7>] + mat[<NUM_LIT>, <NUM_LIT:11>]<EOL>snps[<NUM_LIT:3>] = (mat.sum() - np.diag(mat).sum()) - snps[<NUM_LIT:2>]<EOL>return snps<EOL>

get dstats from the count array and return as a float tuple


@numba.jit('<STR_LIT>', nopython=True)<EOL>def subsample_snps_map(seqchunk, nmask, maparr):

<EOL>rmask = np.zeros(seqchunk.shape[<NUM_LIT:1>], dtype=np.bool_)<EOL>last_loc = -<NUM_LIT:1><EOL>for idx in xrange(maparr.shape[<NUM_LIT:0>]):<EOL><INDENT>if maparr[idx] != last_loc:<EOL><INDENT>if not nmask[idx]:<EOL><INDENT>rmask[idx] = True<EOL><DEDENT>last_loc = maparr[idx]<EOL><DEDENT><DEDENT>return rmask<EOL>

removes ncolumns from snparray prior to matrix calculation, and
subsamples 'linked' snps (those from the same RAD locus) such that
for these four samples only 1 SNP per locus is kept. This information
comes from the 'map' array (map file).


@numba.jit('<STR_LIT>', nopython=True)<EOL>def chunk_to_matrices(narr, mapcol, nmask):

<EOL>mats = np.zeros((<NUM_LIT:3>, <NUM_LIT:16>, <NUM_LIT:16>), dtype=np.uint32)<EOL>last_loc = -<NUM_LIT:1><EOL>for idx in xrange(mapcol.shape[<NUM_LIT:0>]):<EOL><INDENT>if not nmask[idx]:<EOL><INDENT>if not mapcol[idx] == last_loc:<EOL><INDENT>i = narr[:, idx]<EOL>mats[<NUM_LIT:0>, (<NUM_LIT:4>*i[<NUM_LIT:0>])+i[<NUM_LIT:1>], (<NUM_LIT:4>*i[<NUM_LIT:2>])+i[<NUM_LIT:3>]] += <NUM_LIT:1>      <EOL>last_loc = mapcol[idx]<EOL><DEDENT><DEDENT><DEDENT>x = np.uint8(<NUM_LIT:0>)<EOL>for y in np.array([<NUM_LIT:0>, <NUM_LIT:4>, <NUM_LIT:8>, <NUM_LIT:12>], dtype=np.uint8):<EOL><INDENT>for z in np.array([<NUM_LIT:0>, <NUM_LIT:4>, <NUM_LIT:8>, <NUM_LIT:12>], dtype=np.uint8):<EOL><INDENT>mats[<NUM_LIT:1>, y:y+np.uint8(<NUM_LIT:4>), z:z+np.uint8(<NUM_LIT:4>)] = mats[<NUM_LIT:0>, x].reshape(<NUM_LIT:4>, <NUM_LIT:4>)<EOL>mats[<NUM_LIT:2>, y:y+np.uint8(<NUM_LIT:4>), z:z+np.uint8(<NUM_LIT:4>)] = mats[<NUM_LIT:0>, x].reshape(<NUM_LIT:4>, <NUM_LIT:4>).T<EOL>x += np.uint8(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>return mats<EOL>

numba compiled code to get matrix fast.
arr is a 4 x N seq matrix converted to np.int8
I convert the numbers for ATGC into their respective index for the MAT
matrix, and leave all others as high numbers, i.e., -==45, N==78.


return os.path.realpath(path)<EOL>


@numba.jit(nopython=True)<EOL>def shuffle_cols(seqarr, newarr, cols):

for idx in xrange(cols.shape[<NUM_LIT:0>]):<EOL><INDENT>newarr[:, idx] = seqarr[:, cols[idx]]<EOL><DEDENT>return newarr<EOL>

used in bootstrap resampling without a map file


<EOL>for ambig in np.uint8([<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>]):<EOL><INDENT>idx, idy = np.where(tmpseq == ambig)<EOL>res1, res2 = AMBIGS[ambig.view("<STR_LIT>")]<EOL>halfmask = np.random.choice([True, False], idx.shape[<NUM_LIT:0>])<EOL>for i in xrange(halfmask.shape[<NUM_LIT:0>]):<EOL><INDENT>if halfmask[i]:<EOL><INDENT>tmpseq[idx[i], idy[i]] = np.array(res1).view(np.uint8)<EOL><DEDENT>else:<EOL><INDENT>tmpseq[idx[i], idy[i]] = np.array(res2).view(np.uint8)<EOL><DEDENT><DEDENT><DEDENT>return tmpseq<EOL>

returns a seq array with 'RSKYWM' randomly replaced with resolved bases


<EOL>bidx = <NUM_LIT:1><EOL>spans = np.zeros((maparr[-<NUM_LIT:1>, <NUM_LIT:0>], <NUM_LIT:2>), np.uint64)<EOL>for idx in xrange(<NUM_LIT:1>, maparr.shape[<NUM_LIT:0>]):<EOL><INDENT>cur = maparr[idx, <NUM_LIT:0>]<EOL>if cur != bidx:<EOL><INDENT>idy = idx + <NUM_LIT:1><EOL>spans[cur-<NUM_LIT:2>, <NUM_LIT:1>] = idx<EOL>spans[cur-<NUM_LIT:1>, <NUM_LIT:0>] = idx<EOL>bidx = cur<EOL><DEDENT><DEDENT>spans[-<NUM_LIT:1>, <NUM_LIT:1>] = maparr[-<NUM_LIT:1>, -<NUM_LIT:1>]<EOL>return spans<EOL>

get span distance for each locus in original seqarray


width = <NUM_LIT:0><EOL>for idx in xrange(loci.shape[<NUM_LIT:0>]):<EOL><INDENT>width += spans[loci[idx], <NUM_LIT:1>] - spans[loci[idx], <NUM_LIT:0>]<EOL><DEDENT>return width<EOL>

get shape of new bootstrap resampled locus array


@numba.jit(nopython=True)<EOL>def fill_boot(seqarr, newboot, newmap, spans, loci):

<EOL>cidx = <NUM_LIT:0><EOL>for i in xrange(loci.shape[<NUM_LIT:0>]):<EOL><INDENT>x1 = spans[loci[i]][<NUM_LIT:0>]<EOL>x2 = spans[loci[i]][<NUM_LIT:1>]<EOL>cols = seqarr[:, x1:x2]<EOL>cord = np.random.choice(cols.shape[<NUM_LIT:1>], cols.shape[<NUM_LIT:1>], replace=False)<EOL>rcols = cols[:, cord]<EOL>newboot[:, cidx:cidx+cols.shape[<NUM_LIT:1>]] = rcols<EOL>newmap[cidx: cidx+cols.shape[<NUM_LIT:1>], <NUM_LIT:0>] = i+<NUM_LIT:1><EOL>cidx += cols.shape[<NUM_LIT:1>]<EOL><DEDENT>return newboot, newmap<EOL>


if isinstance(data, unicode):<EOL><INDENT>return data.encode("<STR_LIT:utf-8>")<EOL><DEDENT>if isinstance(data, list):<EOL><INDENT>return [_byteify(item, ignore_dicts=True) for item in data]<EOL><DEDENT>if isinstance(data, dict) and not ignore_dicts:<EOL><INDENT>return {<EOL>_byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)<EOL>for key, value in data.iteritems()<EOL>}<EOL><DEDENT>return data<EOL>

converts unicode to utf-8 when reading in json files


def consensus_tree(trees, names=None, cutoff=<NUM_LIT:0.0>):

<EOL>namedict, clade_counts = _find_clades(trees, names=names)<EOL>fclade_counts = _filter_clades(clade_counts, cutoff)<EOL>consens_tree, _ = _build_trees(fclade_counts, namedict)<EOL>return consens_tree, clade_counts<EOL>

An extended majority rule consensus function for ete3.
Modelled on the similar function from scikit-bio tree module. If
cutoff=0.5 then it is a normal majority rule consensus, while if
cutoff=0.0 then subsequent non-conflicting clades are added to the tree.


<EOL>passed = []<EOL>clades = np.array([list(i[<NUM_LIT:0>]) for i in clade_counts], dtype=np.int8)<EOL>counts = np.array([i[<NUM_LIT:1>] for i in clade_counts], dtype=np.float64)<EOL>for idx in xrange(clades.shape[<NUM_LIT:0>]):<EOL><INDENT>conflict = False<EOL>if counts[idx] < cutoff:<EOL><INDENT>continue<EOL><DEDENT>if np.sum(clades[idx]) > <NUM_LIT:1>:<EOL><INDENT>for aidx in passed:<EOL><INDENT>summed = clades[idx] + clades[aidx]<EOL>intersect = np.max(summed) > <NUM_LIT:1><EOL>subset_test0 = np.all(clades[idx] - clades[aidx] >= <NUM_LIT:0>)<EOL>subset_test1 = np.all(clades[aidx] - clades[idx] >= <NUM_LIT:0>)<EOL>invert_test = np.bool_(clades[aidx]) != np.bool_(clades[idx])<EOL>if np.all(invert_test):<EOL><INDENT>counts[aidx] += counts[idx]<EOL>conflict = True<EOL><DEDENT>if intersect:<EOL><INDENT>if (not subset_test0) and (not subset_test1):<EOL><INDENT>conflict = True<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if conflict == False:<EOL><INDENT>passed.append(idx)<EOL><DEDENT><DEDENT>rclades = []<EOL>for idx in passed:<EOL><INDENT>rclades.append((clades[idx], counts[idx]))<EOL><DEDENT>return rclades<EOL>

A subfunc of consensus_tree(). Removes clades that occur
with freq < cutoff.


<EOL>if not names:<EOL><INDENT>names = trees[<NUM_LIT:0>].get_leaf_names()<EOL><DEDENT>ndict = {j:i for i, j in enumerate(names)}<EOL>namedict = {i:j for i, j in enumerate(names)}<EOL>clade_counts = defaultdict(int)<EOL>for tree in trees:<EOL><INDENT>tree.unroot()<EOL>for node in tree.traverse('<STR_LIT>'):<EOL><INDENT>bits = np.zeros(len(tree), dtype=np.bool_)<EOL>for child in node.iter_leaf_names():<EOL><INDENT>bits[ndict[child]] = True<EOL><DEDENT>bitstring = "<STR_LIT>".join([np.binary_repr(i) for i in bits])<EOL>clade_counts[bitstring] += <NUM_LIT:1><EOL><DEDENT><DEDENT>for key, val in clade_counts.items():<EOL><INDENT>clade_counts[key] = val / float(len(trees))<EOL><DEDENT>clade_counts = sorted(clade_counts.items(), <EOL>key=lambda x: x[<NUM_LIT:1>],<EOL>reverse=True)<EOL>return namedict, clade_counts<EOL>

A subfunc of consensus_tree(). Traverses trees to count clade occurrences.
Names are ordered by names, else they are in the order of the first
tree.


<EOL>nodes = {}<EOL>idxarr = np.arange(len(fclade_counts[<NUM_LIT:0>][<NUM_LIT:0>]))<EOL>queue = []<EOL>countdict = defaultdict(int)<EOL>for clade, count in fclade_counts:<EOL><INDENT>mask = np.int_(list(clade)).astype(np.bool)<EOL>ccx = idxarr[mask]<EOL>queue.append((len(ccx), frozenset(ccx)))<EOL>countdict[frozenset(ccx)] = count<EOL><DEDENT>while queue:<EOL><INDENT>queue.sort()<EOL>(clade_size, clade) = queue.pop(<NUM_LIT:0>)<EOL>new_queue = []<EOL>for (_, ancestor) in queue:<EOL><INDENT>if clade.issubset(ancestor):<EOL><INDENT>new_ancestor = (ancestor - clade) | frozenset([clade])          <EOL>countdict[new_ancestor] = countdict.pop(ancestor)<EOL>ancestor = new_ancestor<EOL><DEDENT>new_queue.append((len(ancestor), ancestor))<EOL><DEDENT>if clade_size == <NUM_LIT:1>:<EOL><INDENT>name = list(clade)[<NUM_LIT:0>]<EOL>name = namedict[name]<EOL><DEDENT>else:<EOL><INDENT>name = None <EOL><DEDENT>children = [nodes.pop(c) for c in clade if c in nodes]<EOL>node = ete3.Tree(name=name)    <EOL>for child in children:<EOL><INDENT>node.add_child(child)<EOL><DEDENT>if not node.is_leaf():<EOL><INDENT>node.dist = int(round(<NUM_LIT:100>*countdict[clade]))<EOL>node.support = int(round(<NUM_LIT:100>*countdict[clade]))<EOL><DEDENT>else:<EOL><INDENT>node.dist = int(<NUM_LIT:100>) <EOL>node.support = int(<NUM_LIT:100>)<EOL><DEDENT>nodes[clade] = node<EOL>queue = new_queue<EOL><DEDENT>tre = nodes.values()[<NUM_LIT:0>]<EOL>tre.unroot()<EOL>return tre, list(nodes.values())<EOL>

A subfunc of consensus_tree(). Build an unrooted consensus tree
from filtered clade counts.


if (node.is_leaf() or node.is_root()):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>if len(node.children) > <NUM_LIT:2>:<EOL><INDENT>down_r = node.children[<NUM_LIT:0>]<EOL>down_l = node.children[<NUM_LIT:1>]<EOL>for child in node.children[<NUM_LIT:2>:]:<EOL><INDENT>down_l += child<EOL><DEDENT><DEDENT>else:<EOL><INDENT>down_r, down_l = node.children<EOL><DEDENT>lendr = sum(<NUM_LIT:1> for i in down_r.iter_leaves())<EOL>lendl = sum(<NUM_LIT:1> for i in down_l.iter_leaves())<EOL>up_r = node.get_sisters()[<NUM_LIT:0>]<EOL>lenur = sum(<NUM_LIT:1> for i in up_r.iter_leaves())<EOL>lenul = tots - (lendr + lendl + lenur)<EOL>return lendr * lendl * lenur * lenul<EOL><DEDENT>

get total number of quartets possible for a split


<EOL>names = sorted(totn)<EOL>cdict = {name: idx for idx, name in enumerate(names)}<EOL>if (node.is_leaf() or node.is_root()):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>if len(node.children) > <NUM_LIT:2>:<EOL><INDENT>down_r = node.children[<NUM_LIT:0>]<EOL>down_l = node.children[<NUM_LIT:1>]<EOL>for child in node.children[<NUM_LIT:2>:]:<EOL><INDENT>down_l += child<EOL><DEDENT><DEDENT>else:<EOL><INDENT>down_r, down_l = node.children<EOL><DEDENT>lendr = set(cdict[i] for i in down_r.get_leaf_names())<EOL>lendl = set(cdict[i] for i in down_l.get_leaf_names())<EOL>up_r = node.get_sisters()[<NUM_LIT:0>]<EOL>lenur = set(cdict[i] for i in up_r.get_leaf_names())<EOL>lenul = set(cdict[i] for i in totn - set.union(lendr, lendl, lenur))<EOL><DEDENT>idx = <NUM_LIT:0><EOL>sampled = <NUM_LIT:0><EOL>with h5py.File(data.database.output, '<STR_LIT:r>') as io5:<EOL><INDENT>end = io5["<STR_LIT>"].shape[<NUM_LIT:0>]<EOL>while <NUM_LIT:1>:<EOL><INDENT>if idx >= end:<EOL><INDENT>break<EOL><DEDENT>qrts = io5["<STR_LIT>"][idx:idx+data._chunksize]<EOL>for qrt in qrts:<EOL><INDENT>sqrt = set(qrt)<EOL>if all([sqrt.intersection(i) for i in [lendr, lendl, lenur, lenul]]):<EOL><INDENT>sampled += <NUM_LIT:1><EOL><DEDENT><DEDENT>idx += data._chunksize<EOL><DEDENT><DEDENT>return sampled<EOL>

get total number of quartets sampled for a split


<EOL>oldfiles = [self.files.qdump] +self.database.__dict__.values() +self.trees.__dict__.values()<EOL>for oldfile in oldfiles:<EOL><INDENT>if oldfile:<EOL><INDENT>if os.path.exists(oldfile):<EOL><INDENT>os.remove(oldfile)<EOL><DEDENT><DEDENT><DEDENT>oldcluster = copy.deepcopy(self._ipcluster)<EOL>self.__init__(<EOL>name=self.name, <EOL>data=self.files.data, <EOL>mapfile=self.files.mapfile,<EOL>workdir=self.dirs,<EOL>method=self.params.method,<EOL>guidetreefile=self.files.guidetreefile,<EOL>resolve=self._resolve, <EOL>nboots=self.params.nboots, <EOL>nquartets=self.params.nquartets, <EOL>initarr=True, <EOL>quiet=True,<EOL>cli=self.kwargs.get("<STR_LIT>")<EOL>)<EOL>self._ipcluster = oldcluster<EOL>

Remove all existing results files and reinit the h5 arrays
so that the tetrad object is just like fresh from a CLI start.


self.samples = []<EOL>with iter(open(self.files.data, '<STR_LIT:r>')) as infile:<EOL><INDENT>infile.next().strip().split()<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>self.samples.append(infile.next().split()[<NUM_LIT:0>])<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>

parse sample names from the sequence file


<EOL>try:<EOL><INDENT>spath = open(self.files.data, '<STR_LIT:r>')<EOL><DEDENT>except IOError:<EOL><INDENT>raise IPyradWarningExit(NO_SNP_FILE.format(self.files.data))<EOL><DEDENT>line = spath.readline().strip().split()<EOL>ntax = int(line[<NUM_LIT:0>])<EOL>nbp = int(line[<NUM_LIT:1>])<EOL>if not quiet:<EOL><INDENT>print("<STR_LIT>".format(ntax, nbp))        <EOL><DEDENT>tmpseq = np.zeros((ntax, nbp), dtype=np.uint8)<EOL>with h5py.File(self.database.input, '<STR_LIT:w>') as io5:<EOL><INDENT>io5.create_dataset("<STR_LIT>", (ntax, nbp), dtype=np.uint8)<EOL>io5.create_dataset("<STR_LIT>", (ntax, nbp), dtype=np.uint8)<EOL>io5.create_dataset("<STR_LIT>", (nbp, <NUM_LIT:2>), dtype=np.uint32)<EOL>if self.files.mapfile:<EOL><INDENT>with open(self.files.mapfile, '<STR_LIT:r>') as inmap:<EOL><INDENT>maparr = np.genfromtxt(inmap, dtype=np.uint64)<EOL>io5["<STR_LIT>"][:] = maparr[:, [<NUM_LIT:0>, <NUM_LIT:3>]]<EOL>spans = np.zeros((maparr[-<NUM_LIT:1>, <NUM_LIT:0>], <NUM_LIT:2>), np.uint64)<EOL>spans = get_spans(maparr, spans)<EOL>io5.create_dataset("<STR_LIT>", data=spans)<EOL>if not quiet:<EOL><INDENT>print("<STR_LIT>".format(spans.shape[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>io5["<STR_LIT>"][:, <NUM_LIT:0>] = np.arange(io5["<STR_LIT>"].shape[<NUM_LIT:0>])<EOL><DEDENT>for line, seq in enumerate(spath.readlines()):<EOL><INDENT>tmpseq[line] = np.array(list(seq.split()[-<NUM_LIT:1>])).view(np.uint8)<EOL><DEDENT>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT><EOL>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT>            <EOL>io5["<STR_LIT>"][:] = tmpseq<EOL>if self._resolve:<EOL><INDENT>tmpseq = resolve_ambigs(tmpseq)<EOL><DEDENT>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT:0><EOL>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT:1><EOL>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT:2><EOL>tmpseq[tmpseq == <NUM_LIT>] = <NUM_LIT:3><EOL>io5["<STR_LIT>"][:] = tmpseq<EOL>LOGGER.info("<STR_LIT>", io5["<STR_LIT>"][:, :<NUM_LIT:20>])<EOL>LOGGER.info("<STR_LIT>", io5["<STR_LIT>"][:, :<NUM_LIT:20>])<EOL>LOGGER.info("<STR_LIT>", io5["<STR_LIT>"][:<NUM_LIT:20>, :])<EOL><DEDENT>

Fills the seqarr with the full data set, and creates a bootsarr copy
with the following modifications:

1) converts "-" into "N"s, since they are similarly treated as missing.
2) randomly resolve ambiguities (RSKWYM)
3) convert to uint8 for smaller memory load and faster computation


breaks = <NUM_LIT:2><EOL>if self.params.nquartets < <NUM_LIT>:<EOL><INDENT>breaks = <NUM_LIT:1><EOL><DEDENT>if self.params.nquartets > <NUM_LIT>:<EOL><INDENT>breaks = <NUM_LIT:4><EOL><DEDENT>if self.params.nquartets > <NUM_LIT>:<EOL><INDENT>breaks = <NUM_LIT:8><EOL><DEDENT>self._chunksize = (self.params.nquartets // (breaks * ncpus) +(self.params.nquartets % (breaks * ncpus)))<EOL>LOGGER.info("<STR_LIT>", self.params.nquartets, self._chunksize)<EOL>with h5py.File(self.database.output, '<STR_LIT:w>') as io5:<EOL><INDENT>io5.create_dataset("<STR_LIT>", <EOL>(self.params.nquartets, <NUM_LIT:4>), <EOL>dtype=np.uint16, <EOL>chunks=(self._chunksize, <NUM_LIT:4>))<EOL>io5.create_dataset("<STR_LIT>", <EOL>(self.params.nquartets, <NUM_LIT:4>), <EOL>dtype=np.uint32, <EOL>chunks=(self._chunksize, <NUM_LIT:4>))<EOL>io5.create_group("<STR_LIT>")<EOL><DEDENT>with h5py.File(self.database.input, '<STR_LIT:a>') as io5:<EOL><INDENT>io5.create_dataset("<STR_LIT>", <EOL>(self.params.nquartets, <NUM_LIT:4>), <EOL>dtype=np.uint16, <EOL>chunks=(self._chunksize, <NUM_LIT:4>),<EOL>compression='<STR_LIT>')<EOL>qiter = itertools.combinations(xrange(len(self.samples)), <NUM_LIT:4>)<EOL>i = <NUM_LIT:0><EOL>while i < self.params.nquartets:<EOL><INDENT>if self.params.method != "<STR_LIT:all>":<EOL><INDENT>qiter = []<EOL>while len(qiter) < min(self._chunksize, io5["<STR_LIT>"].shape[<NUM_LIT:0>]):<EOL><INDENT>qiter.append(<EOL>random_combination(range(len(self.samples)), <NUM_LIT:4>))<EOL><DEDENT>dat = np.array(qiter)<EOL><DEDENT>else:<EOL><INDENT>dat = np.array(list(itertools.islice(qiter, self._chunksize)))<EOL><DEDENT>io5["<STR_LIT>"][i:i+self._chunksize] = dat[:io5["<STR_LIT>"].shape[<NUM_LIT:0>] - i]<EOL>i += self._chunksize<EOL><DEDENT><DEDENT>

Find all quartets of samples and store in a large array
Create a chunk size for sampling from the array of quartets.
This should be relatively large so that we don't spend a lot of time
doing I/O, but small enough that jobs finish often for checkpointing.