vdwradii_dat = os.path.join(outdir, "<STR_LIT>")<EOL>if libdir is not None:<EOL><INDENT>filename = os.path.join(libdir, '<STR_LIT>')  <EOL>if not os.path.exists(filename):<EOL><INDENT>msg = '<STR_LIT>'.format(**vars())<EOL>logger.exception(msg)<EOL>raise OSError(msg, errno.ENOENT)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>filename = os.path.join(os.environ['<STR_LIT>'], '<STR_LIT>')<EOL><DEDENT>except KeyError:<EOL><INDENT>try:<EOL><INDENT>filename = os.path.join(os.environ['<STR_LIT>'], '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except KeyError:<EOL><INDENT>msg = "<STR_LIT>"<EOL>logger.exception(msg)<EOL>raise OSError(msg, errno.ENOENT)<EOL><DEDENT><DEDENT><DEDENT>if not os.path.exists(filename):<EOL><INDENT>msg = "<STR_LIT>".format(**vars())<EOL>logger.exception(msg, errno.ENOENT)<EOL>raise OSError(msg)<EOL><DEDENT>patterns = vdw_lipid_resnames + list({x[:<NUM_LIT:3>] for x in vdw_lipid_resnames})<EOL>with open(vdwradii_dat, '<STR_LIT:w>') as outfile:<EOL><INDENT>outfile.write('<STR_LIT>')<EOL>for resname in patterns:<EOL><INDENT>for atom,radius in vdw_lipid_atom_radii.items():<EOL><INDENT>outfile.write('<STR_LIT>'.format(**vars()))<EOL><DEDENT><DEDENT>with open(filename, '<STR_LIT:r>') as infile:<EOL><INDENT>for line in infile:<EOL><INDENT>outfile.write(line)<EOL><DEDENT><DEDENT><DEDENT>logger.debug('<STR_LIT>'.format(**vars()))<EOL>return realpath(vdwradii_dat)<EOL>

Find vdwradii.dat and add special entries for lipids.

    See :data:`gromacs.setup.vdw_lipid_resnames` for lipid
    resnames. Add more if necessary.


def solvate(struct='<STR_LIT>', top='<STR_LIT>',<EOL>distance=<NUM_LIT>, boxtype='<STR_LIT>',<EOL>concentration=<NUM_LIT:0>, cation='<STR_LIT>', anion='<STR_LIT>',<EOL>water='<STR_LIT>', solvent_name='<STR_LIT>', with_membrane=False,<EOL>ndx = '<STR_LIT>', mainselection = '<STR_LIT>',<EOL>dirname='<STR_LIT>',<EOL>**kwargs):

sol = solvate_sol(struct=struct, top=top,<EOL>distance=distance, boxtype=boxtype,<EOL>water=water, solvent_name=solvent_name, <EOL>with_membrane=with_membrane,<EOL>dirname=dirname, **kwargs)<EOL>ion = solvate_ion(struct=sol['<STR_LIT>'], top=top,<EOL>concentration=concentration, cation=cation, anion=anion,<EOL>solvent_name=solvent_name, ndx=ndx,<EOL>mainselection=mainselection, dirname=dirname,<EOL>**kwargs)<EOL>return ion<EOL>

Put protein into box, add water, add counter-ions.

    Currently this really only supports solutes in water. If you need
    to embedd a protein in a membrane then you will require more
    sophisticated approaches.

    However, you *can* supply a protein already inserted in a
    bilayer. In this case you will probably want to set *distance* =
    ``None`` and also enable *with_membrane* = ``True`` (using extra
    big vdw radii for typical lipids).

    .. Note:: The defaults are suitable for solvating a globular
       protein in a fairly tight (increase *distance*!) dodecahedral
       box.

    :Arguments:
      *struct* : filename
          pdb or gro input structure
      *top* : filename
          Gromacs topology
      *distance* : float
          When solvating with water, make the box big enough so that
          at least *distance* nm water are between the solute *struct*
          and the box boundary.
          Set *boxtype*  to ``None`` in order to use a box size in the input
          file (gro or pdb).
      *boxtype* or *bt*: string
          Any of the box types supported by :class:`~gromacs.tools.Editconf`
          (triclinic, cubic, dodecahedron, octahedron). Set the box dimensions
          either with *distance* or the *box* and *angle* keywords.

          If set to ``None`` it will ignore *distance* and use the box
          inside the *struct* file.

          *bt* overrides the value of *boxtype*.
      *box*
          List of three box lengths [A,B,C] that are used by :class:`~gromacs.tools.Editconf`
          in combination with *boxtype* (``bt`` in :program:`editconf`) and *angles*.
          Setting *box* overrides *distance*.
      *angles*
          List of three angles (only necessary for triclinic boxes).
      *concentration* : float
          Concentration of the free ions in mol/l. Note that counter
          ions are added in excess of this concentration.
      *cation* and *anion* : string
          Molecule names of the ions. This depends on the chosen force field.
      *water* : string
          Name of the water model; one of "spc", "spce", "tip3p",
          "tip4p". This should be appropriate for the chosen force
          field. If an alternative solvent is required, simply supply the path to a box
          with solvent molecules (used by :func:`~gromacs.genbox`'s  *cs* argument)
          and also supply the molecule name via *solvent_name*.
      *solvent_name*
          Name of the molecules that make up the solvent (as set in the itp/top).
          Typically needs to be changed when using non-standard/non-water solvents.
          ["SOL"]
      *with_membrane* : bool
           ``True``: use special ``vdwradii.dat`` with 0.1 nm-increased radii on
           lipids. Default is ``False``.
      *ndx* : filename
          How to name the index file that is produced by this function.
      *mainselection* : string
          A string that is fed to :class:`~gromacs.tools.Make_ndx` and
          which should select the solute.
      *dirname* : directory name
          Name of the directory in which all files for the solvation stage are stored.
      *includes*
          List of additional directories to add to the mdp include path
      *kwargs*
          Additional arguments are passed on to
          :class:`~gromacs.tools.Editconf` or are interpreted as parameters to be
          changed in the mdp file.


if len(d) > <NUM_LIT:0>:<EOL><INDENT>wmsg = "<STR_LIT>"+str(d)<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=UsageWarning)<EOL><DEDENT>return len(d) == <NUM_LIT:0><EOL>

Check if any arguments remain in dict *d*.


structure = realpath(struct)<EOL>topology = realpath(top)<EOL>mdp_template = config.get_template(mdp)<EOL>deffnm = deffnm.strip()<EOL>mdrun_args = {} if mdrun_args is None else mdrun_args<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.pop('<STR_LIT>', None)<EOL>mainselection = kwargs.pop('<STR_LIT>', '<STR_LIT>')<EOL>qtot = kwargs.pop('<STR_LIT>', <NUM_LIT:0>)<EOL>mdp = deffnm+'<STR_LIT>'<EOL>tpr = deffnm+'<STR_LIT>'<EOL>logger.info("<STR_LIT>".format(**vars()))<EOL>cbook.add_mdp_includes(topology, kwargs)<EOL>if qtot != <NUM_LIT:0>:<EOL><INDENT>wmsg = "<STR_LIT>".format(**vars())<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=BadParameterWarning)<EOL><DEDENT>with in_dir(dirname):<EOL><INDENT>unprocessed = cbook.edit_mdp(mdp_template, new_mdp=mdp, **kwargs)<EOL>check_mdpargs(unprocessed)<EOL>gromacs.grompp(f=mdp, o=tpr, c=structure, r=structure, p=topology, **unprocessed)<EOL>mdrun_args.update(v=True, stepout=<NUM_LIT:10>, deffnm=deffnm, c=output)<EOL>if mdrunner is None:<EOL><INDENT>mdrun = run.get_double_or_single_prec_mdrun()<EOL>mdrun(**mdrun_args)<EOL><DEDENT>else:<EOL><INDENT>if type(mdrunner) is type:<EOL><INDENT>mdrun = mdrunner(**mdrun_args)<EOL>mdrun.run()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>mdrunner.run(mdrunargs=mdrun_args)<EOL><DEDENT>except AttributeError:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>raise TypeError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>if not os.path.exists(output):<EOL><INDENT>errmsg = "<STR_LIT>"<EOL>logger.error(errmsg)<EOL>raise GromacsError(errmsg)<EOL><DEDENT>final_struct = realpath(output)<EOL><DEDENT>logger.info("<STR_LIT>".format(**vars()))<EOL>return {'<STR_LIT>': final_struct,<EOL>'<STR_LIT>': topology,<EOL>'<STR_LIT>': mainselection,<EOL>}<EOL>

Energy minimize the system.

    This sets up the system (creates run input files) and also runs
    ``mdrun_d``. Thus it can take a while.

    Additional itp files should be in the same directory as the top file.

    Many of the keyword arguments below already have sensible values.

    :Keywords:
       *dirname*
          set up under directory dirname [em]
       *struct*
          input structure (gro, pdb, ...) [solvate/ionized.gro]
       *output*
          output structure (will be put under dirname) [em.pdb]
       *deffnm*
          default name for mdrun-related files [em]
       *top*
          topology file [top/system.top]
       *mdp*
          mdp file (or use the template) [templates/em.mdp]
       *includes*
          additional directories to search for itp files
       *mdrunner*
          :class:`gromacs.run.MDrunner` instance; by default we
          just try :func:`gromacs.mdrun_d` and :func:`gromacs.mdrun` but a
          MDrunner instance gives the user the ability to run mpi jobs
          etc. [None]
       *mdrun_args*
          arguments for *mdrunner* (as a dict), e.g. ``{'nt': 2}``;
          empty by default

          .. versionaddedd:: 0.7.0

       *kwargs*
          remaining key/value pairs that should be changed in the
          template mdp file, eg ``nstxtcout=250, nstfout=250``.

    .. note:: If :func:`~gromacs.mdrun_d` is not found, the function
              falls back to :func:`~gromacs.mdrun` instead.


mdrunner = kwargs.pop('<STR_LIT>', None)<EOL>integrators = kwargs.pop('<STR_LIT>', ['<STR_LIT>', '<STR_LIT>'])<EOL>kwargs.pop('<STR_LIT>', None)  <EOL>nsteps = kwargs.pop('<STR_LIT>', [<NUM_LIT:100>, <NUM_LIT:1000>])<EOL>outputs = ['<STR_LIT>'.format(i, integrator) for i,integrator in enumerate(integrators)]<EOL>outputs[-<NUM_LIT:1>] = kwargs.pop('<STR_LIT>', '<STR_LIT>')<EOL>files = {'<STR_LIT>': kwargs.pop('<STR_LIT>', None)}  <EOL>for i, integrator in enumerate(integrators):<EOL><INDENT>struct = files['<STR_LIT>']<EOL>logger.info("<STR_LIT>", i, integrator, nsteps[i])<EOL>kwargs.update({'<STR_LIT>':struct, '<STR_LIT>':outputs[i],<EOL>'<STR_LIT>':integrator, '<STR_LIT>': nsteps[i]})<EOL>if not integrator == '<STR_LIT>':<EOL><INDENT>kwargs['<STR_LIT>'] = mdrunner<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = None<EOL>logger.warning("<STR_LIT>"<EOL>"<STR_LIT>", i)<EOL><DEDENT>files = energy_minimize(**kwargs)<EOL><DEDENT>return files<EOL>

Run multiple energy minimizations one after each other.

    :Keywords:
      *integrators*
           list of integrators (from 'l-bfgs', 'cg', 'steep')
           [['bfgs', 'steep']]
      *nsteps*
           list of maximum number of steps; one for each integrator in
           in the *integrators* list [[100,1000]]
      *kwargs*
           mostly passed to :func:`gromacs.setup.energy_minimize`

    :Returns: dictionary with paths to final structure ('struct') and
              other files

    :Example:
       Conduct three minimizations:
         1. low memory Broyden-Goldfarb-Fletcher-Shannon (BFGS) for 30 steps
         2. steepest descent for 200 steps
         3. finish with BFGS for another 30 steps
       We also do a multi-processor minimization when possible (i.e. for steep
       (and conjugate gradient) by using a :class:`gromacs.run.MDrunner` class
       for a :program:`mdrun` executable compiled for OpenMP in 64 bit (see
       :mod:`gromacs.run` for details)::

          import gromacs.run
          gromacs.setup.em_schedule(struct='solvate/ionized.gro',
                    mdrunner=gromacs.run.MDrunnerOpenMP64,
                    integrators=['l-bfgs', 'steep', 'l-bfgs'],
                    nsteps=[50,200, 50])

    .. Note:: You might have to prepare the mdp file carefully because at the
              moment one can only modify the *nsteps* parameter on a
              per-minimizer basis.


def _setup_MD(dirname,<EOL>deffnm='<STR_LIT>', mdp=config.templates['<STR_LIT>'],<EOL>struct=None,<EOL>top='<STR_LIT>', ndx=None,<EOL>mainselection='<STR_LIT>',<EOL>qscript=config.qscript_template, qname=None, startdir=None, mdrun_opts="<STR_LIT>", budget=None, walltime=<NUM_LIT:1>/<NUM_LIT>,<EOL>dt=<NUM_LIT>, runtime=<NUM_LIT>, **mdp_kwargs):

if struct is None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>structure = realpath(struct)<EOL>topology = realpath(top)<EOL>try:<EOL><INDENT>index = realpath(ndx)<EOL><DEDENT>except AttributeError:  <EOL><INDENT>index = None        <EOL><DEDENT>qname = mdp_kwargs.pop('<STR_LIT>', qname)    <EOL>qscript = mdp_kwargs.pop('<STR_LIT>', qscript)    <EOL>qscript_template = config.get_template(qscript)<EOL>mdp_template = config.get_template(mdp)<EOL>nsteps = int(float(runtime)/float(dt))<EOL>mdp = deffnm + '<STR_LIT>'<EOL>tpr = deffnm + '<STR_LIT>'<EOL>mainindex = deffnm + '<STR_LIT>'<EOL>final_structure = deffnm + '<STR_LIT>'   <EOL>mdp_parameters = {'<STR_LIT>':nsteps, '<STR_LIT>':dt, '<STR_LIT>': '<STR_LIT>'}<EOL>mdp_parameters.update(mdp_kwargs)<EOL>cbook.add_mdp_includes(topology, mdp_parameters)<EOL>logger.info("<STR_LIT>", vars())<EOL>with in_dir(dirname):<EOL><INDENT>if not (mdp_parameters.get('<STR_LIT>','<STR_LIT>').lower() == '<STR_LIT>' or mainselection is None):<EOL><INDENT>logger.info("<STR_LIT>".format(**vars()))<EOL>groups = make_main_index(structure, selection=mainselection,<EOL>oldndx=index, ndx=mainindex)<EOL>natoms = {g['<STR_LIT:name>']: float(g['<STR_LIT>']) for g in groups}<EOL>tc_group_names = ('<STR_LIT:__main__>', '<STR_LIT>')   <EOL>try:<EOL><INDENT>x = natoms['<STR_LIT:__main__>']/natoms['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>x = <NUM_LIT:0>   <EOL>wmsg = "<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>"<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=AutoCorrectionWarning)<EOL><DEDENT>if x < <NUM_LIT:0.1>:<EOL><INDENT>tau_t = firstof(mdp_parameters.pop('<STR_LIT>', <NUM_LIT:0.1>))<EOL>ref_t = firstof(mdp_parameters.pop('<STR_LIT>', <NUM_LIT>))<EOL>mdp_parameters['<STR_LIT>'] = '<STR_LIT>'<EOL>mdp_parameters['<STR_LIT>'] = tau_t   <EOL>mdp_parameters['<STR_LIT>'] = ref_t   <EOL>mdp_parameters['<STR_LIT>'] = mdp_parameters.pop('<STR_LIT>', ref_t)<EOL>wmsg = "<STR_LIT>""<STR_LIT>""<STR_LIT>"% (x * <NUM_LIT:100>, ref_t, tau_t)<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=AutoCorrectionWarning)<EOL><DEDENT>else:<EOL><INDENT>n_tc_groups = len(tc_group_names)<EOL>tau_t = asiterable(mdp_parameters.pop('<STR_LIT>', <NUM_LIT:0.1>))<EOL>ref_t = asiterable(mdp_parameters.pop('<STR_LIT>', <NUM_LIT>))<EOL>if len(tau_t) != n_tc_groups:<EOL><INDENT>tau_t = n_tc_groups * [tau_t[<NUM_LIT:0>]]<EOL>wmsg = "<STR_LIT>""<STR_LIT>" % (n_tc_groups, tau_t[<NUM_LIT:0>])<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=AutoCorrectionWarning)<EOL><DEDENT>if len(ref_t) != n_tc_groups:<EOL><INDENT>ref_t = n_tc_groups * [ref_t[<NUM_LIT:0>]]<EOL>wmsg = "<STR_LIT>""<STR_LIT>" % (n_tc_groups, ref_t[<NUM_LIT:0>])<EOL>logger.warn(wmsg)<EOL>warnings.warn(wmsg, category=AutoCorrectionWarning)<EOL><DEDENT>mdp_parameters['<STR_LIT>'] = tc_group_names<EOL>mdp_parameters['<STR_LIT>'] = tau_t<EOL>mdp_parameters['<STR_LIT>'] = ref_t<EOL>mdp_parameters['<STR_LIT>'] = mdp_parameters.pop('<STR_LIT>', ref_t[<NUM_LIT:0>])<EOL><DEDENT>index = realpath(mainindex)<EOL><DEDENT>if mdp_parameters.get('<STR_LIT>','<STR_LIT>').lower() == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>if mdp_parameters.get('<STR_LIT>','<STR_LIT>').lower() == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL>mdp_parameters['<STR_LIT>'] = "<STR_LIT>"<EOL><DEDENT>unprocessed = cbook.edit_mdp(mdp_template, new_mdp=mdp, **mdp_parameters)<EOL>check_mdpargs(unprocessed)<EOL>gromacs.grompp(f=mdp, p=topology, c=structure, n=index, o=tpr, **unprocessed)<EOL>runscripts = qsub.generate_submit_scripts(<EOL>qscript_template, deffnm=deffnm, jobname=qname, budget=budget,<EOL>startdir=startdir, mdrun_opts=mdrun_opts, walltime=walltime)<EOL><DEDENT>logger.info("<STR_LIT>", vars())<EOL>logger.info("<STR_LIT>", vars())<EOL>logger.info("<STR_LIT>", vars())<EOL>logger.info("<STR_LIT>", vars())<EOL>logger.info("<STR_LIT>"<EOL>"<STR_LIT>" % vars())<EOL>kwargs = {'<STR_LIT>': realpath(os.path.join(dirname, final_structure)),      <EOL>'<STR_LIT>': topology,<EOL>'<STR_LIT>': index,            <EOL>'<STR_LIT>': runscripts,<EOL>'<STR_LIT>': mainselection,<EOL>'<STR_LIT>': deffnm,        <EOL>}<EOL>kwargs.update(mdp_kwargs)  <EOL>return kwargs<EOL>

Generic function to set up a ``mdrun`` MD simulation.

    See the user functions for usage.


logger.info("<STR_LIT>".format(**vars()))<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')   <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')   <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT:0>')       <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')      <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')  <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')  <EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.setdefault('<STR_LIT>', "<STR_LIT>")<EOL>new_kwargs =  _setup_MD(dirname, **kwargs)<EOL>new_kwargs.pop('<STR_LIT>', None)          <EOL>new_kwargs.pop('<STR_LIT>', None)<EOL>new_kwargs.pop('<STR_LIT>', None)<EOL>return new_kwargs<EOL>

Set up MD with position restraints.

    Additional itp files should be in the same directory as the top file.

    Many of the keyword arguments below already have sensible values. Note that
    setting *mainselection* = ``None`` will disable many of the automated
    choices and is often recommended when using your own mdp file.

    :Keywords:
       *dirname*
          set up under directory dirname [MD_POSRES]
       *struct*
          input structure (gro, pdb, ...) [em/em.pdb]
       *top*
          topology file [top/system.top]
       *mdp*
          mdp file (or use the template) [templates/md.mdp]
       *ndx*
          index file (supply when using a custom mdp)
       *includes*
          additional directories to search for itp files
       *mainselection*
          :program:`make_ndx` selection to select main group ["Protein"]
          (If ``None`` then no canonical index file is generated and
          it is the user's responsibility to set *tc_grps*,
          *tau_t*, and *ref_t* as keyword arguments, or provide the mdp template
          with all parameter pre-set in *mdp* and probably also your own *ndx*
          index file.)
       *deffnm*
          default filename for Gromacs run [md]
       *runtime*
          total length of the simulation in ps [1000]
       *dt*
          integration time step in ps [0.002]
       *qscript*
          script to submit to the queuing system; by default
          uses the template :data:`gromacs.config.qscript_template`, which can
          be manually set to another template from :data:`gromacs.config.templates`;
          can also be a list of template names.
       *qname*
          name to be used for the job in the queuing system [PR_GMX]
       *mdrun_opts*
          option flags for the :program:`mdrun` command in the queuing system
          scripts such as "-stepout 100". [""]
       *kwargs*
          remaining key/value pairs that should be changed in the template mdp
          file, eg ``nstxtcout=250, nstfout=250`` or command line options for
          ``grompp` such as ``maxwarn=1``.

          In particular one can also set **define** and activate
          whichever position restraints have been coded into the itp
          and top file. For instance one could have

             *define* = "-DPOSRES_MainChain -DPOSRES_LIGAND"

          if these preprocessor constructs exist. Note that there
          **must not be any space between "-D" and the value.**

          By default *define* is set to "-DPOSRES".

    :Returns: a dict that can be fed into :func:`gromacs.setup.MD`
              (but check, just in case, especially if you want to
              change the ``define`` parameter in the mdp file)

    .. Note:: The output frequency is drastically reduced for position
              restraint runs by default. Set the corresponding ``nst*``
              variables if you require more output. The `pressure coupling`_
              option *refcoord_scaling* is set to "com" by default (but can
              be changed via *kwargs*) and the pressure coupling
              algorithm itself is set to *Pcoupl* = "Berendsen" to
              run a stable simulation.

    .. _`pressure coupling`: http://manual.gromacs.org/online/mdp_opt.html#pc


logger.info("<STR_LIT>".format(**vars()))<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>kwargs.setdefault('<STR_LIT>', '<STR_LIT>')<EOL>return _setup_MD(dirname, **kwargs)<EOL>

Set up equilibrium MD.

    Additional itp files should be in the same directory as the top file.

    Many of the keyword arguments below already have sensible values. Note that
    setting *mainselection* = ``None`` will disable many of the automated
    choices and is often recommended when using your own mdp file.

    :Keywords:
       *dirname*
          set up under directory dirname [MD]
       *struct*
          input structure (gro, pdb, ...) [MD_POSRES/md_posres.pdb]
       *top*
          topology file [top/system.top]
       *mdp*
          mdp file (or use the template) [templates/md.mdp]
       *ndx*
          index file (supply when using a custom mdp)
       *includes*
          additional directories to search for itp files
       *mainselection*
          ``make_ndx`` selection to select main group ["Protein"]
          (If ``None`` then no canonical index file is generated and
          it is the user's responsibility to set *tc_grps*,
          *tau_t*, and *ref_t* as keyword arguments, or provide the mdp template
          with all parameter pre-set in *mdp* and probably also your own *ndx*
          index file.)
       *deffnm*
          default filename for Gromacs run [md]
       *runtime*
          total length of the simulation in ps [1000]
       *dt*
          integration time step in ps [0.002]
       *qscript*
          script to submit to the queuing system; by default
          uses the template :data:`gromacs.config.qscript_template`, which can
          be manually set to another template from :data:`gromacs.config.templates`;
          can also be a list of template names.
       *qname*
          name to be used for the job in the queuing system [MD_GMX]
       *mdrun_opts*
          option flags for the :program:`mdrun` command in the queuing system
          scripts such as "-stepout 100 -dgdl". [""]
       *kwargs*
          remaining key/value pairs that should be changed in the template mdp
          file, e.g. ``nstxtcout=250, nstfout=250`` or command line options for
          :program`grompp` such as ``maxwarn=1``.

    :Returns: a dict that can be fed into :func:`gromacs.setup.MD`
              (but check, just in case, especially if you want to
              change the *define* parameter in the mdp file)


def scale_dihedrals(mol, dihedrals, scale, banned_lines=None):

if banned_lines is None:<EOL><INDENT>banned_lines = []<EOL><DEDENT>new_dihedrals = []<EOL>for dh in mol.dihedrals:<EOL><INDENT>atypes = dh.atom1.get_atomtype(), dh.atom2.get_atomtype(), dh.atom3.get_atomtype(), dh.atom4.get_atomtype()<EOL>atypes = [a.replace("<STR_LIT:_>", "<STR_LIT>").replace("<STR_LIT:=>","<STR_LIT>") for a in atypes]<EOL>if dh.gromacs['<STR_LIT>'] != []:<EOL><INDENT>for p in dh.gromacs['<STR_LIT>']:<EOL><INDENT>p['<STR_LIT>'] *= scale<EOL><DEDENT>new_dihedrals.append(dh)<EOL>continue<EOL><DEDENT>for iswitch in range(<NUM_LIT:32>):<EOL><INDENT>if  (iswitch%<NUM_LIT:2>==<NUM_LIT:0> ):<EOL><INDENT>a1=atypes[<NUM_LIT:0>]; a2=atypes[<NUM_LIT:1>]; a3=atypes[<NUM_LIT:2>]; a4=atypes[<NUM_LIT:3>]<EOL><DEDENT>else:<EOL><INDENT>a1=atypes[<NUM_LIT:3>]; a2=atypes[<NUM_LIT:2>]; a3=atypes[<NUM_LIT:1>]; a4=atypes[<NUM_LIT:0>]<EOL><DEDENT>if((iswitch//<NUM_LIT:2>)%<NUM_LIT:2>==<NUM_LIT:1>): a1="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:4>)%<NUM_LIT:2>==<NUM_LIT:1>): a2="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:8>)%<NUM_LIT:2>==<NUM_LIT:1>): a3="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:16>)%<NUM_LIT:2>==<NUM_LIT:1>): a4="<STR_LIT:X>";<EOL>key = "<STR_LIT>".format(a1, a2, a3, a4, dh.gromacs['<STR_LIT>'])<EOL>if (key in dihedrals):<EOL><INDENT>for i, dt in enumerate(dihedrals[key]):<EOL><INDENT>dhA = copy.deepcopy(dh)<EOL>param = copy.deepcopy(dt.gromacs['<STR_LIT>'])<EOL>if not dihedrals[key][<NUM_LIT:0>].line in banned_lines:<EOL><INDENT>for p in param: p['<STR_LIT>'] *= scale<EOL><DEDENT>dhA.gromacs['<STR_LIT>'] = param<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>dhA.comment = "<STR_LIT>".format("<STR_LIT:U+0020>".join(<EOL>map(str, banned_lines)), <NUM_LIT:1> if dt.line in banned_lines else <NUM_LIT:0>)<EOL>dhA.comment += "<STR_LIT>".format(<EOL>dhA.atom1.atomtype, dhA.atom2.atomtype, dhA.atom3.atomtype,<EOL>dhA.atom4.atomtype, dt.line).replace("<STR_LIT:_>","<STR_LIT>")<EOL><DEDENT>name = "<STR_LIT>".format(dhA.atom1.atomtype, dhA.atom2.atomtype,<EOL>dhA.atom3.atomtype, dhA.atom4.atomtype).replace("<STR_LIT:_>","<STR_LIT>")<EOL>new_dihedrals.append(dhA)<EOL><DEDENT>break<EOL><DEDENT><DEDENT><DEDENT>mol.dihedrals = new_dihedrals<EOL>return mol<EOL>


def scale_impropers(mol, impropers, scale, banned_lines=None):

if banned_lines is None:<EOL><INDENT>banned_lines = []<EOL><DEDENT>new_impropers = []<EOL>for im in mol.impropers:<EOL><INDENT>atypes = (im.atom1.get_atomtype(), im.atom2.get_atomtype(),<EOL>im.atom3.get_atomtype(), im.atom4.get_atomtype())<EOL>atypes = [a.replace("<STR_LIT:_>", "<STR_LIT>").replace("<STR_LIT:=>", "<STR_LIT>") for a in atypes]<EOL>if im.gromacs['<STR_LIT>'] != []:<EOL><INDENT>for p in im.gromacs['<STR_LIT>']:<EOL><INDENT>p['<STR_LIT>'] *= scale<EOL><DEDENT>new_impropers.append(im)<EOL>continue<EOL><DEDENT>for iswitch in range(<NUM_LIT:32>):<EOL><INDENT>if  (iswitch%<NUM_LIT:2>==<NUM_LIT:0>):<EOL><INDENT>a1=atypes[<NUM_LIT:0>]; a2=atypes[<NUM_LIT:1>]; a3=atypes[<NUM_LIT:2>]; a4=atypes[<NUM_LIT:3>];<EOL><DEDENT>else:<EOL><INDENT>a1=atypes[<NUM_LIT:3>]; a2=atypes[<NUM_LIT:2>]; a3=atypes[<NUM_LIT:1>]; a4=atypes[<NUM_LIT:0>];<EOL><DEDENT>if((iswitch//<NUM_LIT:2>)%<NUM_LIT:2>==<NUM_LIT:1>): a1="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:4>)%<NUM_LIT:2>==<NUM_LIT:1>): a2="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:8>)%<NUM_LIT:2>==<NUM_LIT:1>): a3="<STR_LIT:X>";<EOL>if((iswitch//<NUM_LIT:16>)%<NUM_LIT:2>==<NUM_LIT:1>): a4="<STR_LIT:X>";<EOL>key = "<STR_LIT>".format(a1, a2, a3, a4, im.gromacs['<STR_LIT>'])<EOL>if (key in impropers):<EOL><INDENT>for i, imt in enumerate(impropers[key]):<EOL><INDENT>imA = copy.deepcopy(im)<EOL>param = copy.deepcopy(imt.gromacs['<STR_LIT>'])<EOL>if not impropers[key][<NUM_LIT:0>].line in banned_lines:<EOL><INDENT>for p in param: p['<STR_LIT>'] *= scale<EOL><DEDENT>imA.gromacs['<STR_LIT>'] = param<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>imA.comment = "<STR_LIT>".format(<EOL>"<STR_LIT:U+0020>".join(map(str, banned_lines)),<EOL><NUM_LIT:1> if imt.line in banned_lines else <NUM_LIT:0>,<EOL>imt.atype1, imt.atype2, imt.atype3, imt.atype4, imt.line)<EOL><DEDENT>new_impropers.append(imA)<EOL><DEDENT>break<EOL><DEDENT><DEDENT><DEDENT>mol.impropers = new_impropers<EOL>return mol<EOL>


def partial_tempering(topfile="<STR_LIT>", outfile="<STR_LIT>", banned_lines='<STR_LIT>',<EOL>scale_lipids=<NUM_LIT:1.0>, scale_protein=<NUM_LIT:1.0>):

banned_lines = map(int, banned_lines.split())<EOL>top = TOP(topfile)<EOL>groups = [("<STR_LIT:_>", float(scale_protein)), ("<STR_LIT:=>", float(scale_lipids))]<EOL>cmaptypes = []<EOL>for ct in top.cmaptypes:<EOL><INDENT>cmaptypes.append(ct)<EOL>for gr, scale in groups:<EOL><INDENT>ctA = copy.deepcopy(ct)<EOL>ctA.atype1 += gr<EOL>ctA.atype2 += gr<EOL>ctA.atype3 += gr<EOL>ctA.atype4 += gr<EOL>ctA.atype8 += gr<EOL>ctA.gromacs['<STR_LIT>'] = [ v*scale for v in ct.gromacs['<STR_LIT>'] ]<EOL>cmaptypes.append(ctA)<EOL><DEDENT><DEDENT>logger.debug("<STR_LIT>".format(len(top.cmaptypes), len(cmaptypes)))<EOL>top.cmaptypes = cmaptypes<EOL>atomtypes = []<EOL>for at in top.atomtypes:<EOL><INDENT>atomtypes.append(at)<EOL>for gr, scale in groups:<EOL><INDENT>atA = copy.deepcopy(at)<EOL>atA.atnum = atA.atype<EOL>atA.atype += gr<EOL>atA.gromacs['<STR_LIT>']['<STR_LIT>'] *= scale<EOL>atomtypes.append(atA)<EOL><DEDENT><DEDENT>top.atomtypes = atomtypes<EOL>pairtypes = []<EOL>for pt in top.pairtypes:<EOL><INDENT>pairtypes.append(pt)<EOL>for gr, scale in groups:<EOL><INDENT>ptA = copy.deepcopy(pt)<EOL>ptA.atype1 += gr<EOL>ptA.atype2 += gr<EOL>ptA.gromacs['<STR_LIT>']['<STR_LIT>'] *= scale<EOL>pairtypes.append(ptA)<EOL><DEDENT><DEDENT>top.pairtypes = pairtypes<EOL>bondtypes = []<EOL>for bt in top.bondtypes:<EOL><INDENT>bondtypes.append(bt)<EOL>for gr, scale in groups:<EOL><INDENT>btA = copy.deepcopy(bt)<EOL>btA.atype1 += gr<EOL>btA.atype2 += gr<EOL>bondtypes.append(btA)<EOL><DEDENT><DEDENT>top.bondtypes = bondtypes<EOL>angletypes = []<EOL>for at in top.angletypes:<EOL><INDENT>angletypes.append(at)<EOL>for gr, scale in groups:<EOL><INDENT>atA = copy.deepcopy(at)<EOL>atA.atype1 += gr<EOL>atA.atype2 += gr<EOL>atA.atype3 += gr<EOL>angletypes.append(atA)<EOL><DEDENT><DEDENT>top.angletypes = angletypes<EOL>dihedraltypes = {}<EOL>for dt in top.dihedraltypes:<EOL><INDENT>dt.disabled = True<EOL>dt.comment = "<STR_LIT>".format(<EOL>dt.atype1, dt.atype2, dt.atype3, dt.atype4, dt.line)<EOL>dt.comment = dt.comment.replace("<STR_LIT:_>","<STR_LIT>")<EOL>name = "<STR_LIT>".format(dt.atype1, dt.atype2, dt.atype3, dt.atype4, dt.gromacs['<STR_LIT>'])<EOL>if not name in dihedraltypes:<EOL><INDENT>dihedraltypes[name] = []<EOL><DEDENT>dihedraltypes[name].append(dt)<EOL><DEDENT>logger.debug("<STR_LIT>".format(len(dihedraltypes)))<EOL>impropertypes = {}<EOL>for it in top.impropertypes:<EOL><INDENT>it.disabled = True<EOL>it.comment = "<STR_LIT>".format(it.line)<EOL>name = "<STR_LIT>".format(<EOL>it.atype1, it.atype2, it.atype3, it.atype4, it.gromacs['<STR_LIT>'])<EOL>if not name in impropertypes:<EOL><INDENT>impropertypes[name] = []<EOL><DEDENT>impropertypes[name].append(it)<EOL><DEDENT>logger.debug("<STR_LIT>".format(len(impropertypes)))<EOL>for molname_mol in top.dict_molname_mol:<EOL><INDENT>if not '<STR_LIT>' in molname_mol:<EOL><INDENT>continue<EOL><DEDENT>mol = top.dict_molname_mol[molname_mol]<EOL>for at in mol.atoms:<EOL><INDENT>at.charge *= math.sqrt(scale_protein)<EOL><DEDENT>mol = scale_dihedrals(mol, dihedraltypes, scale_protein, banned_lines)<EOL>mol = scale_impropers(mol, impropertypes, <NUM_LIT:1.0>, banned_lines)<EOL><DEDENT>top.write(outfile)<EOL>

Set up topology for partial tempering (REST2) replica exchange.

    .. versionchanged:: 0.7.0
       Use keyword arguments instead of an `args` Namespace object.


def send(self, use_open_peers=True, queue=True, **kw):

if not use_open_peers:<EOL><INDENT>ip = kw.get('<STR_LIT>')<EOL>port = kw.get('<STR_LIT:port>')<EOL>peer = '<STR_LIT>'.format(ip, port)<EOL>res = arky.rest.POST.peer.transactions(peer=peer, transactions=[self.tx.tx])<EOL><DEDENT>else:<EOL><INDENT>res = arky.core.sendPayload(self.tx.tx)<EOL><DEDENT>if self.tx.success != '<STR_LIT>':<EOL><INDENT>self.tx.error = None<EOL>self.tx.success = True<EOL><DEDENT>else:<EOL><INDENT>self.tx.error = res['<STR_LIT>']<EOL>self.tx.success = False<EOL><DEDENT>self.tx.tries += <NUM_LIT:1><EOL>self.tx.res = res<EOL>if queue:<EOL><INDENT>self.tx.send = True<EOL><DEDENT>self.__save()<EOL>return res<EOL>

send a transaction immediately. Failed transactions are picked up by the TxBroadcaster

:param ip: specific peer IP to send tx to
:param port: port of specific peer
:param use_open_peers: use Arky's broadcast method


def check_confirmations_or_resend(self, use_open_peers=False, **kw):

if self.confirmations() == <NUM_LIT:0>:<EOL><INDENT>self.send(use_open_peers, **kw)<EOL><DEDENT>

check if a tx is confirmed, else resend it.

:param use_open_peers: select random peers fro api/peers endpoint


self.tx.send = True<EOL>self.__save()<EOL>

queue a transaction to be picked up by the background transaction broadcaster


self.tx.send = False<EOL>self.__save()<EOL>

save a transaction, but don't let it be picked up by the broadcaster


if model._meta.app_label == '<STR_LIT>':<EOL><INDENT>return ark_db['<STR_LIT>']<EOL><DEDENT>return None<EOL>

Attempts to read ark_app models go to ARK_DB_NAME.


Attempts to write ark_app models go to ARK_DB_NAME.


def allow_relation(self, obj1, obj2, **hints):

if obj1._meta.app_label == '<STR_LIT>' orobj2._meta.app_label == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT>return None<EOL>

Allow relations if a model in the ark is involved.


def allow_migrate(self, db, app_label, model_name=None, **hints):

if app_label == '<STR_LIT>':<EOL><INDENT>return db == ark_db['<STR_LIT>']<EOL><DEDENT>return None<EOL>

Make sure the ark_app app only appears in the 'ARK_DB_NAME'
database.


for i in range(<NUM_LIT:10>):<EOL><INDENT>tx = TX(<EOL>amount=i+<NUM_LIT:1>,<EOL>secret="<STR_LIT>",<EOL>recipient="<STR_LIT>",<EOL>network="<STR_LIT>",<EOL>)<EOL>tx.queue()<EOL><DEDENT>

We bake some transactions and store them in the db.


def configure_ci_jobs(<EOL>config_url, rosdistro_name, ci_build_name,<EOL>groovy_script=None, dry_run=False):

config = get_config_index(config_url)<EOL>build_files = get_ci_build_files(config, rosdistro_name)<EOL>build_file = build_files[ci_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>targets = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>print('<STR_LIT>', os_name, os_code_name, arch)<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>ci_view_name = get_ci_view_name(rosdistro_name)<EOL>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url) if groovy_script is None else False<EOL>view_configs = {}<EOL>views = {<EOL>ci_view_name: configure_ci_view(<EOL>jenkins, ci_view_name, dry_run=dry_run)<EOL>}<EOL>if not jenkins:<EOL><INDENT>view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(view_configs),<EOL>}<EOL>ci_job_names = []<EOL>job_configs = OrderedDict()<EOL>is_disabled = False<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>try:<EOL><INDENT>job_name, job_config = configure_ci_job(<EOL>config_url, rosdistro_name, ci_build_name,<EOL>os_name, os_code_name, arch,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>jenkins=jenkins, views=views,<EOL>is_disabled=is_disabled,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run,<EOL>trigger_timer=build_file.jenkins_job_schedule)<EOL>ci_job_names.append(job_name)<EOL>if groovy_script is not None:<EOL><INDENT>print("<STR_LIT>" % job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, job_configs, view_configs)<EOL><DEDENT>


if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_ci_build_files(config, rosdistro_name)<EOL>build_file = build_files[ci_build_name]<EOL><DEDENT>if build_targets is not None:<EOL><INDENT>build_file.targets = build_targets<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if arch not in build_file.targets[os_name][os_code_name]:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % arch +<EOL>'<STR_LIT>' % '<STR_LIT:U+002CU+0020>'.join(sorted(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>if len(build_file.underlay_from_ci_jobs) > <NUM_LIT:1>:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>' +<EOL>'<STR_LIT>' % len(build_file.underlay_from_ci_jobs))<EOL><DEDENT>underlay_source_job = None<EOL>if build_file.underlay_from_ci_jobs:<EOL><INDENT>underlay_source_job = get_ci_job_name(<EOL>rosdistro_name, os_name, os_code_name, arch,<EOL>build_file.underlay_from_ci_jobs[<NUM_LIT:0>])<EOL>underlay_source_paths = (underlay_source_paths or []) +['<STR_LIT>']<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>view_name = get_ci_view_name(rosdistro_name)<EOL>configure_ci_view(jenkins, view_name, dry_run=dry_run)<EOL><DEDENT>job_name = get_ci_job_name(<EOL>rosdistro_name, os_name, os_code_name, arch, ci_build_name)<EOL>job_config = _get_ci_job_config(<EOL>index, rosdistro_name, build_file, os_name,<EOL>os_code_name, arch,<EOL>build_file.repos_files,<EOL>underlay_source_job,<EOL>underlay_source_paths,<EOL>trigger_timer,<EOL>is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>return job_name, job_config<EOL>

Configure a single Jenkins CI job.

This includes the following steps:
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the ci/run_ci_job.py script


def configure_release_jobs(<EOL>config_url, rosdistro_name, release_build_name, groovy_script=None,<EOL>dry_run=False, whitelist_package_names=None):

config = get_config_index(config_url)<EOL>build_files = get_release_build_files(config, rosdistro_name)<EOL>build_file = build_files[release_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>platforms = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>platforms.append((os_name, os_code_name))<EOL><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name in platforms:<EOL><INDENT>print('<STR_LIT>' % (os_name, os_code_name, '<STR_LIT:U+002CU+0020>'.join(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>pkg_names = dist_file.release_packages.keys()<EOL>filtered_pkg_names = build_file.filter_packages(pkg_names)<EOL>explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names)<EOL>if explicitly_ignored_pkg_names:<EOL><INDENT>print(('<STR_LIT>' +<EOL>'<STR_LIT>') %<EOL>('<STR_LIT>' if build_file.skip_ignored_packages else '<STR_LIT>'))<EOL>for pkg_name in sorted(explicitly_ignored_pkg_names):<EOL><INDENT>print('<STR_LIT>', pkg_name)<EOL><DEDENT><DEDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL>if explicitly_ignored_pkg_names:<EOL><INDENT>direct_dependencies = {}<EOL>for pkg_name in pkg_names:<EOL><INDENT>direct_dependencies[pkg_name] = _get_direct_dependencies(<EOL>pkg_name, dist_cache, pkg_names) or set([])<EOL><DEDENT>ignored_pkg_names = set(explicitly_ignored_pkg_names)<EOL>while True:<EOL><INDENT>implicitly_ignored_pkg_names = _get_downstream_package_names(<EOL>ignored_pkg_names, direct_dependencies)<EOL>if implicitly_ignored_pkg_names - ignored_pkg_names:<EOL><INDENT>ignored_pkg_names |= implicitly_ignored_pkg_names<EOL>continue<EOL><DEDENT>break<EOL><DEDENT>implicitly_ignored_pkg_names =ignored_pkg_names - explicitly_ignored_pkg_names<EOL>if implicitly_ignored_pkg_names:<EOL><INDENT>print(('<STR_LIT>' +<EOL>'<STR_LIT>') % ('<STR_LIT>'<EOL>if build_file.skip_ignored_packages else '<STR_LIT>'))<EOL>for pkg_name in sorted(implicitly_ignored_pkg_names):<EOL><INDENT>print('<STR_LIT>', pkg_name)<EOL><DEDENT>filtered_pkg_names =set(filtered_pkg_names) - implicitly_ignored_pkg_names<EOL><DEDENT><DEDENT>jenkins = False<EOL>if groovy_script is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>all_view_configs = {}<EOL>all_job_configs = OrderedDict()<EOL>job_name, job_config = configure_import_package_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>all_job_configs[job_name] = job_config<EOL><DEDENT>job_name, job_config = configure_sync_packages_to_main_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>all_job_configs[job_name] = job_config<EOL><DEDENT>for os_name, os_code_name in platforms:<EOL><INDENT>for arch in sorted(build_file.targets[os_name][os_code_name]):<EOL><INDENT>job_name, job_config = configure_sync_packages_to_testing_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>os_code_name, arch,<EOL>config=config, build_file=build_file, jenkins=jenkins,<EOL>dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>all_job_configs[job_name] = job_config<EOL><DEDENT><DEDENT><DEDENT>targets = []<EOL>for os_name, os_code_name in platforms:<EOL><INDENT>targets.append((os_name, os_code_name, '<STR_LIT:source>'))<EOL>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT>views = configure_release_views(<EOL>jenkins, rosdistro_name, release_build_name, targets,<EOL>dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>all_view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(views),<EOL>}<EOL>from catkin_pkg.package import parse_package_string<EOL>from ros_buildfarm.common import topological_order_packages<EOL>pkgs = {}<EOL>for pkg_name in pkg_names:<EOL><INDENT>if pkg_name not in dist_cache.release_package_xmls:<EOL><INDENT>print("<STR_LIT>" %<EOL>(pkg_name), file=sys.stderr)<EOL>continue<EOL><DEDENT>pkg_xml = dist_cache.release_package_xmls[pkg_name]<EOL>pkg = parse_package_string(pkg_xml)<EOL>pkgs[pkg_name] = pkg<EOL><DEDENT>ordered_pkg_tuples = topological_order_packages(pkgs)<EOL>other_build_files = [v for k, v in build_files.items() if k != release_build_name]<EOL>all_source_job_names = []<EOL>all_binary_job_names = []<EOL>for pkg_name in [p.name for _, p in ordered_pkg_tuples]:<EOL><INDENT>if whitelist_package_names:<EOL><INDENT>if pkg_name not in whitelist_package_names:<EOL><INDENT>print("<STR_LIT>" %<EOL>pkg_name, file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>pkg = dist_file.release_packages[pkg_name]<EOL>repo_name = pkg.repository_name<EOL>repo = dist_file.repositories[repo_name]<EOL>is_disabled = pkg_name not in filtered_pkg_names<EOL>if is_disabled and build_file.skip_ignored_packages:<EOL><INDENT>print("<STR_LIT>" %<EOL>(pkg_name, repo_name), file=sys.stderr)<EOL>continue<EOL><DEDENT>if not repo.release_repository:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT>") % (pkg_name, repo_name), file=sys.stderr)<EOL>continue<EOL><DEDENT>if not repo.release_repository.version:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT:version>") % (pkg_name, repo_name), file=sys.stderr)<EOL>continue<EOL><DEDENT>for os_name, os_code_name in platforms:<EOL><INDENT>other_build_files_same_platform = []<EOL>for other_build_file in other_build_files:<EOL><INDENT>if os_name not in other_build_file.targets:<EOL><INDENT>continue<EOL><DEDENT>if os_code_name not in other_build_file.targets[os_name]:<EOL><INDENT>continue<EOL><DEDENT>other_build_files_same_platform.append(other_build_file)<EOL><DEDENT>try:<EOL><INDENT>source_job_names, binary_job_names, job_configs =configure_release_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>pkg_name, os_name, os_code_name,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>dist_cache=dist_cache,<EOL>jenkins=jenkins, views=views,<EOL>generate_import_package_job=False,<EOL>generate_sync_packages_jobs=False,<EOL>is_disabled=is_disabled,<EOL>other_build_files_same_platform=other_build_files_same_platform,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run)<EOL>all_source_job_names += source_job_names<EOL>all_binary_job_names += binary_job_names<EOL>if groovy_script is not None:<EOL><INDENT>print('<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(source_job_names + binary_job_names))<EOL>for source_job_name in source_job_names:<EOL><INDENT>all_job_configs[source_job_name] = job_configs[source_job_name]<EOL><DEDENT>for binary_job_name in binary_job_names:<EOL><INDENT>all_job_configs[binary_job_name] = job_configs[binary_job_name]<EOL><DEDENT><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(all_job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>if not whitelist_package_names:<EOL><INDENT>for os_name, os_code_name in platforms:<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>binary_view = get_release_binary_view_name(<EOL>rosdistro_name, release_build_name,<EOL>os_name, os_code_name, arch)<EOL>binary_job_prefix = '<STR_LIT>' % binary_view<EOL>excluded_job_names = set([<EOL>j for j in all_binary_job_names<EOL>if j.startswith(binary_job_prefix)])<EOL>if groovy_script is None:<EOL><INDENT>print("<STR_LIT>" %<EOL>binary_job_prefix)<EOL>from ros_buildfarm.jenkins import remove_jobs<EOL>remove_jobs(<EOL>jenkins, binary_job_prefix, excluded_job_names,<EOL>dry_run=dry_run)<EOL><DEDENT>else:<EOL><INDENT>binary_key = '<STR_LIT>' %(os_name, os_code_name, arch)<EOL>groovy_data['<STR_LIT>'][binary_key] =(binary_job_prefix, excluded_job_names)<EOL><DEDENT><DEDENT><DEDENT>for os_name, os_code_name in platforms:<EOL><INDENT>other_source_job_names = []<EOL>for other_release_build_name in [<EOL>k for k in build_files.keys() if k != release_build_name]:<EOL><INDENT>other_build_file = build_files[other_release_build_name]<EOL>other_dist_file = get_distribution_file(<EOL>index, rosdistro_name, other_build_file)<EOL>if not other_dist_file:<EOL><INDENT>continue<EOL><DEDENT>if os_name not in other_build_file.targets oros_code_name not in other_build_file.targets[os_name]:<EOL><INDENT>continue<EOL><DEDENT>if other_build_file.skip_ignored_packages:<EOL><INDENT>filtered_pkg_names = other_build_file.filter_packages(<EOL>pkg_names)<EOL><DEDENT>else:<EOL><INDENT>filtered_pkg_names = pkg_names<EOL><DEDENT>for pkg_name in sorted(filtered_pkg_names):<EOL><INDENT>pkg = other_dist_file.release_packages[pkg_name]<EOL>repo_name = pkg.repository_name<EOL>repo = other_dist_file.repositories[repo_name]<EOL>if not repo.release_repository:<EOL><INDENT>continue<EOL><DEDENT>if not repo.release_repository.version:<EOL><INDENT>continue<EOL><DEDENT>other_job_name = get_sourcedeb_job_name(<EOL>rosdistro_name, other_release_build_name,<EOL>pkg_name, os_name, os_code_name)<EOL>other_source_job_names.append(other_job_name)<EOL><DEDENT><DEDENT>source_view_prefix = get_release_source_view_name(<EOL>rosdistro_name, os_name, os_code_name)<EOL>source_job_prefix = '<STR_LIT>' % source_view_prefix<EOL>excluded_job_names = set([<EOL>j for j in (all_source_job_names + other_source_job_names)<EOL>if j.startswith(source_job_prefix)])<EOL>if groovy_script is None:<EOL><INDENT>print("<STR_LIT>" %<EOL>source_job_prefix)<EOL>from ros_buildfarm.jenkins import remove_jobs<EOL>remove_jobs(<EOL>jenkins, source_job_prefix, excluded_job_names,<EOL>dry_run=dry_run)<EOL><DEDENT>else:<EOL><INDENT>source_key = '<STR_LIT>' % (os_name, os_code_name)<EOL>groovy_data['<STR_LIT>'][source_key] = (<EOL>source_job_prefix, excluded_job_names)<EOL><DEDENT><DEDENT><DEDENT>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(all_view_configs), len(all_job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, all_job_configs,<EOL>view_configs=all_view_configs)<EOL><DEDENT>

Configure all Jenkins release jobs.

L{configure_release_job} will be invoked for every released package and
target which matches the build file criteria.

Additionally a job to import Debian packages into the Debian repository is
created.


if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_release_build_files(config, rosdistro_name)<EOL>build_file = build_files[release_build_name]<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>pkg_names = dist_file.release_packages.keys()<EOL>if pkg_name not in pkg_names:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % pkg_name +<EOL>'<STR_LIT>' + '<STR_LIT:U+002CU+0020>'.join(sorted(pkg_names)))<EOL><DEDENT>pkg = dist_file.release_packages[pkg_name]<EOL>repo_name = pkg.repository_name<EOL>repo = dist_file.repositories[repo_name]<EOL>if not repo.release_repository:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if not repo.release_repository.version:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if dist_cache is None and(build_file.notify_maintainers or<EOL>build_file.abi_incompatibility_assumed):<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>targets = []<EOL>targets.append((os_name, os_code_name, '<STR_LIT:source>'))<EOL>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT>configure_release_views(<EOL>jenkins, rosdistro_name, release_build_name, targets,<EOL>dry_run=dry_run)<EOL><DEDENT>if generate_import_package_job:<EOL><INDENT>configure_import_package_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config=config, build_file=build_file, jenkins=jenkins,<EOL>dry_run=dry_run)<EOL><DEDENT>if generate_sync_packages_jobs:<EOL><INDENT>configure_sync_packages_to_main_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config=config, build_file=build_file, jenkins=jenkins,<EOL>dry_run=dry_run)<EOL>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>configure_sync_packages_to_testing_job(<EOL>config_url, rosdistro_name, release_build_name,<EOL>os_code_name, arch,<EOL>config=config, build_file=build_file, jenkins=jenkins,<EOL>dry_run=dry_run)<EOL><DEDENT><DEDENT>source_job_names = []<EOL>binary_job_names = []<EOL>job_configs = {}<EOL>source_job_name = get_sourcedeb_job_name(<EOL>rosdistro_name, release_build_name,<EOL>pkg_name, os_name, os_code_name)<EOL>is_source_disabled = is_disabled<EOL>if is_source_disabled and other_build_files_same_platform:<EOL><INDENT>for other_build_file in other_build_files_same_platform:<EOL><INDENT>if other_build_file.filter_packages([pkg_name]):<EOL><INDENT>is_source_disabled = False<EOL>break<EOL><DEDENT><DEDENT><DEDENT>job_config = _get_sourcedeb_job_config(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config, build_file, os_name, os_code_name,<EOL>pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache,<EOL>is_disabled=is_source_disabled,<EOL>other_build_files_same_platform=other_build_files_same_platform)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, source_job_name, job_config, dry_run=dry_run)<EOL><DEDENT>source_job_names.append(source_job_name)<EOL>job_configs[source_job_name] = job_config<EOL>dependency_names = []<EOL>if build_file.abi_incompatibility_assumed:<EOL><INDENT>dependency_names = _get_direct_dependencies(<EOL>pkg_name, dist_cache, pkg_names)<EOL>if dependency_names is None:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT>") % pkg_name, file=sys.stderr)<EOL>return source_job_names, binary_job_names, job_configs<EOL><DEDENT><DEDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>if filter_arches and arch not in filter_arches:<EOL><INDENT>continue<EOL><DEDENT>job_name = get_binarydeb_job_name(<EOL>rosdistro_name, release_build_name,<EOL>pkg_name, os_name, os_code_name, arch)<EOL>upstream_job_names = [source_job_name] + [<EOL>get_binarydeb_job_name(<EOL>rosdistro_name, release_build_name,<EOL>dependency_name, os_name, os_code_name, arch)<EOL>for dependency_name in dependency_names]<EOL>job_config = _get_binarydeb_job_config(<EOL>config_url, rosdistro_name, release_build_name,<EOL>config, build_file, os_name, os_code_name, arch,<EOL>pkg_name, repo_name, repo.release_repository,<EOL>dist_cache=dist_cache, upstream_job_names=upstream_job_names,<EOL>is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>binary_job_names.append(job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT>return source_job_names, binary_job_names, job_configs<EOL>

Configure a Jenkins release job.

The following jobs are created for each package:
- M source jobs, one for each OS node name
- M * N binary jobs, one for each combination of OS code name and arch


def configure_doc_jobs(<EOL>config_url, rosdistro_name, doc_build_name, groovy_script=None,<EOL>dry_run=False, whitelist_repository_names=None):

config = get_config_index(config_url)<EOL>build_files = get_doc_build_files(config, rosdistro_name)<EOL>build_file = build_files[doc_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>dist_cache = None<EOL>if build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>targets = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>print('<STR_LIT>', os_name, os_code_name, arch)<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)<EOL>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url) if groovy_script is None else False<EOL>view_configs = {}<EOL>views = {}<EOL>views[doc_view_name] = configure_doc_view(<EOL>jenkins, doc_view_name, dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(view_configs),<EOL>}<EOL>repo_names = dist_file.repositories.keys()<EOL>filtered_repo_names = build_file.filter_repositories(repo_names)<EOL>job_names = []<EOL>job_configs = OrderedDict()<EOL>for repo_name in sorted(repo_names):<EOL><INDENT>if whitelist_repository_names:<EOL><INDENT>if repo_name not in whitelist_repository_names:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>repo_name, file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>is_disabled = repo_name not in filtered_repo_names<EOL>if is_disabled and build_file.skip_ignored_repositories:<EOL><INDENT>print("<STR_LIT>" % repo_name,<EOL>file=sys.stderr)<EOL>continue<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.doc_repository:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>if not repo.doc_repository.version:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>for os_name, os_code_name, arch in targets:<EOL><INDENT>try:<EOL><INDENT>job_name, job_config = configure_doc_job(<EOL>config_url, rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>dist_cache=dist_cache, jenkins=jenkins, views=views,<EOL>is_disabled=is_disabled,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run)<EOL>job_names.append(job_name)<EOL>if groovy_script is not None:<EOL><INDENT>print("<STR_LIT>" % job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>job_prefix = '<STR_LIT>' % doc_view_name<EOL>if not whitelist_repository_names:<EOL><INDENT>groovy_data['<STR_LIT>']['<STR_LIT>'] = (job_prefix, job_names)<EOL>if groovy_script is None:<EOL><INDENT>from ros_buildfarm.jenkins import remove_jobs<EOL>print('<STR_LIT>')<EOL>remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run)<EOL><DEDENT><DEDENT>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(view_configs), len(job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, job_configs, view_configs=view_configs)<EOL><DEDENT>

Configure all Jenkins doc jobs.

L{configure_doc_job} will be invoked for doc repository and target
which matches the build file criteria.


def configure_doc_job(<EOL>config_url, rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>config=None, build_file=None,<EOL>index=None, dist_file=None, dist_cache=None,<EOL>jenkins=None, views=None,<EOL>is_disabled=False,<EOL>groovy_script=None,<EOL>doc_repository=None,<EOL>dry_run=False):

if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_doc_build_files(config, rosdistro_name)<EOL>build_file = build_files[doc_build_name]<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>repo_names = dist_file.repositories.keys()<EOL>if repo_name is not None:<EOL><INDENT>if repo_name not in repo_names:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name +<EOL>'<STR_LIT>' %<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(repo_names)))<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.doc_repository:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if not repo.doc_repository.version:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>doc_repository = repo.doc_repository<EOL><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if arch not in build_file.targets[os_name][os_code_name]:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % arch +<EOL>'<STR_LIT>' % '<STR_LIT:U+002CU+0020>'.join(sorted(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>if dist_cache is None and build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>view_name = get_doc_view_name(<EOL>rosdistro_name, doc_build_name)<EOL>configure_doc_view(jenkins, view_name, dry_run=dry_run)<EOL><DEDENT>job_name = get_doc_job_name(<EOL>rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch)<EOL>job_config = _get_doc_job_config(<EOL>config, config_url, rosdistro_name, doc_build_name,<EOL>build_file, os_name, os_code_name, arch, doc_repository,<EOL>repo_name, dist_cache=dist_cache, is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>return job_name, job_config<EOL>

Configure a single Jenkins doc job.

This includes the following steps:
- clone the doc repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the run_doc_job.py script


def write_groovy_script_and_configs(<EOL>filename, content, job_configs, view_configs=None):

with open(filename, '<STR_LIT:w>') as h:<EOL><INDENT>h.write(content)<EOL><DEDENT>if view_configs:<EOL><INDENT>view_config_dir = os.path.join(os.path.dirname(filename), '<STR_LIT>')<EOL>if not os.path.isdir(view_config_dir):<EOL><INDENT>os.makedirs(view_config_dir)<EOL><DEDENT>for config_name, config_body in view_configs.items():<EOL><INDENT>config_filename = os.path.join(view_config_dir, config_name)<EOL>with open(config_filename, '<STR_LIT:w>') as config_fh:<EOL><INDENT>config_fh.write(config_body)<EOL><DEDENT><DEDENT><DEDENT>job_config_dir = os.path.join(os.path.dirname(filename), '<STR_LIT>')<EOL>if not os.path.isdir(job_config_dir):<EOL><INDENT>os.makedirs(job_config_dir)<EOL><DEDENT>format_str = '<STR_LIT>' + str(len(str(len(job_configs)))) + '<STR_LIT:d>'<EOL>i = <NUM_LIT:0><EOL>for config_name, config_body in job_configs.items():<EOL><INDENT>i += <NUM_LIT:1><EOL>config_filename = os.path.join(<EOL>job_config_dir,<EOL>format_str % i + '<STR_LIT:U+0020>' + config_name)<EOL>with open(config_filename, '<STR_LIT:w>') as config_fh:<EOL><INDENT>config_fh.write(config_body)<EOL><DEDENT><DEDENT>

Write out the groovy script and configs to file.

    This writes the reconfigure script to the file location
    and places the expanded configs in subdirectories 'view_configs' /
    'job_configs' that the script can then access when run.


from catkin_pkg.topological_order import _PackageDecorator<EOL>from catkin_pkg.topological_order import _sort_decorated_packages<EOL>decorators_by_name = {}<EOL>for path, package in packages.items():<EOL><INDENT>decorators_by_name[package.name] = _PackageDecorator(package, path)<EOL><DEDENT>for decorator in decorators_by_name.values():<EOL><INDENT>decorator.depends_for_topological_order = set([])<EOL>all_depends =decorator.package.build_depends + decorator.package.buildtool_depends +decorator.package.run_depends + decorator.package.test_depends<EOL>unique_depend_names = set([<EOL>d.name for d in all_depends if d.name in decorators_by_name.keys()])<EOL>for name in unique_depend_names:<EOL><INDENT>if name in decorator.depends_for_topological_order:<EOL><INDENT>continue<EOL><DEDENT>decorators_by_name[name]._add_recursive_run_depends(<EOL>decorators_by_name, decorator.depends_for_topological_order)<EOL><DEDENT><DEDENT>ordered_pkg_tuples = _sort_decorated_packages(decorators_by_name)<EOL>for pkg_path, pkg in ordered_pkg_tuples:<EOL><INDENT>if pkg_path is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>' % pkg)<EOL><DEDENT><DEDENT>return ordered_pkg_tuples<EOL>

Order packages topologically.

First returning packages which have message generators and then
the rest based on all direct depends and indirect recursive run_depends.

:param packages: A dict mapping relative paths to ``Package`` objects ``dict``
:returns: A list of tuples containing the relative path and a ``Package`` object, ``list``


def get_version_status(<EOL>package_descriptors, targets, repos_data,<EOL>strip_version=False, strip_os_code_name=False):

status = {}<EOL>for package_descriptor in package_descriptors.values():<EOL><INDENT>pkg_name = package_descriptor.pkg_name<EOL>debian_pkg_name = package_descriptor.debian_pkg_name<EOL>ref_version = package_descriptor.version<EOL>if strip_version:<EOL><INDENT>ref_version = _strip_version_suffix(ref_version)<EOL><DEDENT>status[pkg_name] = {}<EOL>for target in targets:<EOL><INDENT>statuses = []<EOL>for repo_data in repos_data:<EOL><INDENT>version = repo_data.get(target, {}).get(debian_pkg_name, None)<EOL>if strip_version:<EOL><INDENT>version = _strip_version_suffix(version)<EOL><DEDENT>if strip_os_code_name:<EOL><INDENT>version = _strip_os_code_name_suffix(<EOL>version, target.os_code_name)<EOL><DEDENT>if ref_version:<EOL><INDENT>if not version:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>elif version.startswith(ref_version):  <EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>if _version_is_gt_other(version, ref_version):<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if not version:<EOL><INDENT>statuses.append('<STR_LIT:ignore>')<EOL><DEDENT>else:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>status[pkg_name][target] = statuses<EOL><DEDENT><DEDENT>return status<EOL>

For each package and target check if it is affected by a sync.

This is the case when the package version in the testing repo is different
from the version in the main repo.

:return: a dict indexed by package names containing
  dicts indexed by targets containing
  a list of status strings (one for each repo)


global version_regex<EOL>if not version:<EOL><INDENT>return version<EOL><DEDENT>match = version_regex.search(version)<EOL>return match.group(<NUM_LIT:0>) if match else version<EOL>

Remove trailing junk from the version number.

>>> strip_version_suffix('')
''
>>> strip_version_suffix('None')
'None'
>>> strip_version_suffix('1.2.3-4trusty-20140131-1359-+0000')
'1.2.3-4'
>>> strip_version_suffix('1.2.3-foo')
'1.2.3'


def get_homogeneous(package_descriptors, targets, repos_data):

homogeneous = {}<EOL>for package_descriptor in package_descriptors.values():<EOL><INDENT>pkg_name = package_descriptor.pkg_name<EOL>debian_pkg_name = package_descriptor.debian_pkg_name<EOL>versions = []<EOL>for repo_data in repos_data:<EOL><INDENT>versions.append(set([]))<EOL>for target in targets:<EOL><INDENT>version = _strip_version_suffix(<EOL>repo_data.get(target, {}).get(debian_pkg_name, None))<EOL>versions[-<NUM_LIT:1>].add(version)<EOL><DEDENT><DEDENT>homogeneous[pkg_name] = max([len(v) for v in versions]) == <NUM_LIT:1><EOL><DEDENT>return homogeneous<EOL>

For each package check if the version in one repo is equal for all targets.

The version could be different in different repos though.

:return: a dict indexed by package names containing a boolean flag


def get_package_counts(package_descriptors, targets, repos_data):

counts = {}<EOL>for target in targets:<EOL><INDENT>counts[target] = [<NUM_LIT:0>] * len(repos_data)<EOL><DEDENT>for package_descriptor in package_descriptors.values():<EOL><INDENT>debian_pkg_name = package_descriptor.debian_pkg_name<EOL>for target in targets:<EOL><INDENT>for i, repo_data in enumerate(repos_data):<EOL><INDENT>version = repo_data.get(target, {}).get(debian_pkg_name, None)<EOL>if version:<EOL><INDENT>counts[target][i] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return counts<EOL>

Get the number of packages per target and repository.

:return: a dict indexed by targets containing
  a list of integer values (one for each repo)


def get_jenkins_job_urls(<EOL>rosdistro_name, jenkins_url, release_build_name, targets):

urls = {}<EOL>for target in targets:<EOL><INDENT>view_name = get_release_view_name(<EOL>rosdistro_name, release_build_name,<EOL>target.os_name, target.os_code_name, target.arch)<EOL>base_url = jenkins_url + '<STR_LIT>' %(view_name, view_name)<EOL>if target.arch == '<STR_LIT:source>':<EOL><INDENT>urls[target] = base_url + '<STR_LIT>' %(target.os_name, target.os_code_name)<EOL><DEDENT>else:<EOL><INDENT>urls[target] = base_url + '<STR_LIT>' %(target.os_name, target.os_code_name, target.arch)<EOL><DEDENT><DEDENT>return urls<EOL>

Get the Jenkins job urls for each target.

The placeholder {pkg} needs to be replaced with the ROS package name.

:return: a dict indexed by targets containing a string


def configure_devel_jobs(<EOL>config_url, rosdistro_name, source_build_name, groovy_script=None,<EOL>dry_run=False, whitelist_repository_names=None):

config = get_config_index(config_url)<EOL>build_files = get_source_build_files(config, rosdistro_name)<EOL>build_file = build_files[source_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>dist_cache = None<EOL>if build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>targets = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>print('<STR_LIT>', os_name, os_code_name, arch)<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>devel_view_name = get_devel_view_name(<EOL>rosdistro_name, source_build_name, pull_request=False)<EOL>pull_request_view_name = get_devel_view_name(<EOL>rosdistro_name, source_build_name, pull_request=True)<EOL>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url) if groovy_script is None else False<EOL>view_configs = {}<EOL>views = {}<EOL>if build_file.test_commits_force is not False:<EOL><INDENT>views[devel_view_name] = configure_devel_view(<EOL>jenkins, devel_view_name, dry_run=dry_run)<EOL><DEDENT>if build_file.test_pull_requests_force is not False:<EOL><INDENT>views[pull_request_view_name] = configure_devel_view(<EOL>jenkins, pull_request_view_name, dry_run=dry_run)<EOL><DEDENT>if not jenkins:<EOL><INDENT>view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(view_configs),<EOL>}<EOL>repo_names = dist_file.repositories.keys()<EOL>filtered_repo_names = build_file.filter_repositories(repo_names)<EOL>devel_job_names = []<EOL>pull_request_job_names = []<EOL>job_configs = OrderedDict()<EOL>for repo_name in sorted(repo_names):<EOL><INDENT>if whitelist_repository_names:<EOL><INDENT>if repo_name not in whitelist_repository_names:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>repo_name, file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>is_disabled = repo_name not in filtered_repo_names<EOL>if is_disabled and build_file.skip_ignored_repositories:<EOL><INDENT>print("<STR_LIT>" % repo_name,<EOL>file=sys.stderr)<EOL>continue<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.source_repository:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>if not repo.source_repository.version:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>job_types = []<EOL>if build_file.test_commits_force is False:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT>") % repo_name)<EOL><DEDENT>elif repo.source_repository.test_commits is False:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT>") % repo_name)<EOL><DEDENT>elif repo.source_repository.test_commits is None andnot build_file.test_commits_default:<EOL><INDENT>print(("<STR_LIT>" +<EOL>"<STR_LIT>") % repo_name)<EOL><DEDENT>else:<EOL><INDENT>job_types.append('<STR_LIT>')<EOL><DEDENT>if not is_disabled:<EOL><INDENT>if build_file.test_pull_requests_force is False:<EOL><INDENT>pass<EOL><DEDENT>elif repo.source_repository.test_pull_requests is False:<EOL><INDENT>pass<EOL><DEDENT>elif repo.source_repository.test_pull_requests is None andnot build_file.test_pull_requests_default:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>job_types.append('<STR_LIT>')<EOL><DEDENT><DEDENT>for job_type in job_types:<EOL><INDENT>pull_request = job_type == '<STR_LIT>'<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>try:<EOL><INDENT>job_name, job_config = configure_devel_job(<EOL>config_url, rosdistro_name, source_build_name,<EOL>repo_name, os_name, os_code_name, arch, pull_request,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>dist_cache=dist_cache, jenkins=jenkins, views=views,<EOL>is_disabled=is_disabled,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run)<EOL>if not pull_request:<EOL><INDENT>devel_job_names.append(job_name)<EOL><DEDENT>else:<EOL><INDENT>pull_request_job_names.append(job_name)<EOL><DEDENT>if groovy_script is not None:<EOL><INDENT>print("<STR_LIT>" % job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>devel_job_prefix = '<STR_LIT>' % devel_view_name<EOL>pull_request_job_prefix = '<STR_LIT>' % pull_request_view_name<EOL>if not whitelist_repository_names:<EOL><INDENT>groovy_data['<STR_LIT>']['<STR_LIT>'] =(devel_job_prefix, devel_job_names)<EOL>groovy_data['<STR_LIT>']['<STR_LIT>'] =(pull_request_job_prefix, pull_request_job_names)<EOL>if groovy_script is None:<EOL><INDENT>from ros_buildfarm.jenkins import remove_jobs<EOL>print('<STR_LIT>')<EOL>remove_jobs(<EOL>jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run)<EOL>print('<STR_LIT>')<EOL>remove_jobs(<EOL>jenkins, pull_request_job_prefix, pull_request_job_names,<EOL>dry_run=dry_run)<EOL><DEDENT><DEDENT>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(view_configs), len(job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, job_configs, view_configs=view_configs)<EOL><DEDENT>

Configure all Jenkins devel jobs.

L{configure_release_job} will be invoked for source repository and target
which matches the build file criteria.


def configure_devel_job(<EOL>config_url, rosdistro_name, source_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>pull_request=False,<EOL>config=None, build_file=None,<EOL>index=None, dist_file=None, dist_cache=None,<EOL>jenkins=None, views=None,<EOL>is_disabled=False,<EOL>groovy_script=None,<EOL>source_repository=None,<EOL>build_targets=None,<EOL>dry_run=False):

if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_source_build_files(config, rosdistro_name)<EOL>build_file = build_files[source_build_name]<EOL><DEDENT>if build_targets is not None:<EOL><INDENT>build_file.targets = build_targets<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>repo_names = dist_file.repositories.keys()<EOL>if repo_name is not None:<EOL><INDENT>if repo_name not in repo_names:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name +<EOL>'<STR_LIT>' %<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(repo_names)))<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.source_repository:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if not repo.source_repository.version:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>source_repository = repo.source_repository<EOL><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if arch not in build_file.targets[os_name][os_code_name]:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % arch +<EOL>'<STR_LIT>' % '<STR_LIT:U+002CU+0020>'.join(sorted(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>if dist_cache is None and build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>view_name = get_devel_view_name(<EOL>rosdistro_name, source_build_name, pull_request=pull_request)<EOL>configure_devel_view(jenkins, view_name, dry_run=dry_run)<EOL><DEDENT>job_name = get_devel_job_name(<EOL>rosdistro_name, source_build_name,<EOL>repo_name, os_name, os_code_name, arch, pull_request)<EOL>job_config = _get_devel_job_config(<EOL>index, config, rosdistro_name, source_build_name,<EOL>build_file, os_name, os_code_name, arch, source_repository,<EOL>repo_name, pull_request, job_name, dist_cache=dist_cache,<EOL>is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>return job_name, job_config<EOL>

Configure a single Jenkins devel job.

This includes the following steps:
- clone the source repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the release/run_devel_job.py script


parser = ArgumentParser()<EOL>add_to_parser(parser, usage)<EOL>return parser<EOL>

Creates an argument parser from a condensed usage string in the format of:

This is the program description

:pos_arg_name int
    This is the help message
    which can span multiple lines
:-o --optional_arg str default_value
    The type can be any valid python type
:-eo --extra-option
    This adds args.extra_option as a bool
    which is False by default

Args:
    usage (str): Usage string in the format described above
Returns:
    ArgumentParser: parser object created from the described string


def add_to_parser(parser, usage, ignore_existing=False):

usage = '<STR_LIT:\n>' + usage<EOL>first_line = [i for i in usage.split('<STR_LIT:\n>') if i][<NUM_LIT:0>]<EOL>indent = '<STR_LIT:U+0020>' * (len(first_line) - len(first_line.lstrip('<STR_LIT:U+0020>')))<EOL>usage = usage.replace('<STR_LIT:\n>' + indent, '<STR_LIT:\n>')<EOL>usage = usage.replace('<STR_LIT>', '<STR_LIT>')<EOL>defaults = {}<EOL>description, *descriptors = usage.split('<STR_LIT>')<EOL>description = description.replace('<STR_LIT:\n>', '<STR_LIT:U+0020>').strip()<EOL>if description and (not parser.description or not ignore_existing):<EOL><INDENT>parser.description = description<EOL><DEDENT>for descriptor in descriptors:<EOL><INDENT>try:<EOL><INDENT>options, *info = descriptor.split('<STR_LIT:\n>')<EOL>info = '<STR_LIT:U+0020>'.join(i for i in info if i).replace('<STR_LIT:U+0020>', '<STR_LIT>')<EOL>if options.count('<STR_LIT:U+0020>') == <NUM_LIT:1>:<EOL><INDENT>if options[<NUM_LIT:0>] == '<STR_LIT:->':<EOL><INDENT>short, long = options.split('<STR_LIT:U+0020>')<EOL>var_name = long.strip('<STR_LIT:->').replace('<STR_LIT:->', '<STR_LIT:_>')<EOL>parser.add_argument(short, long, dest=var_name, action='<STR_LIT:store_true>', help=info)<EOL>defaults[var_name] = False<EOL><DEDENT>else:<EOL><INDENT>short, typ = options.split('<STR_LIT:U+0020>')<EOL>parser.add_argument(short, type=types[typ], help=info)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>short, long, typ, default = options.split('<STR_LIT:U+0020>')<EOL>info = info.rstrip() + '<STR_LIT>' + default<EOL>default = '<STR_LIT>' if default == '<STR_LIT:->' else default<EOL>parser.add_argument(short, long, type=types[typ], default=default, help=info)<EOL><DEDENT><DEDENT>except ArgumentError:<EOL><INDENT>if not ignore_existing:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>print(e.__class__.__name__ + '<STR_LIT>' + str(e))<EOL>print('<STR_LIT>')<EOL>print(descriptor)<EOL>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT>

Add arguments described in the usage string to the
parser. View more details at the <create_parser> docs

Args:
    parser (ArgumentParser): parser to add arguments to
    usage (str): Usage string in the format described above
    ignore_existing (bool): Ignore any arguments that have already been defined


if len(self._attached_bundles) == <NUM_LIT:0>:<EOL><INDENT>raise NoBundlesAttached("<STR_LIT>")<EOL><DEDENT>for bundle in self._attached_bundles:<EOL><INDENT>processed_bundle = {<EOL>'<STR_LIT:path>': bundle.path,<EOL>'<STR_LIT:description>': bundle.description,<EOL>'<STR_LIT>': []<EOL>}<EOL>for (bp, description) in bundle.blueprints:<EOL><INDENT>blueprint = self._register_blueprint(app, bp, bundle.path,<EOL>self.get_bp_path(bp), description)<EOL>processed_bundle['<STR_LIT>'].append(blueprint)<EOL><DEDENT>self._registered_bundles.append(processed_bundle)<EOL><DEDENT>

Initializes Journey extension

        :param app: App passed from constructor or directly to init_app
        :raises:
            - NoBundlesAttached if no bundles has been attached attached


return self._registered_bundles<EOL>

Returns a detailed list of bundles along with blueprints and routes

        :return: List of blueprint routes


routes = []<EOL>for bundle in self._registered_bundles:<EOL><INDENT>bundle_path = bundle['<STR_LIT:path>']<EOL>for blueprint in bundle['<STR_LIT>']:<EOL><INDENT>bp_path = blueprint['<STR_LIT:path>']<EOL>for child in blueprint['<STR_LIT>']:<EOL><INDENT>routes.append(<EOL>(<EOL>child['<STR_LIT>'],<EOL>bundle_path + bp_path + child['<STR_LIT:path>'],<EOL>child['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return routes<EOL>

Returns simple info about registered blueprints

        :return: Tuple containing endpoint, path and allowed methods for each route


for attached_bundle in self._attached_bundles:<EOL><INDENT>if path == attached_bundle.path:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Checks if a bundle exists at the provided path

        :param path: Bundle path
        :return: bool


if not isinstance(bundle, BlueprintBundle):<EOL><INDENT>raise IncompatibleBundle('<STR_LIT>'<EOL>.format(BlueprintBundle))<EOL><DEDENT>elif len(bundle.blueprints) == <NUM_LIT:0>:<EOL><INDENT>raise MissingBlueprints("<STR_LIT>")<EOL><DEDENT>elif self._bundle_exists(bundle.path):<EOL><INDENT>raise ConflictingPath("<STR_LIT>".format(bundle.path))<EOL><DEDENT>elif self._journey_path == bundle.path == '<STR_LIT:/>':<EOL><INDENT>raise ConflictingPath("<STR_LIT>".format(bundle.path))<EOL><DEDENT>self._attached_bundles.append(bundle)<EOL>

Attaches a bundle object

        :param bundle: :class:`flask_journey.BlueprintBundle` object
        :raises:
            - IncompatibleBundle if the bundle is not of type `BlueprintBundle`
            - ConflictingPath if a bundle already exists at bundle.path
            - MissingBlueprints if the bundle doesn't contain any blueprints


def _register_blueprint(self, app, bp, bundle_path, child_path, description):

base_path = sanitize_path(self._journey_path + bundle_path + child_path)<EOL>app.register_blueprint(bp, url_prefix=base_path)<EOL>return {<EOL>'<STR_LIT:name>': bp.name,<EOL>'<STR_LIT:path>': child_path,<EOL>'<STR_LIT>': bp.import_name,<EOL>'<STR_LIT:description>': description,<EOL>'<STR_LIT>': self.get_blueprint_routes(app, base_path)<EOL>}<EOL>

Register and return info about the registered blueprint

        :param bp: :class:`flask.Blueprint` object
        :param bundle_path: the URL prefix of the bundle
        :param child_path: blueprint relative to the bundle path
        :return: Dict with info about the blueprint


return bp.url_prefix or '<STR_LIT:/>' + bp.name<EOL>

Returns url_prefix if set, otherwise bp.name prefixed with a slash.

        :param bp: :class:`flask.Blueprint` object
        :return: blueprint name


routes = []<EOL>for child in app.url_map.iter_rules():<EOL><INDENT>if child.rule.startswith(base_path):<EOL><INDENT>relative_path = child.rule[len(base_path):]<EOL>routes.append({<EOL>'<STR_LIT:path>': relative_path,<EOL>'<STR_LIT>': child.endpoint,<EOL>'<STR_LIT>': list(child.methods)<EOL>})<EOL><DEDENT><DEDENT>return routes<EOL>

Returns detailed information about registered blueprint routes matching the `BlueprintBundle` path

        :param app: App instance to obtain rules from
        :param base_path: Base path to return detailed route info for
        :return: List of route detail dicts


def attach_bp(self, bp, description='<STR_LIT>'):

if not isinstance(bp, Blueprint):<EOL><INDENT>raise InvalidBlueprint('<STR_LIT>'.format(Blueprint))<EOL><DEDENT>self.blueprints.append((bp, description))<EOL>

Attaches a flask.Blueprint to the bundle

        :param bp: :class:`flask.Blueprint` object
        :param description: Optional description string
        :raises:
            - InvalidBlueprint if the Blueprint is not of type `flask.Blueprint`


if path == '<STR_LIT:/>':  <EOL><INDENT>return path<EOL><DEDENT>if path[:<NUM_LIT:1>] != '<STR_LIT:/>':<EOL><INDENT>raise InvalidPath('<STR_LIT>')<EOL><DEDENT>path = re.sub(r'<STR_LIT>', '<STR_LIT:/>', path)<EOL>return path.rstrip('<STR_LIT:/>')<EOL>

Performs sanitation of the path after validating

    :param path: path to sanitize
    :return: path
    :raises:
        - InvalidPath if the path doesn't start with a slash


if obj is not None and not isinstance(obj, Schema):<EOL><INDENT>raise IncompatibleSchema('<STR_LIT>'.format(Schema))<EOL><DEDENT>return obj<EOL>

Ensures the passed schema instance is compatible

    :param obj: object to validate
    :return: obj
    :raises:
        - IncompatibleSchema if the passed schema is of an incompatible type


kwargs['<STR_LIT>'] = kwargs.pop('<STR_LIT>', False)<EOL>body = _validate_schema(kwargs.pop('<STR_LIT>', None))<EOL>query = _validate_schema(kwargs.pop('<STR_LIT>', None))<EOL>output = _validate_schema(kwargs.pop('<STR_LIT>', None))<EOL>validate = kwargs.pop('<STR_LIT>', True)<EOL>def decorator(f):<EOL><INDENT>@bp.route(*args, **kwargs)<EOL>@wraps(f)<EOL>def wrapper(*inner_args, **inner_kwargs):<EOL><INDENT>"""<STR_LIT>"""<EOL>try:<EOL><INDENT>if query is not None:<EOL><INDENT>query.strict = validate<EOL>url = furl(request.url)<EOL>inner_kwargs['<STR_LIT>'] = query.load(data=url.args)<EOL><DEDENT>if body is not None:<EOL><INDENT>body.strict = validate<EOL>json_data = request.get_json()<EOL>if json_data is None:<EOL><INDENT>json_data = {}<EOL><DEDENT>inner_kwargs['<STR_LIT>'] = body.load(data=json_data)<EOL><DEDENT><DEDENT>except ValidationError as err:<EOL><INDENT>return jsonify(err.messages), <NUM_LIT><EOL><DEDENT>if output:<EOL><INDENT>data = output.dump(f(*inner_args, **inner_kwargs))<EOL>return jsonify(data[<NUM_LIT:0>])<EOL><DEDENT>return f(*inner_args, **inner_kwargs)<EOL><DEDENT>return f<EOL><DEDENT>return decorator<EOL>

Journey route decorator

    Enables simple serialization, deserialization and validation of Flask routes with the help of Marshmallow.

    :param bp: :class:`flask.Blueprint` object
    :param args: args to pass along to `Blueprint.route`
    :param kwargs:
        - :strict_slashes: Enable / disable strict slashes (default False)
        - :validate: Enable / disable body/query validation (default True)
        - :_query: Unmarshal Query string into this schema
        - :_body: Unmarshal JSON body into this schema
        - :marshal_with: Serialize the output with this schema
    :raises:
        - ValidationError if the query parameters or JSON body fails validation


with io.open('<STR_LIT>') as input_file:<EOL><INDENT>for line in input_file:<EOL><INDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>return ast.parse(line).body[<NUM_LIT:0>].value.s<EOL><DEDENT><DEDENT><DEDENT>

Parses main module and fetches version attribute from the syntax tree

    :return: flask-journey version


compressed_bits = cStringIO.StringIO()<EOL>f = gzip.GzipFile(fileobj=compressed_bits, mode='<STR_LIT:wb>')<EOL>f.write(input_str)<EOL>f.close()<EOL>return compressed_bits.getvalue()<EOL>

Compress the log message in order to send less bytes to the wire.


for filename in ('<STR_LIT>', os.path.expanduser('<STR_LIT>')):<EOL><INDENT>project_id, access_token, api_domain = _parse_config_file_impl(filename)<EOL>if project_id is not Noneand access_token is not Noneand api_domain is not None:<EOL><INDENT>return project_id, access_token, api_domain<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return None, None, None<EOL><DEDENT>

Find the .splunk_logger config file in the current directory, or in the
user's home and parse it. The one in the current directory has precedence.

:return: A tuple with:
            - project_id
            - access_token


try:<EOL><INDENT>doc = yaml.load(file(filename).read())<EOL>project_id = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>access_token = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>api_domain = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>return project_id, access_token, api_domain<EOL><DEDENT>except:<EOL><INDENT>return None, None, None<EOL><DEDENT>

Format for the file is:

     credentials:
         project_id: ...
         access_token: ...
         api_domain: ...

:param filename: The filename to parse
:return: A tuple with:
            - project_id
            - access_token
            - api_domain


args = []<EOL>for k, v in kwargs.iteritems():<EOL><INDENT>if isinstance(v, basestring):<EOL><INDENT>qv = v.encode('<STR_LIT:utf-8>') if isinstance(v, unicode) else v<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT>elif isinstance(v, collections.Iterable):<EOL><INDENT>for i in v:<EOL><INDENT>qv = i.encode('<STR_LIT:utf-8>') if isinstance(i, unicode) else str(i)<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>qv = str(v)<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT><DEDENT>return '<STR_LIT:&>'.join(args)<EOL>


ids = []<EOL>for v in args:<EOL><INDENT>if isinstance(v, basestring):<EOL><INDENT>qv = v.encode('<STR_LIT:utf-8>') if isinstance(v, unicode) else v<EOL>ids.append(urllib.quote(qv))<EOL><DEDENT>else:<EOL><INDENT>qv = str(v)<EOL>ids.append(urllib.quote(qv))<EOL><DEDENT><DEDENT>return '<STR_LIT:;>'.join(ids)<EOL>


<EOL>ingest_task = Task('<STR_LIT>')<EOL>ingest_task.inputs.items = output_port_value<EOL>ingest_task.impersonation_allowed = True<EOL>stage_task = Task('<STR_LIT>')<EOL>stage_task.inputs.destination = '<STR_LIT>'<EOL>stage_task.inputs.data = ingest_task.outputs.result.value<EOL>self.definition['<STR_LIT>'].append(ingest_task.generate_task_workflow_json())<EOL>self.definition['<STR_LIT>'].append(stage_task.generate_task_workflow_json())<EOL>

append two required tasks to the given output to ingest to VS


interface = Auth(**kwargs)<EOL>self.base_url = '<STR_LIT>'<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.logger = interface.logger<EOL>

Construct an instance of an AnswerFactory Recipe

Args:
    **kwargs

Returns:
    An instance of a Recipe.


self.logger.debug('<STR_LIT>' + recipe_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': recipe_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves an AnswerFactory Recipe by id

Args:
    recipe_id The id of the recipe

Returns:
    A JSON representation of the recipe


self.logger.debug('<STR_LIT>')<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves a list of AnswerFactory Recipes

Args:
    None

Returns:
    A list of JSON representations of recipes


<EOL>if '<STR_LIT:id>' in recipe and recipe['<STR_LIT:id>'] is not None:<EOL><INDENT>self.logger.debug("<STR_LIT>" + json.dumps(recipe))<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': recipe['<STR_LIT:id>']<EOL>}<EOL>r = self.gbdx_connection.put(url, json=recipe)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>return recipe['<STR_LIT:id>']<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug("<STR_LIT>" + json.dumps(recipe))<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>r = self.gbdx_connection.post(url, json=recipe)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>recipe_json = r.json()<EOL>return recipe_json['<STR_LIT:id>']<EOL><DEDENT>

Saves an AnswerFactory Recipe

Args:
    recipe (dict): Dictionary specifying a recipe

Returns:
    AnswerFactory Recipe id


self.logger.debug('<STR_LIT>' + recipe_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': recipe_id<EOL>}<EOL>r = self.gbdx_connection.delete(url)<EOL>r.raise_for_status()<EOL>

Deletes an AnswerFactory Recipe by id

Args:
     recipe_id: The id of the recipe to delete

Returns:
     Nothing


Construct an instance of an AnswerFactory Project

Args:
    **kwargs

Returns:
    An instance of a Project.


self.logger.debug('<STR_LIT>' + project_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': project_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves an AnswerFactory Project by id

Args:
    project_id

Returns:
    A JSON representation of the project


<EOL>if '<STR_LIT:id>' in project and project['<STR_LIT:id>'] is not None:<EOL><INDENT>self.logger.debug('<STR_LIT>' + json.dumps(project))<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': project['<STR_LIT:id>']<EOL>}<EOL>r = self.gbdx_connection.put(url, json=project)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>return project['<STR_LIT:id>']<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug('<STR_LIT>' + json.dumps(project))<EOL>url = self.base_url<EOL>r = self.gbdx_connection.post(url, json=project)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>project_json = r.json()<EOL>return project_json['<STR_LIT:id>']<EOL><DEDENT>

Saves an AnswerFactory Project

Args:
    project (dict): Dictionary specifying an AnswerFactory Project.

Returns:
    AnswerFactory Project id


self.logger.debug('<STR_LIT>' + project_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': project_id<EOL>}<EOL>r = self.gbdx_connection.delete(url)<EOL>r.raise_for_status()<EOL>

Deletes a project by id

Args:
     project_id: The project id to delete

Returns:
     Nothing


interface = Auth(**kwargs)<EOL>self.base_url = '<STR_LIT>' % interface.root_url<EOL>self.workflows_url = '<STR_LIT>' % self.base_url<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.s3 = S3()<EOL>self.logger = interface.logger<EOL>

Construct the Workflow instance

        Returns:
            An instance of the Workflow class.


<EOL>try:<EOL><INDENT>r = self.gbdx_connection.post(self.workflows_url, json=workflow)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>" % r.status_code)<EOL>print("<STR_LIT>" % r.text)<EOL>r.raise_for_status()<EOL><DEDENT>workflow_id = r.json()['<STR_LIT:id>']<EOL>return workflow_id<EOL><DEDENT>except TypeError:<EOL><INDENT>self.logger.debug('<STR_LIT>')<EOL><DEDENT>

Launches GBDX workflow.

        Args:
            workflow (dict): Dictionary specifying workflow tasks.

        Returns:
            Workflow id (str).


self.logger.debug('<STR_LIT>' + workflow_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()['<STR_LIT:state>']<EOL>

Checks workflow status.

         Args:
             workflow_id (str): Workflow id.

         Returns:
             Workflow status (str).


self.logger.debug('<STR_LIT>' + workflow_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Get existing workflow state and task information.

         Args:
             workflow_id (str): Workflow id.

         Returns:
             Workflow object (dict).


def get_stdout(self, workflow_id, task_id):

url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id, '<STR_LIT>': task_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.text<EOL>

Get stdout for a particular task.

         Args:
             workflow_id (str): Workflow id.
             task_id (str): Task id.

         Returns:
             Stdout of the task (string).


def get_stderr(self, workflow_id, task_id):

Get stderr for a particular task.

         Args:
             workflow_id (str): Workflow id.
             task_id (str): Task id.

         Returns:
             Stderr of the task (string).


self.logger.debug('<STR_LIT>' + workflow_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>return r.json()['<STR_LIT>']<EOL>

Get workflow events.

         Args:
             workflow_id (str): Workflow id.

         Returns:
             List of workflow events.


self.logger.debug('<STR_LIT>' + workflow_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id<EOL>}<EOL>r = self.gbdx_connection.post(url, data='<STR_LIT>')<EOL>r.raise_for_status()<EOL>

Cancels a running workflow.

           Args:
               workflow_id (str): Workflow id.

           Returns:
               Nothing


<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>try:<EOL><INDENT>r = self.gbdx_connection.post(url, json=batch_workflow)<EOL>batch_workflow_id = r.json()['<STR_LIT>']<EOL>return batch_workflow_id<EOL><DEDENT>except TypeError as e:<EOL><INDENT>self.logger.debug('<STR_LIT>'.format(e))<EOL><DEDENT>

Launches GBDX batch workflow.

        Args:
            batch_workflow (dict): Dictionary specifying batch workflow tasks.

        Returns:
            Batch Workflow id (str).