if not hasattr(settings, '<STR_LIT>'):<EOL><INDENT>return {}<EOL><DEDENT>return {'<STR_LIT>': settings.ROLLBAR}<EOL>

Grabs the rollbar settings to make them available to templates.


try:<EOL><INDENT>from pyramid_debugtoolbar import tbtools<EOL><DEDENT>except ImportError:<EOL><INDENT>return<EOL><DEDENT>rollbar_web_base = settings.get('<STR_LIT>', DEFAULT_WEB_BASE)<EOL>if rollbar_web_base.endswith('<STR_LIT:/>'):<EOL><INDENT>rollbar_web_base = rollbar_web_base[:-<NUM_LIT:1>]<EOL><DEDENT>def insert_rollbar_console(request, html):<EOL><INDENT>item_uuid = request.environ.get('<STR_LIT>')<EOL>if not item_uuid:<EOL><INDENT>return html<EOL><DEDENT>url = '<STR_LIT>' % (rollbar_web_base, item_uuid)<EOL>link = '<STR_LIT>' % url<EOL>new_data = "<STR_LIT>" % link<EOL>insertion_marker = "<STR_LIT>"<EOL>replacement = insertion_marker + new_data<EOL>return html.replace(insertion_marker, replacement, <NUM_LIT:1>)<EOL><DEDENT>old_render_full = tbtools.Traceback.render_full<EOL>def new_render_full(self, request, *args, **kw):<EOL><INDENT>html = old_render_full(self, request, *args, **kw)<EOL>return insert_rollbar_console(request, html)<EOL><DEDENT>tbtools.Traceback.render_full = new_render_full<EOL>

Patches the pyramid_debugtoolbar (if installed) to display a link to the related rollbar item.


settings = config.registry.settings<EOL>config.add_tween('<STR_LIT>', over=EXCVIEW)<EOL>if asbool(settings.get('<STR_LIT>', True)):<EOL><INDENT>patch_debugtoolbar(settings)<EOL><DEDENT>def hook(request, data):<EOL><INDENT>data['<STR_LIT>'] = '<STR_LIT>'<EOL>if request:<EOL><INDENT>request.environ['<STR_LIT>'] = data['<STR_LIT>']<EOL>if request.matched_route:<EOL><INDENT>data['<STR_LIT>'] = request.matched_route.name<EOL><DEDENT><DEDENT><DEDENT>rollbar.BASE_DATA_HOOK = hook<EOL>kw = parse_settings(settings)<EOL>access_token = kw.pop('<STR_LIT>')<EOL>environment = kw.pop('<STR_LIT>', '<STR_LIT>')<EOL>if kw.get('<STR_LIT>'):<EOL><INDENT>kw['<STR_LIT>'] = set([str.strip(x) for x in kw.get('<STR_LIT>').split('<STR_LIT:\n>') if x])<EOL><DEDENT>if kw.get('<STR_LIT>'):<EOL><INDENT>r = DottedNameResolver()<EOL>exception_level_filters = []<EOL>for line in kw.get('<STR_LIT>').split('<STR_LIT:\n>'):<EOL><INDENT>if line:<EOL><INDENT>dotted_path, level = line.split()<EOL>try:<EOL><INDENT>cls = r.resolve(dotted_path)<EOL>exception_level_filters.append((cls, level))<EOL><DEDENT>except ImportError:<EOL><INDENT>log.error('<STR_LIT>' % dotted_path)<EOL><DEDENT><DEDENT><DEDENT>kw['<STR_LIT>'] = exception_level_filters<EOL><DEDENT>kw['<STR_LIT>'] = asbool(kw.get('<STR_LIT>', True))<EOL>rollbar.init(access_token, environment, **kw)<EOL>


<EOL>job_info = job.to_dict()<EOL>job_info['<STR_LIT:data>'] = repr(job_info['<STR_LIT:data>'])<EOL>extra_data = {'<STR_LIT>': job_info}<EOL>payload_data = {'<STR_LIT>': '<STR_LIT>'}<EOL>rollbar.report_exc_info(exc_info, extra_data=extra_data, payload_data=payload_data)<EOL>return True<EOL>

Called by RQ when there is a failure in a worker.

NOTE: Make sure that in your RQ worker process, rollbar.init() has been called with
handler='blocking'. The default handler, 'thread', does not work from inside an RQ worker.


<EOL>foo()<EOL>self.transport.write(data)<EOL>

As soon as any data is received, write it back.


<EOL>for fn in (_get_bottle_request,<EOL>_get_flask_request,<EOL>_get_pyramid_request,<EOL>_get_pylons_request):<EOL><INDENT>try:<EOL><INDENT>req = fn()<EOL>if req is not None:<EOL><INDENT>return req<EOL><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return None<EOL>

Get the current request object. Implementation varies on
library support. Modified below when we know which framework
is being used.


def init(access_token, environment='<STR_LIT>', scrub_fields=None, url_fields=None, **kw):

global SETTINGS, agent_log, _initialized, _transforms, _serialize_transform, _threads<EOL>if scrub_fields is not None:<EOL><INDENT>SETTINGS['<STR_LIT>'] = list(scrub_fields)<EOL><DEDENT>if url_fields is not None:<EOL><INDENT>SETTINGS['<STR_LIT>'] = list(url_fields)<EOL><DEDENT>SETTINGS = dict_merge(SETTINGS, kw)<EOL>if _initialized:<EOL><INDENT>if not SETTINGS.get('<STR_LIT>'):<EOL><INDENT>log.warning('<STR_LIT>')<EOL><DEDENT>return<EOL><DEDENT>SETTINGS['<STR_LIT>'] = access_token<EOL>SETTINGS['<STR_LIT>'] = environment<EOL>if SETTINGS.get('<STR_LIT>'):<EOL><INDENT>logging.basicConfig()<EOL><DEDENT>if SETTINGS.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>agent_log = _create_agent_log()<EOL><DEDENT>_serialize_transform = SerializableTransform(safe_repr=SETTINGS['<STR_LIT>']['<STR_LIT>'],<EOL>whitelist_types=SETTINGS['<STR_LIT>']['<STR_LIT>'])<EOL>_transforms = [<EOL>ScrubRedactTransform(),<EOL>_serialize_transform,<EOL>ScrubTransform(suffixes=[(field,) for field in SETTINGS['<STR_LIT>']], redact_char='<STR_LIT:*>'),<EOL>ScrubUrlTransform(suffixes=[(field,) for field in SETTINGS['<STR_LIT>']], params_to_scrub=SETTINGS['<STR_LIT>'])<EOL>]<EOL>shortener_keys = [<EOL>('<STR_LIT>', '<STR_LIT:POST>'),<EOL>('<STR_LIT>', '<STR_LIT>'),<EOL>('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT:POST>'),<EOL>('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>'),<EOL>]<EOL>if SETTINGS['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT:code>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT:args>', '<STR_LIT:*>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT>', '<STR_LIT:*>'))<EOL>shortener_keys.append(('<STR_LIT:body>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:*>', '<STR_LIT>', '<STR_LIT:*>'))<EOL><DEDENT>shortener_keys.extend(SETTINGS['<STR_LIT>'])<EOL>shortener = ShortenerTransform(safe_repr=SETTINGS['<STR_LIT>']['<STR_LIT>'],<EOL>keys=shortener_keys,<EOL>**SETTINGS['<STR_LIT>']['<STR_LIT>'])<EOL>_transforms.append(shortener)<EOL>_threads = queue.Queue()<EOL>events.reset()<EOL>filters.add_builtin_filters(SETTINGS)<EOL>_initialized = True<EOL>

Saves configuration variables in this module's SETTINGS.

access_token: project access token. Get this from the Rollbar UI:
              - click "Settings" in the top nav
              - click "Projects" in the left nav
              - copy-paste the appropriate token.
environment: environment name. Can be any string; suggestions: 'production', 'development',
             'staging', 'yourname'
**kw: provided keyword arguments will override keys in SETTINGS.


@functools.wraps(f)<EOL>def wrapper(event, context):<EOL><INDENT>global _CURRENT_LAMBDA_CONTEXT<EOL>_CURRENT_LAMBDA_CONTEXT = context<EOL>try:<EOL><INDENT>result = f(event, context)<EOL>return wait(lambda: result)<EOL><DEDENT>except:<EOL><INDENT>cls, exc, trace = sys.exc_info()<EOL>report_exc_info((cls, exc, trace.tb_next))<EOL>wait()<EOL>raise<EOL><DEDENT><DEDENT>return wrapper<EOL>

Decorator for making error handling on AWS Lambda easier


def report_exc_info(exc_info=None, request=None, extra_data=None, payload_data=None, level=None, **kw):

if exc_info is None:<EOL><INDENT>exc_info = sys.exc_info()<EOL><DEDENT>try:<EOL><INDENT>return _report_exc_info(exc_info, request, extra_data, payload_data, level=level)<EOL><DEDENT>except Exception as e:<EOL><INDENT>log.exception("<STR_LIT>", e)<EOL><DEDENT>

Reports an exception to Rollbar, using exc_info (from calling sys.exc_info())

exc_info: optional, should be the result of calling sys.exc_info(). If omitted, sys.exc_info() will be called here.
request: optional, a WebOb, Werkzeug-based or Sanic request object.
extra_data: optional, will be included in the 'custom' section of the payload
payload_data: optional, dict that will override values in the final payload
              (e.g. 'level' or 'fingerprint')
kw: provided for legacy purposes; unused.

Example usage:

rollbar.init(access_token='YOUR_PROJECT_ACCESS_TOKEN')
try:
    do_something()
except:
    rollbar.report_exc_info(sys.exc_info(), request, {'foo': 'bar'}, {'level': 'warning'})


def report_message(message, level='<STR_LIT:error>', request=None, extra_data=None, payload_data=None):

try:<EOL><INDENT>return _report_message(message, level, request, extra_data, payload_data)<EOL><DEDENT>except Exception as e:<EOL><INDENT>log.exception("<STR_LIT>", e)<EOL><DEDENT>

Reports an arbitrary string message to Rollbar.

message: the string body of the message
level: level to report at. One of: 'critical', 'error', 'warning', 'info', 'debug'
request: the request object for the context of the message
extra_data: dictionary of params to include with the message. 'body' is reserved.
payload_data: param names to pass in the 'data' level of the payload; overrides defaults.


payload = events.on_payload(payload)<EOL>if payload is False:<EOL><INDENT>return<EOL><DEDENT>payload_str = _serialize_payload(payload)<EOL>handler = SETTINGS.get('<STR_LIT>')<EOL>if handler == '<STR_LIT>':<EOL><INDENT>_send_payload(payload_str, access_token)<EOL><DEDENT>elif handler == '<STR_LIT>':<EOL><INDENT>agent_log.error(payload_str)<EOL><DEDENT>elif handler == '<STR_LIT>':<EOL><INDENT>if TornadoAsyncHTTPClient is None:<EOL><INDENT>log.error('<STR_LIT>')<EOL>return<EOL><DEDENT>_send_payload_tornado(payload_str, access_token)<EOL><DEDENT>elif handler == '<STR_LIT>':<EOL><INDENT>if AppEngineFetch is None:<EOL><INDENT>log.error('<STR_LIT>')<EOL>return<EOL><DEDENT>_send_payload_appengine(payload_str, access_token)<EOL><DEDENT>elif handler == '<STR_LIT>':<EOL><INDENT>if treq is None:<EOL><INDENT>log.error('<STR_LIT>')<EOL>return<EOL><DEDENT>_send_payload_twisted(payload_str, access_token)<EOL><DEDENT>else:<EOL><INDENT>thread = threading.Thread(target=_send_payload, args=(payload_str, access_token))<EOL>_threads.put(thread)<EOL>thread.start()<EOL><DEDENT>

Sends a payload object, (the result of calling _build_payload() + _serialize_payload()).
Uses the configured handler from SETTINGS['handler']

Available handlers:
- 'blocking': calls _send_payload() (which makes an HTTP request) immediately, blocks on it
- 'thread': starts a single-use thread that will call _send_payload(). returns immediately.
- 'agent': writes to a log file to be processed by rollbar-agent
- 'tornado': calls _send_payload_tornado() (which makes an async HTTP request using tornado's AsyncHTTPClient)
- 'gae': calls _send_payload_appengine() (which makes a blocking call to Google App Engine)
- 'twisted': calls _send_payload_twisted() (which makes an async HTTP reqeust using Twisted and Treq)


def search_items(title, return_fields=None, access_token=None, endpoint=None, **search_fields):

if not title:<EOL><INDENT>return []<EOL><DEDENT>if return_fields is not None:<EOL><INDENT>return_fields = '<STR_LIT:U+002C>'.join(return_fields)<EOL><DEDENT>return _get_api('<STR_LIT>',<EOL>title=title,<EOL>fields=return_fields,<EOL>access_token=access_token,<EOL>endpoint=endpoint,<EOL>**search_fields)<EOL>

Searches a project for items that match the input criteria.

title: all or part of the item's title to search for.
return_fields: the fields that should be returned for each item.
        e.g. ['id', 'project_id', 'status'] will return a dict containing
             only those fields for each item.
access_token: a project access token. If this is not provided,
              the one provided to init() will be used instead.
search_fields: additional fields to include in the search.
        currently supported: status, level, environment


log_file = SETTINGS['<STR_LIT>']<EOL>if not log_file.endswith('<STR_LIT>'):<EOL><INDENT>log.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>log_file = DEFAULTS['<STR_LIT>']<EOL><DEDENT>retval = logging.getLogger('<STR_LIT>')<EOL>handler = logging.FileHandler(log_file, '<STR_LIT:a>', '<STR_LIT:utf-8>')<EOL>formatter = logging.Formatter('<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>retval.addHandler(handler)<EOL>retval.setLevel(logging.WARNING)<EOL>return retval<EOL>

Creates .rollbar log file for use with rollbar-agent


def _report_exc_info(exc_info, request, extra_data, payload_data, level=None):

if not _check_config():<EOL><INDENT>return<EOL><DEDENT>filtered_level = _filtered_level(exc_info[<NUM_LIT:1>])<EOL>if level is None:<EOL><INDENT>level = filtered_level<EOL><DEDENT>filtered_exc_info = events.on_exception_info(exc_info,<EOL>request=request,<EOL>extra_data=extra_data,<EOL>payload_data=payload_data,<EOL>level=level)<EOL>if filtered_exc_info is False:<EOL><INDENT>return<EOL><DEDENT>cls, exc, trace = filtered_exc_info<EOL>data = _build_base_data(request)<EOL>if level is not None:<EOL><INDENT>data['<STR_LIT>'] = level<EOL><DEDENT>trace_chain = _walk_trace_chain(cls, exc, trace)<EOL>extra_trace_data = None<EOL>if len(trace_chain) > <NUM_LIT:1>:<EOL><INDENT>data['<STR_LIT:body>'] = {<EOL>'<STR_LIT>': trace_chain<EOL>}<EOL>if payload_data and ('<STR_LIT:body>' in payload_data) and ('<STR_LIT>' in payload_data['<STR_LIT:body>']):<EOL><INDENT>extra_trace_data = payload_data['<STR_LIT:body>']['<STR_LIT>']<EOL>del payload_data['<STR_LIT:body>']['<STR_LIT>']<EOL><DEDENT><DEDENT>else:<EOL><INDENT>data['<STR_LIT:body>'] = {<EOL>'<STR_LIT>': trace_chain[<NUM_LIT:0>]<EOL>}<EOL><DEDENT>if extra_data:<EOL><INDENT>extra_data = extra_data<EOL>if not isinstance(extra_data, dict):<EOL><INDENT>extra_data = {'<STR_LIT:value>': extra_data}<EOL><DEDENT>if extra_trace_data:<EOL><INDENT>extra_data = dict_merge(extra_data, extra_trace_data)<EOL><DEDENT>data['<STR_LIT>'] = extra_data<EOL><DEDENT>if extra_trace_data and not extra_data:<EOL><INDENT>data['<STR_LIT>'] = extra_trace_data<EOL><DEDENT>request = _get_actual_request(request)<EOL>_add_request_data(data, request)<EOL>_add_person_data(data, request)<EOL>_add_lambda_context_data(data)<EOL>data['<STR_LIT>'] = _build_server_data()<EOL>if payload_data:<EOL><INDENT>data = dict_merge(data, payload_data)<EOL><DEDENT>payload = _build_payload(data)<EOL>send_payload(payload, payload.get('<STR_LIT>'))<EOL>return data['<STR_LIT>']<EOL>


def _report_message(message, level, request, extra_data, payload_data):

if not _check_config():<EOL><INDENT>return<EOL><DEDENT>filtered_message = events.on_message(message,<EOL>request=request,<EOL>extra_data=extra_data,<EOL>payload_data=payload_data,<EOL>level=level)<EOL>if filtered_message is False:<EOL><INDENT>return<EOL><DEDENT>data = _build_base_data(request, level=level)<EOL>data['<STR_LIT:body>'] = {<EOL>'<STR_LIT:message>': {<EOL>'<STR_LIT:body>': filtered_message<EOL>}<EOL>}<EOL>if extra_data:<EOL><INDENT>extra_data = extra_data<EOL>data['<STR_LIT:body>']['<STR_LIT:message>'].update(extra_data)<EOL><DEDENT>request = _get_actual_request(request)<EOL>_add_request_data(data, request)<EOL>_add_person_data(data, request)<EOL>_add_lambda_context_data(data)<EOL>data['<STR_LIT>'] = _build_server_data()<EOL>if payload_data:<EOL><INDENT>data = dict_merge(data, payload_data)<EOL><DEDENT>payload = _build_payload(data)<EOL>send_payload(payload, payload.get('<STR_LIT>'))<EOL>return data['<STR_LIT>']<EOL>


if hasattr(request, '<STR_LIT>'):<EOL><INDENT>rollbar_person_prop = request.rollbar_person<EOL>try:<EOL><INDENT>person = rollbar_person_prop()<EOL><DEDENT>except TypeError:<EOL><INDENT>person = rollbar_person_prop<EOL><DEDENT>if person and isinstance(person, dict):<EOL><INDENT>return person<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>if hasattr(request, '<STR_LIT:user>'):<EOL><INDENT>user_prop = request.user<EOL>try:<EOL><INDENT>user = user_prop()<EOL><DEDENT>except TypeError:<EOL><INDENT>user = user_prop<EOL><DEDENT>if not user:<EOL><INDENT>return None<EOL><DEDENT>elif isinstance(user, dict):<EOL><INDENT>return user<EOL><DEDENT>else:<EOL><INDENT>retval = {}<EOL>if getattr(user, '<STR_LIT:id>', None):<EOL><INDENT>retval['<STR_LIT:id>'] = text(user.id)<EOL><DEDENT>elif getattr(user, '<STR_LIT>', None):<EOL><INDENT>retval['<STR_LIT:id>'] = text(user.user_id)<EOL><DEDENT>if retval.get('<STR_LIT:id>'):<EOL><INDENT>username = getattr(user, '<STR_LIT:username>', None)<EOL>email = getattr(user, '<STR_LIT:email>', None)<EOL>retval.update({<EOL>'<STR_LIT:username>': username,<EOL>'<STR_LIT:email>': email<EOL>})<EOL><DEDENT>return retval<EOL><DEDENT><DEDENT>if hasattr(request, '<STR_LIT>'):<EOL><INDENT>user_id_prop = request.user_id<EOL>try:<EOL><INDENT>user_id = user_id_prop()<EOL><DEDENT>except TypeError:<EOL><INDENT>user_id = user_id_prop<EOL><DEDENT>if not user_id:<EOL><INDENT>return None<EOL><DEDENT>return {'<STR_LIT:id>': text(user_id)}<EOL><DEDENT>

Returns a dictionary describing the logged-in user using data from `request.

Try request.rollbar_person first, then 'user', then 'user_id'


global _CURRENT_LAMBDA_CONTEXT<EOL>context = _CURRENT_LAMBDA_CONTEXT<EOL>if context is None:<EOL><INDENT>return<EOL><DEDENT>try:<EOL><INDENT>lambda_data = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': context.get_remaining_time_in_millis(),<EOL>'<STR_LIT>': context.function_name,<EOL>'<STR_LIT>': context.function_version,<EOL>'<STR_LIT>': context.invoked_function_arn,<EOL>'<STR_LIT>': context.aws_request_id,<EOL>}<EOL>}<EOL>if '<STR_LIT>' in data:<EOL><INDENT>data['<STR_LIT>'] = dict_merge(data['<STR_LIT>'], lambda_data)<EOL><DEDENT>else:<EOL><INDENT>data['<STR_LIT>'] = lambda_data<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>log.exception("<STR_LIT>", e)<EOL><DEDENT>finally:<EOL><INDENT>_CURRENT_LAMBDA_CONTEXT = None<EOL><DEDENT>

Attempts to add information from the lambda context if it exists


try:<EOL><INDENT>request_data = _build_request_data(request)<EOL><DEDENT>except Exception as e:<EOL><INDENT>log.exception("<STR_LIT>", e)<EOL><DEDENT>else:<EOL><INDENT>if request_data:<EOL><INDENT>_filter_ip(request_data, SETTINGS['<STR_LIT>'])<EOL>data['<STR_LIT>'] = request_data<EOL><DEDENT><DEDENT>

Attempts to build request data; if successful, sets the 'request' key on `data`.


def _check_add_locals(frame, frame_num, total_frames):

<EOL>return any(((frame_num == total_frames - <NUM_LIT:1>),<EOL>('<STR_LIT:root>' in SETTINGS and (frame.get('<STR_LIT:filename>') or '<STR_LIT>').lower().startswith((SETTINGS['<STR_LIT:root>'] or '<STR_LIT>').lower()))))<EOL>

Returns True if we should record local variables for the given frame.


<EOL>if WebobBaseRequest and isinstance(request, WebobBaseRequest):<EOL><INDENT>return _build_webob_request_data(request)<EOL><DEDENT>if DjangoHttpRequest and isinstance(request, DjangoHttpRequest):<EOL><INDENT>return _build_django_request_data(request)<EOL><DEDENT>if RestFrameworkRequest and isinstance(request, RestFrameworkRequest):<EOL><INDENT>return _build_django_request_data(request)<EOL><DEDENT>if WerkzeugRequest and isinstance(request, WerkzeugRequest):<EOL><INDENT>return _build_werkzeug_request_data(request)<EOL><DEDENT>if TornadoRequest and isinstance(request, TornadoRequest):<EOL><INDENT>return _build_tornado_request_data(request)<EOL><DEDENT>if BottleRequest and isinstance(request, BottleRequest):<EOL><INDENT>return _build_bottle_request_data(request)<EOL><DEDENT>if SanicRequest and isinstance(request, SanicRequest):<EOL><INDENT>return _build_sanic_request_data(request)<EOL><DEDENT>if FalconRequest and isinstance(request, FalconRequest):<EOL><INDENT>return _build_falcon_request_data(request)<EOL><DEDENT>if isinstance(request, dict) and '<STR_LIT>' in request:<EOL><INDENT>return _build_wsgi_request_data(request)<EOL><DEDENT>return None<EOL>

Returns a dictionary containing data from the request.
Can handle webob or werkzeug-based request objects.


<EOL>server_data = {<EOL>'<STR_LIT:host>': socket.gethostname(),<EOL>'<STR_LIT>': os.getpid()<EOL>}<EOL>argv = getattr(sys, '<STR_LIT>', None)<EOL>if argv:<EOL><INDENT>server_data['<STR_LIT>'] = argv<EOL><DEDENT>for key in ['<STR_LIT>', '<STR_LIT:root>']:<EOL><INDENT>if SETTINGS.get(key):<EOL><INDENT>server_data[key] = SETTINGS[key]<EOL><DEDENT><DEDENT>return server_data<EOL>

Returns a dictionary containing information about the server environment.


for k, v in iteritems(data):<EOL><INDENT>data[k] = _transform(v, key=(k,))<EOL><DEDENT>payload = {<EOL>'<STR_LIT>': SETTINGS['<STR_LIT>'],<EOL>'<STR_LIT:data>': data<EOL>}<EOL>return payload<EOL>


if not isinstance(b, dict):<EOL><INDENT>return b<EOL><DEDENT>result = a<EOL>for k, v in b.items():<EOL><INDENT>if k in result and isinstance(result[k], dict):<EOL><INDENT>result[k] = dict_merge(result[k], v)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>result[k] = copy.deepcopy(v)<EOL><DEDENT>except:<EOL><INDENT>result[k] = '<STR_LIT>' % (v,)<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

Recursively merges dict's. not just simple a['key'] = b['key'], if
both a and bhave a key who's value is a dict then dict_merge is called
on both values and the result stored in the returned dictionary.


self.notify_level = _checkLevel(level)<EOL>

Override so we set the effective level for which
log records we notify Rollbar about instead of which
records we save to the history.


logging.Handler.setLevel(self, level)<EOL>

Use this method to determine which records we record history
for. Use setLevel() to determine which level we report records
to Rollbar for.


def find_files(src_dir, pattern, skip_private=True):

for root, dirs, files in os.walk(src_dir):<EOL><INDENT>dirname = osp.basename(root)<EOL>if skip_private and dirname != '<STR_LIT:.>' and dirname.startswith('<STR_LIT:.>'):<EOL><INDENT>continue<EOL><DEDENT>for f in glob.fnmatch.filter(files, pattern):<EOL><INDENT>yield osp.join(root, f)<EOL><DEDENT><DEDENT>

Find all files recursively contained in `src_dir`

    Parameters
    ----------
    src_dir: str
        The toplevel directory
    pattern: str
        A file pattern suitable for glob
    skip_private: bool
        If True, directories with starting with single ``'.'`` are ignored


if six.PY2:<EOL><INDENT>from urlparse import urlparse as urlsplit<EOL><DEDENT>else:<EOL><INDENT>from urllib.parse import urlsplit<EOL><DEDENT>info = urlsplit(url)<EOL>domain = info.netloc<EOL>url_type = '<STR_LIT>' if domain == '<STR_LIT>' else '<STR_LIT:url>'<EOL>return '<STR_LIT>' % (url_type, info.path)<EOL>

Return the link to the Jupyter nbviewer for the given notebook url


return directives.choice(argument, ('<STR_LIT:left>', '<STR_LIT>', '<STR_LIT:right>'))<EOL>

Conversion function for the "align" option.


return self.THUMBNAIL_TEMPLATE.format(<EOL>snippet=self.get_description()[<NUM_LIT:1>], thumbnail=self.thumb_file,<EOL>ref_name=self.reference)<EOL>

The string for creating the thumbnail of this example


code_example = self.code_example<EOL>if code_example is None:<EOL><INDENT>return None<EOL><DEDENT>return self.CODE_TEMPLATE.format(<EOL>snippet=self.get_description()[<NUM_LIT:1>], code=code_example,<EOL>ref_name=self.reference)<EOL>

The string for creating a code example for the gallery


if self._code_example is not None:<EOL><INDENT>return self._code_example<EOL><DEDENT>return getattr(self.nb.metadata, '<STR_LIT>', None)<EOL>

The code example out of the notebook metadata


if self._supplementary_files is not None:<EOL><INDENT>return self._supplementary_files<EOL><DEDENT>return getattr(self.nb.metadata, '<STR_LIT>', None)<EOL>

The supplementary files of this notebook


if self._other_supplementary_files is not None:<EOL><INDENT>return self._other_supplementary_files<EOL><DEDENT>return getattr(self.nb.metadata, '<STR_LIT>', None)<EOL>


return '<STR_LIT>' + self.outfile.replace(os.path.sep, '<STR_LIT:_>').lower()<EOL>


if self._url is not None:<EOL><INDENT>url = self._url<EOL><DEDENT>else:<EOL><INDENT>url = getattr(self.nb.metadata, '<STR_LIT:url>', None)<EOL><DEDENT>if url is not None:<EOL><INDENT>return nbviewer_link(url)<EOL><DEDENT>

The url on jupyter nbviewer for this notebook or None if unknown


def __init__(self, infile, outfile, disable_warnings=True,<EOL>preprocess=True, clear=True, code_example=None,<EOL>supplementary_files=None, other_supplementary_files=None,<EOL>thumbnail_figure=None, url=None, insert_bokeh=False,<EOL>insert_bokeh_widgets=False, tag_options={}):

self.infile = infile<EOL>self.outfile = outfile<EOL>self.preprocess = preprocess<EOL>self.clear = clear<EOL>self._code_example = code_example<EOL>self._supplementary_files = supplementary_files<EOL>self._other_supplementary_files = other_supplementary_files<EOL>self._thumbnail_figure = thumbnail_figure<EOL>self._url = url<EOL>self.insert_bokeh = insert_bokeh<EOL>self.insert_bokeh_widgets = insert_bokeh_widgets<EOL>self.tag_options = tag_options<EOL>self.process_notebook(disable_warnings)<EOL>self.create_thumb()<EOL>

Parameters
----------
infile: str
    path to the existing notebook
outfile: str
    path to the new notebook
disable_warnings: bool
    Boolean to control whether warnings shall be included in the rst
    file or not
preprocess: bool
    If True, the notebook is processed when generating the rst file
clear: bool
    If True, the output in the download notebook is cleared
code_example: str
    A python code sample that shall be used instead of a thumbnail
    figure in the gallery. Note that you can also include a
    ``'code_example'`` key in the metadata of the notebook
supplementary_files: list of str
    Supplementary data files that shall be copied to the output
    directory and inserted in the rst file for download
other_supplementary_files: list of str
    Other supplementary data files that shall be copied but not
    inserted for download
thumbnail_figure: int
    The number of the figure that shall be used for download or a path
    to a file
url: str
    The url where to download the notebook
insert_bokeh: False or str
    The version string for bokeh to use for the style sheet
insert_bokeh_widgets: bool or str
    The version string for bokeh to use for the widgets style sheet
tag_options: dict
    A dictionary with traitlets for the
    :class:`nbconvert.preprocessors.TagRemovePreprocessor`


return os.path.splitext(self.outfile)[<NUM_LIT:0>] + os.path.extsep + ending<EOL>

get the output file with the specified `ending`


infile = self.infile<EOL>outfile = self.outfile<EOL>in_dir = os.path.dirname(infile) + os.path.sep<EOL>odir = os.path.dirname(outfile) + os.path.sep<EOL>create_dirs(os.path.join(odir, '<STR_LIT>'))<EOL>ep = nbconvert.preprocessors.ExecutePreprocessor(<EOL>timeout=<NUM_LIT>)<EOL>cp = nbconvert.preprocessors.ClearOutputPreprocessor(<EOL>timeout=<NUM_LIT>)<EOL>self.nb = nb = nbformat.read(infile, nbformat.current_nbformat)<EOL>if disable_warnings:<EOL><INDENT>for i, cell in enumerate(nb.cells):<EOL><INDENT>if cell['<STR_LIT>'] == '<STR_LIT:code>':<EOL><INDENT>cell = cell.copy()<EOL>break<EOL><DEDENT><DEDENT>cell = cell.copy()<EOL>cell.source =

Process the notebook and create all the pictures and files

        This method runs the notebook using the :mod:`nbconvert` and
        :mod:`nbformat` modules. It creates the :attr:`outfile` notebook,
        a python and a rst file


raw_rst, resources = nbconvert.export_by_name('<STR_LIT>', nb)<EOL>rst_content = '<STR_LIT>'<EOL>i0 = <NUM_LIT:0><EOL>m = None<EOL>bokeh_str = '<STR_LIT>'<EOL>if '<STR_LIT>' in raw_rst and self.insert_bokeh:<EOL><INDENT>bokeh_str += self.BOKEH_TEMPLATE.format(<EOL>version=self.insert_bokeh)<EOL><DEDENT>if '<STR_LIT>' in raw_rst and self.insert_bokeh_widgets:<EOL><INDENT>bokeh_str += self.BOKEH_WIDGETS_TEMPLATE.format(<EOL>version=self.insert_bokeh_widgets)<EOL><DEDENT>for m in code_blocks.finditer(raw_rst):<EOL><INDENT>lines = m.group().splitlines(True)<EOL>header, content = lines[<NUM_LIT:0>], '<STR_LIT>'.join(lines[<NUM_LIT:1>:])<EOL>no_magics = magic_patt.sub('<STR_LIT>', content)<EOL>if no_magics.strip():<EOL><INDENT>rst_content += (<EOL>raw_rst[i0:m.start()] + bokeh_str + header + no_magics)<EOL>bokeh_str = '<STR_LIT>'<EOL>i0 = m.end()<EOL><DEDENT>else:<EOL><INDENT>rst_content += raw_rst[i0:m.start()]<EOL>i0 = m.end()<EOL><DEDENT><DEDENT>if m is not None:<EOL><INDENT>rst_content += bokeh_str + raw_rst[m.end():]<EOL><DEDENT>else:<EOL><INDENT>rst_content = raw_rst<EOL><DEDENT>rst_content = '<STR_LIT>' % self.reference +rst_content<EOL>url = self.url<EOL>if url is not None:<EOL><INDENT>rst_content += self.CODE_DOWNLOAD_NBVIEWER.format(<EOL>pyfile=os.path.basename(self.py_file),<EOL>nbfile=os.path.basename(self.outfile),<EOL>url=url)<EOL><DEDENT>else:<EOL><INDENT>rst_content += self.CODE_DOWNLOAD.format(<EOL>pyfile=os.path.basename(self.py_file),<EOL>nbfile=os.path.basename(self.outfile))<EOL><DEDENT>supplementary_files = self.supplementary_files<EOL>other_supplementary_files = self.other_supplementary_files<EOL>if supplementary_files or other_supplementary_files:<EOL><INDENT>for f in (supplementary_files or []) + (<EOL>other_supplementary_files or []):<EOL><INDENT>if not os.path.exists(os.path.join(odir, f)):<EOL><INDENT>copyfile(os.path.join(in_dir, f), os.path.join(odir, f))<EOL><DEDENT><DEDENT><DEDENT>if supplementary_files:<EOL><INDENT>rst_content += self.data_download(supplementary_files)<EOL><DEDENT>rst_file = self.get_out_file()<EOL>outputs = sorted(resources['<STR_LIT>'], key=rst_content.find)<EOL>base = os.path.join('<STR_LIT>', os.path.splitext(<EOL>os.path.basename(self.infile))[<NUM_LIT:0>] + '<STR_LIT>')<EOL>out_map = {os.path.basename(original): base % i<EOL>for i, original in enumerate(outputs)}<EOL>for original, final in six.iteritems(out_map):<EOL><INDENT>rst_content = rst_content.replace(original, final)<EOL><DEDENT>with open(rst_file, '<STR_LIT:w>')as f:<EOL><INDENT>f.write(rst_content.rstrip() + '<STR_LIT:\n>')<EOL><DEDENT>pictures = []<EOL>for original in outputs:<EOL><INDENT>fname = os.path.join(odir, out_map[os.path.basename(original)])<EOL>pictures.append(fname)<EOL>if six.PY3:<EOL><INDENT>f = open(fname, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>f = open(fname, '<STR_LIT:w>')<EOL><DEDENT>f.write(resources['<STR_LIT>'][original])<EOL>f.close()<EOL><DEDENT>self.pictures = pictures<EOL>

Create the rst file from the notebook node


<EOL>if list(map(int, re.findall('<STR_LIT>', nbconvert.__version__))) >= [<NUM_LIT:4>, <NUM_LIT:2>]:<EOL><INDENT>py_file = os.path.basename(self.py_file)<EOL><DEDENT>else:<EOL><INDENT>py_file = self.py_file<EOL><DEDENT>try:<EOL><INDENT>level = logger.logger.level<EOL><DEDENT>except AttributeError:<EOL><INDENT>level = logger.level<EOL><DEDENT>spr.call(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>' + py_file, '<STR_LIT>' % level,<EOL>self.outfile])<EOL>with open(self.py_file) as f:<EOL><INDENT>py_content = f.read()<EOL><DEDENT>py_content = re.sub('<STR_LIT>', '<STR_LIT>',<EOL>py_content, flags=re.MULTILINE)<EOL>with open(self.py_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(py_content)<EOL><DEDENT>

Create the python script from the notebook node


if len(files) > <NUM_LIT:1>:<EOL><INDENT>return self.DATA_DOWNLOAD % (<EOL>('<STR_LIT>' + '<STR_LIT:U+0020>'*<NUM_LIT:8>) + ('<STR_LIT:\n>' + '<STR_LIT:U+0020>'*<NUM_LIT:8>).join(<EOL>'<STR_LIT>' % f for f in files))<EOL><DEDENT>return self.DATA_DOWNLOAD % '<STR_LIT>' % files[<NUM_LIT:0>]<EOL>

Create the rst string to download supplementary data


thumbnail_figure = self.copy_thumbnail_figure()<EOL>if thumbnail_figure is not None:<EOL><INDENT>if isinstance(thumbnail_figure, six.string_types):<EOL><INDENT>pic = thumbnail_figure<EOL><DEDENT>else:<EOL><INDENT>pic = self.pictures[thumbnail_figure]<EOL><DEDENT>self.save_thumbnail(pic)<EOL><DEDENT>else:<EOL><INDENT>for pic in self.pictures[::-<NUM_LIT:1>]:<EOL><INDENT>if pic.endswith('<STR_LIT>'):<EOL><INDENT>self.save_thumbnail(pic)<EOL>return<EOL><DEDENT><DEDENT><DEDENT>


def split_header(s, get_header=True):<EOL><INDENT>s = s.lstrip().rstrip()<EOL>parts = s.splitlines()<EOL>if parts[<NUM_LIT:0>].startswith('<STR_LIT:#>'):<EOL><INDENT>if get_header:<EOL><INDENT>header = re.sub('<STR_LIT>', '<STR_LIT>', parts.pop(<NUM_LIT:0>))<EOL>if not parts:<EOL><INDENT>return header, '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>header = '<STR_LIT>'<EOL><DEDENT>rest = '<STR_LIT:\n>'.join(parts).lstrip().split('<STR_LIT>')<EOL>desc = rest[<NUM_LIT:0>].replace('<STR_LIT:\n>', '<STR_LIT:U+0020>')<EOL>return header, desc<EOL><DEDENT>else:<EOL><INDENT>if get_header:<EOL><INDENT>if parts[<NUM_LIT:0>].startswith(('<STR_LIT:=>', '<STR_LIT:->')):<EOL><INDENT>parts = parts[<NUM_LIT:1>:]<EOL><DEDENT>header = parts.pop(<NUM_LIT:0>)<EOL>if parts and parts[<NUM_LIT:0>].startswith(('<STR_LIT:=>', '<STR_LIT:->')):<EOL><INDENT>parts.pop(<NUM_LIT:0>)<EOL><DEDENT>if not parts:<EOL><INDENT>return header, '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>header = '<STR_LIT>'<EOL><DEDENT>rest = '<STR_LIT:\n>'.join(parts).lstrip().split('<STR_LIT>')<EOL>desc = rest[<NUM_LIT:0>].replace('<STR_LIT:\n>', '<STR_LIT:U+0020>')<EOL>return header, desc<EOL><DEDENT><DEDENT>first_cell = self.nb['<STR_LIT>'][<NUM_LIT:0>]<EOL>if not first_cell['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>', '<STR_LIT>'<EOL><DEDENT>header, desc = split_header(first_cell['<STR_LIT:source>'])<EOL>if not desc and len(self.nb['<STR_LIT>']) > <NUM_LIT:1>:<EOL><INDENT>second_cell = self.nb['<STR_LIT>'][<NUM_LIT:1>]<EOL>if second_cell['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>_, desc = split_header(second_cell['<STR_LIT:source>'], False)<EOL><DEDENT><DEDENT>return header, desc<EOL>

Get summary and description of this notebook


def scale_image(self, in_fname, out_fname, max_width, max_height):

<EOL>try:<EOL><INDENT>from PIL import Image<EOL><DEDENT>except ImportError:<EOL><INDENT>import Image<EOL><DEDENT>img = Image.open(in_fname)<EOL>width_in, height_in = img.size<EOL>scale_w = max_width / float(width_in)<EOL>scale_h = max_height / float(height_in)<EOL>if height_in * scale_w <= max_height:<EOL><INDENT>scale = scale_w<EOL><DEDENT>else:<EOL><INDENT>scale = scale_h<EOL><DEDENT>if scale >= <NUM_LIT:1.0> and in_fname == out_fname:<EOL><INDENT>return<EOL><DEDENT>width_sc = int(round(scale * width_in))<EOL>height_sc = int(round(scale * height_in))<EOL>img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)<EOL>thumb = Image.new('<STR_LIT>', (max_width, max_height), (<NUM_LIT:255>, <NUM_LIT:255>, <NUM_LIT:255>))<EOL>pos_insert = (<EOL>(max_width - width_sc) // <NUM_LIT:2>, (max_height - height_sc) // <NUM_LIT:2>)<EOL>thumb.paste(img, pos_insert)<EOL>thumb.save(out_fname)<EOL>

Scales an image with the same aspect ratio centered in an
           image with a given max_width and max_height
           if in_fname == out_fname the image can only be scaled down


thumb_dir = os.path.join(os.path.dirname(image_path), '<STR_LIT>')<EOL>create_dirs(thumb_dir)<EOL>thumb_file = os.path.join(thumb_dir,<EOL>'<STR_LIT>' % self.reference)<EOL>if os.path.exists(image_path):<EOL><INDENT>logger.info('<STR_LIT>', image_path, thumb_file)<EOL>self.scale_image(image_path, thumb_file, <NUM_LIT>, <NUM_LIT>)<EOL><DEDENT>self.thumb_file = thumb_file<EOL>


return os.path.relpath(self.thumb_file, base_dir)<EOL>

Get the relative path to the thumb nail of this notebook


ret = None<EOL>if self._thumbnail_figure is not None:<EOL><INDENT>if not isstring(self._thumbnail_figure):<EOL><INDENT>ret = self._thumbnail_figure<EOL><DEDENT>else:<EOL><INDENT>ret = osp.join(osp.dirname(self.outfile),<EOL>osp.basename(self._thumbnail_figure))<EOL>copyfile(self._thumbnail_figure, ret)<EOL>return ret<EOL><DEDENT><DEDENT>elif hasattr(self.nb.metadata, '<STR_LIT>'):<EOL><INDENT>if not isstring(self.nb.metadata.thumbnail_figure):<EOL><INDENT>ret = self.nb.metadata.thumbnail_figure<EOL><DEDENT>else:<EOL><INDENT>ret = osp.join(osp.dirname(self.outfile), '<STR_LIT>',<EOL>osp.basename(self.nb.metadata.thumbnail_figure))<EOL>copyfile(osp.join(osp.dirname(self.infile),<EOL>self.nb.metadata.thumbnail_figure),<EOL>ret)<EOL><DEDENT><DEDENT>return ret<EOL>


def __init__(self, examples_dirs=['<STR_LIT>'], gallery_dirs=None,<EOL>pattern='<STR_LIT>', disable_warnings=True,<EOL>dont_preprocess=[], preprocess=True, clear=True,<EOL>dont_clear=[], code_examples={}, supplementary_files={},<EOL>other_supplementary_files={}, thumbnail_figures={},<EOL>urls=None, insert_bokeh=False, insert_bokeh_widgets=False,<EOL>remove_all_outputs_tags=set(), remove_cell_tags=set(),<EOL>remove_input_tags=set(), remove_single_output_tags=set()):

if isinstance(examples_dirs, six.string_types):<EOL><INDENT>examples_dirs = [examples_dirs]<EOL><DEDENT>if gallery_dirs is None:<EOL><INDENT>gallery_dirs = list(map(os.path.basename, examples_dirs))<EOL><DEDENT>if isinstance(gallery_dirs, six.string_types):<EOL><INDENT>gallery_dirs = [gallery_dirs]<EOL><DEDENT>for i, s in enumerate(examples_dirs):<EOL><INDENT>if not s.endswith(os.path.sep):<EOL><INDENT>examples_dirs[i] += os.path.sep<EOL><DEDENT><DEDENT>for i, s in enumerate(gallery_dirs):<EOL><INDENT>if not s.endswith(os.path.sep):<EOL><INDENT>gallery_dirs[i] += os.path.sep<EOL><DEDENT><DEDENT>self.in_dir = examples_dirs<EOL>self.out_dir = gallery_dirs<EOL>if isinstance(pattern, six.string_types):<EOL><INDENT>pattern = re.compile(pattern)<EOL><DEDENT>self.pattern = pattern<EOL>self.disable_warnings = disable_warnings<EOL>self.dont_preprocess = dont_preprocess<EOL>self.preprocess = preprocess<EOL>self.clear = clear<EOL>self.dont_clear = dont_clear<EOL>self.code_examples = code_examples<EOL>self.supplementary_files = supplementary_files<EOL>self.osf = other_supplementary_files<EOL>self.thumbnail_figures = thumbnail_figures<EOL>if urls is None or isinstance(urls, (dict, six.string_types)):<EOL><INDENT>urls = [urls] * len(self.in_dir)<EOL><DEDENT>self._all_urls = urls<EOL>if insert_bokeh and not isstring(insert_bokeh):<EOL><INDENT>import bokeh<EOL>insert_bokeh = bokeh.__version__<EOL><DEDENT>if insert_bokeh_widgets and not isstring(insert_bokeh_widgets):<EOL><INDENT>import bokeh<EOL>insert_bokeh_widgets = bokeh.__version__<EOL><DEDENT>tag_options = {<EOL>'<STR_LIT>': remove_all_outputs_tags,<EOL>'<STR_LIT>': remove_cell_tags,<EOL>'<STR_LIT>': remove_input_tags,<EOL>'<STR_LIT>': remove_single_output_tags}<EOL>self._nbp_kws = {'<STR_LIT>': insert_bokeh,<EOL>'<STR_LIT>': insert_bokeh_widgets,<EOL>'<STR_LIT>': tag_options}<EOL>

Parameters
----------
examples_dirs: list of str
    list containing the directories to loop through. Default:
    ``['../examples']``
gallerys_dirs: list of str
    None or list of directories where the rst files shall be created.
    If None, the current working directory and the name of the
    corresponding directory in the `examples_dirs` is used. Default:
    ``None``
pattern: list of str
    str. The pattern to use to find the ipython  notebooks.
    Default: ``'example_.+.ipynb'``
disable_warnings: bool
    Boolean controlling whether warnings shall be disabled when
    processing the examples. Defaultt: True
preprocess: bool or list of str
    If True, all examples (except those specified in the
    `dont_preprocess` item) will be preprocessed when creating the rst
    files. Otherwise it might be a list of files that shall be
    preprocessed.
dont_preprocess: bool or list of str
    If True, no example will be preprocessed when creating the rst
    files. Otherwise it might be a list of files that shall not be
    preprocessed
clear: bool or list of str
    If True, the output in all notebooks to download will be cleared.
    Otherwise it might be a list of notebook files of whom to clear the
    output
dont_clear: bool or list of str
    If True, the output in all notebooks to download will not be
    cleared. Otherwise it might be a list of notebook files  of whom
    not to clear the output
code_examples: dict
    A mapping from filename to code samples that shall be used instead
    of a thumbnail figure in the gallery. Note that you can also
    include a  ``'code_example'`` key in the metadata of the notebook
supplementary_files: dict
    A mapping from filename to a list of supplementary data files that
    shall copied to the documentation directory and can be downloaded.
    Note that you can also include a  ``'supplementary_files'`` key in
    the metadata of the notebook
other_supplementary_files: dict
    A mapping from filename to a list of other supplementary data files
    that shall copied to the documentation directory but can not be
    downloaded (e.g. pictures).
    Note that you can also include a  ``'other_supplementary_files'``
    key in the metadata of the notebook
thumbnail_figures: dict
    A mapping from filename to an integer or the path of a file to use
    for the thumbnail
urls: str or dict
    The urls where to download the notebook. Necessary to provide a
    link to the jupyter nbviewer. If string, the path to the notebook
    will be appended for each example notebook. Otherwise it should be
    a dictionary with links for the given notebook
insert_bokeh: bool or str
    If True, the bokeh js [1]_ and the stylesheet [2]_ are inserted in
    the notebooks that have bokeh loaded (using the installed or
    specified bokeh version)
insert_bokeh_widgets: bool or str
    If True, the bokeh widget js [3]_ is inserted in the notebooks that
    have bokeh loaded (using the installed or specified bokeh version)
remove_all_outputs_tags: set
    Tags indicating cells for which the outputs are to be removed,
    matches tags in cell.metadata.tags.
remove_cell_tags: set
    Tags indicating which cells are to be removed, matches tags in
    cell.metadata.tags.
remove_input_tags: set
    Tags indicating cells for which input is to be removed,
    matches tags in cell.metadata.tags.
remove_single_output_tags: set
    Tags indicating which individual outputs are to be removed, matches
    output i tags in cell.outputs[i].metadata.tags.

References
----------
.. [1] http://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js
.. [2] http://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css
.. [3] http://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js


for i, (base_dir, target_dir, paths) in enumerate(zip(<EOL>self.in_dir, self.out_dir, map(os.walk, self.in_dir))):<EOL><INDENT>self._in_dir_count = i<EOL>self.recursive_processing(base_dir, target_dir, paths)<EOL><DEDENT>

Create the rst files from the input directories in the
        :attr:`in_dir` attribute


def recursive_processing(self, base_dir, target_dir, it):

try:<EOL><INDENT>file_dir, dirs, files = next(it)<EOL><DEDENT>except StopIteration:<EOL><INDENT>return '<STR_LIT>', []<EOL><DEDENT>readme_files = {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}<EOL>if readme_files.intersection(files):<EOL><INDENT>foutdir = file_dir.replace(base_dir, target_dir)<EOL>create_dirs(foutdir)<EOL>this_nbps = [<EOL>NotebookProcessor(<EOL>infile=f,<EOL>outfile=os.path.join(foutdir, os.path.basename(f)),<EOL>disable_warnings=self.disable_warnings,<EOL>preprocess=(<EOL>(self.preprocess is True or f in self.preprocess) and<EOL>not (self.dont_preprocess is True or<EOL>f in self.dont_preprocess)),<EOL>clear=((self.clear is True or f in self.clear) and not<EOL>(self.dont_clear is True or f in self.dont_clear)),<EOL>code_example=self.code_examples.get(f),<EOL>supplementary_files=self.supplementary_files.get(f),<EOL>other_supplementary_files=self.osf.get(f),<EOL>thumbnail_figure=self.thumbnail_figures.get(f),<EOL>url=self.get_url(f.replace(base_dir, '<STR_LIT>')),<EOL>**self._nbp_kws)<EOL>for f in map(lambda f: os.path.join(file_dir, f),<EOL>filter(self.pattern.match, files))]<EOL>readme_file = next(iter(readme_files.intersection(files)))<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>', []<EOL><DEDENT>labels = OrderedDict()<EOL>this_label = '<STR_LIT>' + foutdir.replace(os.path.sep, '<STR_LIT:_>')<EOL>if this_label.endswith('<STR_LIT:_>'):<EOL><INDENT>this_label = this_label[:-<NUM_LIT:1>]<EOL><DEDENT>for d in dirs:<EOL><INDENT>label, nbps = self.recursive_processing(<EOL>base_dir, target_dir, it)<EOL>if label:<EOL><INDENT>labels[label] = nbps<EOL><DEDENT><DEDENT>s = "<STR_LIT>" % this_label<EOL>with open(os.path.join(file_dir, readme_file)) as f:<EOL><INDENT>s += f.read().rstrip() + '<STR_LIT>'<EOL><DEDENT>s += "<STR_LIT>"<EOL>s += '<STR_LIT>'.join('<STR_LIT>' % os.path.splitext(os.path.basename(<EOL>nbp.get_out_file()))[<NUM_LIT:0>] for nbp in this_nbps)<EOL>for d in dirs:<EOL><INDENT>findex = os.path.join(d, '<STR_LIT>')<EOL>if os.path.exists(os.path.join(foutdir, findex)):<EOL><INDENT>s += '<STR_LIT>' % os.path.splitext(findex)[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>s += '<STR_LIT:\n>'<EOL>for nbp in this_nbps:<EOL><INDENT>code_div = nbp.code_div<EOL>if code_div is not None:<EOL><INDENT>s += code_div + '<STR_LIT:\n>'<EOL><DEDENT>else:<EOL><INDENT>s += nbp.thumbnail_div + '<STR_LIT:\n>'<EOL><DEDENT><DEDENT>s += "<STR_LIT>"<EOL>for label, nbps in labels.items():<EOL><INDENT>s += '<STR_LIT>' % (<EOL>label)<EOL>for nbp in nbps:<EOL><INDENT>code_div = nbp.code_div<EOL>if code_div is not None:<EOL><INDENT>s += code_div + '<STR_LIT:\n>'<EOL><DEDENT>else:<EOL><INDENT>s += nbp.thumbnail_div + '<STR_LIT:\n>'<EOL><DEDENT><DEDENT>s += "<STR_LIT>"<EOL><DEDENT>s += '<STR_LIT:\n>'<EOL>with open(os.path.join(foutdir, '<STR_LIT>'), '<STR_LIT:w>') as f:<EOL><INDENT>f.write(s)<EOL><DEDENT>return this_label, list(chain(this_nbps, *labels.values()))<EOL>

Method to recursivly process the notebooks in the `base_dir`

        Parameters
        ----------
        base_dir: str
            Path to the base example directory (see the `examples_dir`
            parameter for the :class:`Gallery` class)
        target_dir: str
            Path to the output directory for the rst files (see the
            `gallery_dirs` parameter for the :class:`Gallery` class)
        it: iterable
            The iterator over the subdirectories and files in `base_dir`
            generated by the :func:`os.walk` function


app.config.html_static_path.append(os.path.join(<EOL>os.path.dirname(__file__), '<STR_LIT>'))<EOL>config = app.config.example_gallery_config<EOL>insert_bokeh = config.get('<STR_LIT>')<EOL>if insert_bokeh:<EOL><INDENT>if not isstring(insert_bokeh):<EOL><INDENT>import bokeh<EOL>insert_bokeh = bokeh.__version__<EOL><DEDENT>app.add_stylesheet(<EOL>NotebookProcessor.BOKEH_STYLE_SHEET.format(<EOL>version=insert_bokeh))<EOL>app.add_javascript(<EOL>NotebookProcessor.BOKEH_JS.format(version=insert_bokeh))<EOL><DEDENT>insert_bokeh_widgets = config.get('<STR_LIT>')<EOL>if insert_bokeh_widgets:<EOL><INDENT>if not isstring(insert_bokeh_widgets):<EOL><INDENT>import bokeh<EOL>insert_bokeh_widgets = bokeh.__version__<EOL><DEDENT>app.add_stylesheet(<EOL>NotebookProcessor.BOKEH_WIDGETS_STYLE_SHEET.format(<EOL>version=insert_bokeh_widgets))<EOL>app.add_javascript(<EOL>NotebookProcessor.BOKEH_WIDGETS_JS.format(<EOL>version=insert_bokeh_widgets))<EOL><DEDENT>if not app.config.process_examples:<EOL><INDENT>return<EOL><DEDENT>cls(**app.config.example_gallery_config).process_directories()<EOL>

Class method to create a :class:`Gallery` instance from the
        configuration of a sphinx application


urls = self.urls<EOL>if isinstance(urls, dict):<EOL><INDENT>return urls.get(nbfile)<EOL><DEDENT>elif isstring(urls):<EOL><INDENT>if not urls.endswith('<STR_LIT:/>'):<EOL><INDENT>urls += '<STR_LIT:/>'<EOL><DEDENT>return urls + nbfile<EOL><DEDENT>

Return the url corresponding to the given notebook file

        Parameters
        ----------
        nbfile: str
            The path of the notebook relative to the corresponding
            :attr:``in_dir``

        Returns
        -------
        str or None
            The url or None if no url has been specified


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters = ['<STR_LIT>']<EOL>output_names = ['<STR_LIT>', '<STR_LIT>']<EOL>if r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT:2>', '<STR_LIT>'])<EOL>output_names.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>output_files = [os.path.join(work_dir, x) for x in output_names]<EOL>tarball_files(tar_name='<STR_LIT>', file_paths=output_files, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Run Fastqc on the input reads

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq read 1
:param str r2_id: FileStoreID of fastq read 2
:return: FileStoreID of fastQC output (tarball)
:rtype: str


def run_cutadapt(job, r1_id, r2_id, fwd_3pr_adapter, rev_3pr_adapter):

work_dir = job.fileStore.getLocalTempDir()<EOL>if r2_id:<EOL><INDENT>require(rev_3pr_adapter, "<STR_LIT>")<EOL><DEDENT>parameters = ['<STR_LIT>', fwd_3pr_adapter,<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if r1_id and r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', rev_3pr_adapter,<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>if r1_id and r2_id:<EOL><INDENT>r1_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>r2_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>r1_cut_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>r2_cut_id = None<EOL><DEDENT>return r1_cut_id, r2_cut_id<EOL>

Adapter trimming for RNA-seq data

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq read 1
:param str r2_id: FileStoreID of fastq read 2 (if paired data)
:param str fwd_3pr_adapter: Adapter sequence for the forward 3' adapter
:param str rev_3pr_adapter: Adapter sequence for the reverse 3' adapter (second fastq pair)
:return: R1 and R2 FileStoreIDs
:rtype: tuple


job.fileStore.logToMaster('<STR_LIT>')<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(ref_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=command,<EOL>tool='<STR_LIT>')<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Use SAMtools to create reference index file

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str ref_id: FileStoreID for the reference genome
:return: FileStoreID for reference index
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters = ['<STR_LIT:index>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=parameters,<EOL>tool='<STR_LIT>')<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Runs SAMtools index to create a BAM index file

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID of the BAM file
:return: FileStoreID for BAM index file
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=parameters,<EOL>tool='<STR_LIT>')<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Runs SAMtools view to create a SAM file.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID of the BAM file
:return: FileStoreID for SAM file
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Sorts BAM file using SAMtools sort

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:return: FileStoreID for sorted BAM file
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Mark reads as PCR duplicates using SAMtools rmdup

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:return: FileStoreID for sorted BAM file
:rtype: str


def run_sambamba_index(job, bam, sort_by_name=False):

work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT:index>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Indexes a BAM file using Sambamba.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:return: FileStoreID for sorted BAM file
:rtype: str


def run_sambamba_sort(job, bam, sort_by_name=False):

work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>', str(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>if sort_by_name:<EOL><INDENT>command.append('<STR_LIT>')<EOL><DEDENT>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Sorts BAM file using Sambamba sort

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param boolean sort_by_name: If true, sorts by read name instead of coordinate.
:return: FileStoreID for sorted BAM file
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Marks reads as PCR duplicates using Sambamba

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:return: FileStoreID for sorted BAM file
:rtype: str


work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(sam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>']<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Marks reads as PCR duplicates using SAMBLASTER

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str sam: FileStoreID for SAM file
:return: FileStoreID for deduped SAM file
:rtype: str


job.fileStore.logToMaster('<STR_LIT>')<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(ref_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>')<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Uses Picard to create reference sequence dictionary

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str ref_id: FileStoreID for the reference genome fasta file
:return: FileStoreID for sequence dictionary file
:rtype: str


def picard_mark_duplicates(job, bam, bai, validation_stringency='<STR_LIT>'):

work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(bai, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % validation_stringency.upper()]<EOL>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return bam, bai<EOL>

Runs Picard MarkDuplicates on a BAM file. Requires that the BAM file be coordinate sorted.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str validation_stringency: BAM file validation stringency, default is LENIENT
:return: FileStoreIDs for BAM and BAI files
:rtype: tuple


def run_picard_sort(job, bam, sort_by_name=False):

work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(bam, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']<EOL>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>if sort_by_name:<EOL><INDENT>command.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>command.append('<STR_LIT>')<EOL><DEDENT>start_time = time.time()<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Sorts BAM file using Picard SortSam

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param boolean sort_by_name: If true, sorts by read name instead of coordinate.
:return: FileStoreID for sorted BAM file
:rtype: str


def run_gatk_preprocessing(job, bam, bai, ref, ref_dict, fai, g1k, mills, dbsnp, realign=False, unsafe=False):

<EOL>mdups_disk = PromisedRequirement(lambda bam_, bai_: <NUM_LIT:2> * (bam_.size + bai_.size), bam, bai)<EOL>mdups = job.wrapJobFn(picard_mark_duplicates,<EOL>bam,<EOL>bai,<EOL>cores=job.cores,<EOL>disk=mdups_disk,<EOL>memory=job.memory)<EOL>bqsr_input_bam = mdups.rv(<NUM_LIT:0>)<EOL>bqsr_input_bai = mdups.rv(<NUM_LIT:1>)<EOL>genome_ref_size = ref.size + ref_dict.size + fai.size<EOL>if realign:<EOL><INDENT>indel_ref_size = mills.size + g1k.size + genome_ref_size<EOL>realigner_target_disk = PromisedRequirement(lambda bam_, bai_, ref_size:<EOL>bam_.size + bai_.size + <NUM_LIT:2> * ref_size,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>indel_ref_size)<EOL>realigner_target = job.wrapJobFn(run_realigner_target_creator,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>ref, ref_dict, fai,<EOL>g1k, mills,<EOL>unsafe=unsafe,<EOL>cores=<NUM_LIT:1>,  <EOL>disk=realigner_target_disk,<EOL>memory=job.memory)<EOL>indel_realign_disk = PromisedRequirement(lambda bam_, bai_, intervals, ref_size:<EOL><NUM_LIT:2> * (bam_.size + bai_.size) + intervals.size + ref_size,<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>realigner_target.rv(),<EOL>indel_ref_size)<EOL>indel_realign = job.wrapJobFn(run_indel_realignment,<EOL>realigner_target.rv(),<EOL>mdups.rv(<NUM_LIT:0>),<EOL>mdups.rv(<NUM_LIT:1>),<EOL>ref, ref_dict, fai,<EOL>g1k, mills,<EOL>unsafe=unsafe,<EOL>cores=<NUM_LIT:1>,  <EOL>disk=indel_realign_disk,<EOL>memory=job.memory)<EOL>mdups.addChild(realigner_target)<EOL>realigner_target.addChild(indel_realign)<EOL>bqsr_input_bam = indel_realign.rv(<NUM_LIT:0>)<EOL>bqsr_input_bai = indel_realign.rv(<NUM_LIT:1>)<EOL><DEDENT>bqsr_ref_size = dbsnp.size + mills.size + genome_ref_size<EOL>base_recal_disk = PromisedRequirement(lambda bam_, bai_, ref_size:<EOL>bam_.size + bai_.size + <NUM_LIT:2> * ref_size,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>bqsr_ref_size)<EOL>base_recal = job.wrapJobFn(run_base_recalibration,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>ref, ref_dict, fai,<EOL>dbsnp, mills,<EOL>unsafe=unsafe,<EOL>cores=job.cores,<EOL>disk=base_recal_disk,<EOL>memory=job.memory)<EOL>recalibrate_reads_disk = PromisedRequirement(lambda bam_, bai_, recal, ref_size:<EOL><NUM_LIT:2> * (bam_.size + bai_.size) + recal.size + ref_size,<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>base_recal.rv(),<EOL>genome_ref_size)<EOL>recalibrate_reads = job.wrapJobFn(apply_bqsr_recalibration,<EOL>base_recal.rv(),<EOL>bqsr_input_bam,<EOL>bqsr_input_bai,<EOL>ref, ref_dict, fai,<EOL>unsafe=unsafe,<EOL>cores=job.cores,<EOL>disk=recalibrate_reads_disk,<EOL>memory=job.memory)<EOL>job.addChild(mdups)<EOL>mdups.addFollowOn(base_recal)<EOL>base_recal.addChild(recalibrate_reads)<EOL>return recalibrate_reads.rv(<NUM_LIT:0>), recalibrate_reads.rv(<NUM_LIT:1>)<EOL>

GATK Preprocessing Pipeline
0: Mark duplicates
1: Create INDEL realignment intervals
2: Realign INDELs
3: Recalibrate base quality scores
4: Apply base score recalibration

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference sequence dictionary file
:param str fai: FileStoreID for reference fasta index file
:param str g1k: FileStoreID for 1000 Genomes VCF file
:param str mills: FileStoreID for Mills VCF file
:param str dbsnp: FileStoreID for dbSNP VCF file
:param bool realign: If True, then runs GATK INDEL realignment"
:param bool unsafe: If True, runs GATK tools in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreIDs for BAM and BAI files
:rtype: tuple(str, str)


def run_realigner_target_creator(job, bam, bai, ref, ref_dict, fai, g1k, mills, unsafe=False):

inputs = {'<STR_LIT>': ref,<EOL>'<STR_LIT>': fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': bam,<EOL>'<STR_LIT>': bai,<EOL>'<STR_LIT>': g1k,<EOL>'<STR_LIT>': mills}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>if unsafe:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir,<EOL>parameters=parameters,<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Creates intervals file needed for INDEL realignment

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference sequence dictionary file
:param str fai: FileStoreID for reference fasta index file
:param str g1k: FileStoreID for 1000 Genomes VCF file
:param str mills: FileStoreID for Mills VCF file
:param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreID for realignment intervals file
:rtype: str


def run_indel_realignment(job, intervals, bam, bai, ref, ref_dict, fai, g1k, mills, unsafe=False):

inputs = {'<STR_LIT>': ref,<EOL>'<STR_LIT>': fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': bam,<EOL>'<STR_LIT>': bai,<EOL>'<STR_LIT>': intervals,<EOL>'<STR_LIT>': g1k,<EOL>'<STR_LIT>': mills}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if unsafe:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir,<EOL>parameters=parameters,<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>indel_bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>indel_bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return indel_bam, indel_bai<EOL>

Realigns BAM file at realignment target intervals

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str intervals: FileStoreID for INDEL realignment intervals file
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference sequence dictionary file
:param str fai: FileStoreID for reference fasta index file
:param str g1k: FileStoreID for 1000 Genomes VCF file
:param str mills: FileStoreID for Mills VCF file
:param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreIDs for realigned BAM and BAI files
:rtype: tuple(str, str)


def run_base_recalibration(job, bam, bai, ref, ref_dict, fai, dbsnp, mills, unsafe=False):

inputs = {'<STR_LIT>': ref,<EOL>'<STR_LIT>': fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': bam,<EOL>'<STR_LIT>': bai,<EOL>'<STR_LIT>': dbsnp,<EOL>'<STR_LIT>': mills}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if unsafe:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir,<EOL>parameters=parameters,<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Creates recalibration table for Base Quality Score Recalibration

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference genome sequence dictionary file
:param str fai: FileStoreID for reference genome fasta index file
:param str dbsnp: FileStoreID for dbSNP VCF file
:param str mills: FileStoreID for Mills VCF file
:param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreID for the recalibration table file
:rtype: str


def apply_bqsr_recalibration(job, table, bam, bai, ref, ref_dict, fai, unsafe=False):

inputs = {'<STR_LIT>': ref,<EOL>'<STR_LIT>': fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': table,<EOL>'<STR_LIT>': bam,<EOL>'<STR_LIT>': bai}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(int(job.cores)),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>if unsafe:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>docker_parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory),<EOL>'<STR_LIT>', '<STR_LIT>'.format(work_dir)]<EOL>start_time = time.time()<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir,<EOL>parameters=parameters,<EOL>dockerParameters=docker_parameters)<EOL>end_time = time.time()<EOL>_log_runtime(job, start_time, end_time, "<STR_LIT>")<EOL>output_bam = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>output_bai = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return output_bam, output_bai<EOL>

Creates BAM file with recalibrated base quality scores

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str table: FileStoreID for BQSR recalibration table file
:param str bam: FileStoreID for BAM file
:param str bai: FileStoreID for BAM index file
:param str ref: FileStoreID for reference genome fasta file
:param str ref_dict: FileStoreID for reference genome sequence dictionary file
:param str fai: FileStoreID for reference genome fasta index file
:param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
:return: FileStoreIDs for recalibrated BAM and BAI files
:rtype: tuple(str, str)


job.fileStore.logToMaster('<STR_LIT>')<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(ref_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT:index>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=command,<EOL>tool='<STR_LIT>')<EOL>ids = {}<EOL>for output in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>ids[output.split('<STR_LIT:.>')[-<NUM_LIT:1>]] = (job.fileStore.writeGlobalFile(os.path.join(work_dir, output)))<EOL><DEDENT>return ids['<STR_LIT>'], ids['<STR_LIT>'], ids['<STR_LIT>'], ids['<STR_LIT>'], ids['<STR_LIT>']<EOL>

Use BWA to create reference index files

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str ref_id: FileStoreID for the reference genome
:return: FileStoreIDs for BWA index files
:rtype: tuple(str, str, str, str, str)


job.fileStore.logToMaster('<STR_LIT>')<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>job.fileStore.readGlobalFile(ref_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=command,<EOL>tool='<STR_LIT>')<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Use Samtools to create reference index file

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str ref_id: FileStoreID for the reference genome
:return: FileStoreID for reference index
:rtype: str


def run_kallisto(job, r1_id, r2_id, kallisto_index_url):

work_dir = job.fileStore.getLocalTempDir()<EOL>download_url(job, url=kallisto_index_url, name='<STR_LIT>', work_dir=work_dir)<EOL>parameters = ['<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(job.cores),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:100>',<EOL>'<STR_LIT>']<EOL>if r1_id and r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>output_files = [os.path.join(work_dir, x) for x in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']]<EOL>tarball_files(tar_name='<STR_LIT>', file_paths=output_files, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

RNA quantification via Kallisto

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq (pair 1)
:param str r2_id: FileStoreID of fastq (pair 2 if applicable, otherwise pass None for single-end)
:param str kallisto_index_url: FileStoreID for Kallisto index file
:return: FileStoreID from Kallisto output
:rtype: str


def run_rsem(job, bam_id, rsem_ref_url, paired=True):

work_dir = job.fileStore.getLocalTempDir()<EOL>download_url(job, url=rsem_ref_url, name='<STR_LIT>', work_dir=work_dir)<EOL>subprocess.check_call(['<STR_LIT>', '<STR_LIT>', os.path.join(work_dir, '<STR_LIT>'), '<STR_LIT>', work_dir])<EOL>os.remove(os.path.join(work_dir, '<STR_LIT>'))<EOL>rsem_files = []<EOL>for root, directories, files in os.walk(work_dir):<EOL><INDENT>rsem_files.extend([os.path.join(root, x) for x in files])<EOL><DEDENT>ref_prefix = [os.path.basename(os.path.splitext(x)[<NUM_LIT:0>]) for x in rsem_files if '<STR_LIT>' in x][<NUM_LIT:0>]<EOL>ref_folder = os.path.join('<STR_LIT>', os.listdir(work_dir)[<NUM_LIT:0>]) if len(os.listdir(work_dir)) == <NUM_LIT:1> else '<STR_LIT>'<EOL>job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>output_prefix = '<STR_LIT>'<EOL>parameters = ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', str(job.cores),<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>os.path.join(ref_folder, ref_prefix),<EOL>output_prefix]<EOL>if paired:<EOL><INDENT>parameters = ['<STR_LIT>'] + parameters<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>parameters=parameters, workDir=work_dir)<EOL>gene_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '<STR_LIT>'))<EOL>isoform_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, output_prefix + '<STR_LIT>'))<EOL>return gene_id, isoform_id<EOL>

RNA quantification with RSEM

:param JobFunctionWrappingJob job: Passed automatically by Toil
:param str bam_id: FileStoreID of transcriptome bam for quantification
:param str rsem_ref_url: URL of RSEM reference (tarball)
:param bool paired: If True, uses parameters for paired end data
:return: FileStoreIDs for RSEM's gene and isoform output
:rtype: str


def run_rsem_postprocess(job, rsem_gene_id, rsem_isoform_id):

work_dir = job.fileStore.getLocalTempDir()<EOL>genes = job.fileStore.readGlobalFile(rsem_gene_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>iso = job.fileStore.readGlobalFile(rsem_isoform_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>command = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>parameters=command, workDir=work_dir)<EOL>hugo_files = [os.path.join(work_dir, x) for x in ['<STR_LIT>', '<STR_LIT>']]<EOL>tarball_files('<STR_LIT>', file_paths=[os.path.join(work_dir, x) for x in [genes, iso]], output_dir=work_dir)<EOL>tarball_files('<STR_LIT>', file_paths=[os.path.join(work_dir, x) for x in hugo_files], output_dir=work_dir)<EOL>rsem_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>hugo_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return rsem_id, hugo_id<EOL>

Parses RSEMs output to produce the separate .tab files (TPM, FPKM, counts) for both gene and isoform.
These are two-column files: Genes and Quantifications.
HUGO files are also provided that have been mapped from Gencode/ENSEMBLE names.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str rsem_gene_id: FileStoreID of rsem_gene_ids
:param str rsem_isoform_id: FileStoreID of rsem_isoform_ids
:return: FileStoreID from RSEM post process tarball
:rytpe: str


def run_star(job, r1_id, r2_id, star_index_url, wiggle=False, sort=True):

work_dir = job.fileStore.getLocalTempDir()<EOL>download_url(job, url=star_index_url, name='<STR_LIT>', work_dir=work_dir)<EOL>subprocess.check_call(['<STR_LIT>', '<STR_LIT>', os.path.join(work_dir, '<STR_LIT>'), '<STR_LIT>', work_dir])<EOL>os.remove(os.path.join(work_dir, '<STR_LIT>'))<EOL>star_index = os.path.join('<STR_LIT>', os.listdir(work_dir)[<NUM_LIT:0>]) if len(os.listdir(work_dir)) == <NUM_LIT:1> else '<STR_LIT>'<EOL>parameters = ['<STR_LIT>', str(job.cores),<EOL>'<STR_LIT>', star_index,<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT:1>',<EOL>'<STR_LIT>', '<STR_LIT:1>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>if sort:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>aligned_bam = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>aligned_bam = '<STR_LIT>'<EOL><DEDENT>if wiggle:<EOL><INDENT>parameters.extend(['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>if r1_id and r2_id:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, '<STR_LIT>'))<EOL>parameters.extend(['<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>aligned_bam_path = os.path.join(work_dir, aligned_bam)<EOL>if sort:<EOL><INDENT>assert(os.stat(aligned_bam_path).st_size > <NUM_LIT:0>, '<STR_LIT>')<EOL><DEDENT>aligned_id = job.fileStore.writeGlobalFile(aligned_bam_path)<EOL>transcriptome_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>log_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>sj_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>if wiggle:<EOL><INDENT>wiggle_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>return transcriptome_id, aligned_id, wiggle_id, log_id, sj_id<EOL><DEDENT>else:<EOL><INDENT>return transcriptome_id, aligned_id, log_id, sj_id<EOL><DEDENT>

Performs alignment of fastqs to bam via STAR

--limitBAMsortRAM step added to deal with memory explosion when sorting certain samples.
The value was chosen to complement the recommended amount of memory to have when running STAR (60G)

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str r1_id: FileStoreID of fastq (pair 1)
:param str r2_id: FileStoreID of fastq (pair 2 if applicable, else pass None)
:param str star_index_url: STAR index tarball
:param bool wiggle: If True, will output a wiggle file and return it
:return: FileStoreID from RSEM
:rtype: str


def run_bwakit(job, config, sort=True, trim=False, mark_secondary=False):

work_dir = job.fileStore.getLocalTempDir()<EOL>rg = None<EOL>inputs = {'<STR_LIT>': config.ref,<EOL>'<STR_LIT>': config.fai,<EOL>'<STR_LIT>': config.amb,<EOL>'<STR_LIT>': config.ann,<EOL>'<STR_LIT>': config.bwt,<EOL>'<STR_LIT>': config.pac,<EOL>'<STR_LIT>': config.sa}<EOL>samples = []<EOL>realignment = False<EOL>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>inputs['<STR_LIT>'] = config.r1<EOL>samples.append('<STR_LIT>')<EOL><DEDENT>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>inputs['<STR_LIT>'] = config.r2<EOL>samples.append('<STR_LIT>')<EOL><DEDENT>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>inputs['<STR_LIT>'] = config.bam<EOL>samples.append('<STR_LIT>')<EOL>realignment = True<EOL><DEDENT>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>inputs['<STR_LIT>'] = config.sam<EOL>samples.append('<STR_LIT>')<EOL>realignment = True<EOL><DEDENT>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>inputs['<STR_LIT>'] = config.alt<EOL><DEDENT>for name, fileStoreID in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(fileStoreID, os.path.join(work_dir, name))<EOL><DEDENT>if getattr(config, '<STR_LIT>', None):<EOL><INDENT>rg = config.rg_line<EOL><DEDENT>elif all(getattr(config, elem, None) for elem in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']):<EOL><INDENT>rg = "<STR_LIT>".format(config.uuid)  <EOL>rg_attributes = [config.library, config.platform, config.program_unit, config.uuid]<EOL>for tag, info in zip(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'], rg_attributes):<EOL><INDENT>rg += '<STR_LIT>'.format(tag, info)<EOL><DEDENT><DEDENT>elif realignment:<EOL><INDENT>rg = None<EOL><DEDENT>opt_args = []<EOL>if sort:<EOL><INDENT>opt_args.append('<STR_LIT>')<EOL><DEDENT>if trim:<EOL><INDENT>opt_args.append('<STR_LIT>')<EOL><DEDENT>if mark_secondary:<EOL><INDENT>opt_args.append('<STR_LIT>')<EOL><DEDENT>parameters = ['<STR_LIT>', str(job.cores)] + opt_args + ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if rg is not None:<EOL><INDENT>parameters = ['<STR_LIT>', rg] + parameters<EOL><DEDENT>for sample in samples:<EOL><INDENT>parameters.append('<STR_LIT>'.format(sample))<EOL><DEDENT>dockerCall(job=job, tool='<STR_LIT>',<EOL>parameters=parameters, workDir=work_dir)<EOL>job.fileStore.logToMaster('<STR_LIT>'.format(config.uuid))<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Runs BWA-Kit to align single or paired-end fastq files or realign SAM/BAM files.

:param JobFunctionWrappingJob job: Passed by Toil automatically
:param Namespace config: A configuration object that holds strings as attributes.
    The attributes must be accessible via the dot operator.
    The config must have:
    config.r1               FileStoreID for FASTQ file, or None if realigning SAM/BAM
    config.r2               FileStoreID for paired FASTQ file, or None if single-ended
    config.bam              FileStoreID for BAM file to be realigned, or None if aligning fastq
    config.sam              FileStoreID for SAM file to be realigned, or None if aligning fastq
    config.ref              FileStoreID for the reference genome
    config.fai              FileStoreID for the reference index file
    config.amb              FileStoreID for the reference amb file
    config.ann              FileStoreID for the reference ann file
    config.bwt              FileStoreID for the reference bwt file
    config.pac              FileStoreID for the reference pac file
    config.sa               FileStoreID for the reference sa file
    config.alt              FileStoreID for the reference alt (or None)
    config.rg_line          The read group value to use (or None -- see below)
    config.library          Read group attribute: library
    config.platform         Read group attribute: platform
    config.program_unit     Read group attribute: program unit
    config.uuid             Read group attribute: sample ID

    If specifying config.rg_line, use the following format:
        BAM read group header line (@RG), as defined on page 3 of the SAM spec.
        Tabs should be escaped, e.g., @RG\\tID:foo\\tLB:bar...
        for the read group "foo" from sequencing library "bar".
        Multiple @RG lines can be defined, but should be split by an escaped newline \\n,
        e.g., @RG\\tID:foo\\t:LB:bar\\n@RG\\tID:santa\\tLB:cruz.

:param bool sort: If True, sorts the BAM
:param bool trim: If True, performs adapter trimming
:param bool mark_secondary: If True, mark shorter split reads as secondary
:return: FileStoreID of BAM
:rtype: str


def _make_parameters(master_ip, default_parameters, memory, arguments, override_parameters):

<EOL>require((override_parameters is not None or memory is not None) and<EOL>(override_parameters is None or memory is None),<EOL>"<STR_LIT>")<EOL>parameters = []<EOL>if memory is not None:<EOL><INDENT>parameters = ["<STR_LIT>", "<STR_LIT>" % (master_ip, SPARK_MASTER_PORT),<EOL>"<STR_LIT>", "<STR_LIT>" % memory,<EOL>"<STR_LIT>", "<STR_LIT>" % memory,<EOL>"<STR_LIT>", ("<STR_LIT>" % (master_ip, HDFS_MASTER_PORT))]<EOL><DEDENT>else:<EOL><INDENT>parameters.extend(override_parameters)<EOL><DEDENT>parameters.extend(default_parameters)<EOL>parameters.append('<STR_LIT>')<EOL>parameters.extend(arguments)<EOL>return parameters<EOL>

Makes a Spark Submit style job submission line.

:param masterIP: The Spark leader IP address.
:param default_parameters: Application specific Spark configuration parameters.
:param memory: The memory to allocate to each Spark driver and executor.
:param arguments: Arguments to pass to the submitted job.
:param override_parameters: Parameters passed by the user, that override our defaults.

:type masterIP: MasterAddress
:type default_parameters: list of string
:type arguments: list of string
:type memory: int or None
:type override_parameters: list of string or None


def call_conductor(job, master_ip, src, dst, memory=None, override_parameters=None):

arguments = ["<STR_LIT>", src, dst]<EOL>docker_parameters = ['<STR_LIT>', '<STR_LIT:none>', master_ip.docker_parameters(["<STR_LIT>"])]<EOL>dockerCall(job=job,<EOL>tool="<STR_LIT>",<EOL>parameters=_make_parameters(master_ip,<EOL>[], <EOL>memory,<EOL>arguments,<EOL>override_parameters),<EOL>dockerParameters=docker_parameters)<EOL>

Invokes the Conductor container to copy files between S3 and HDFS and vice versa.
Find Conductor at https://github.com/BD2KGenomics/conductor.

:param toil.Job.job job: The Toil Job calling this function
:param masterIP: The Spark leader IP address.
:param src: URL of file to copy.
:param src: URL of location to copy file to.
:param memory: Gigabytes of memory to provision for Spark driver/worker.
:param override_parameters: Parameters passed by the user, that override our defaults.

:type masterIP: MasterAddress
:type src: string
:type dst: string
:type memory: int or None
:type override_parameters: list of string or None


if run_local:<EOL><INDENT>master = ["<STR_LIT>", "<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>master = ["<STR_LIT>",<EOL>("<STR_LIT>" % (master_ip, SPARK_MASTER_PORT)),<EOL>"<STR_LIT>", ("<STR_LIT>" % (master_ip, HDFS_MASTER_PORT)),]<EOL><DEDENT>default_params = (master + [<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>"])<EOL>if native_adam_path is None:<EOL><INDENT>docker_parameters = ['<STR_LIT>', '<STR_LIT:none>', master_ip.docker_parameters(["<STR_LIT>"])]<EOL>dockerCall(job=job,<EOL>tool="<STR_LIT>",<EOL>dockerParameters=docker_parameters,<EOL>parameters=_make_parameters(master_ip,<EOL>default_params,<EOL>memory,<EOL>arguments,<EOL>override_parameters))<EOL><DEDENT>else:<EOL><INDENT>check_call([os.path.join(native_adam_path, "<STR_LIT>")] +<EOL>default_params +<EOL>arguments)<EOL><DEDENT>

Invokes the ADAM container. Find ADAM at https://github.com/bigdatagenomics/adam.

:param toil.Job.job job: The Toil Job calling this function
:param masterIP: The Spark leader IP address.
:param arguments: Arguments to pass to ADAM.
:param memory: Gigabytes of memory to provision for Spark driver/worker.
:param override_parameters: Parameters passed by the user, that override our defaults.
:param native_adam_path: Path to ADAM executable. If not provided, Docker is used.
:param run_local: If true, runs Spark with the --master local[*] setting, which uses
  all cores on the local machine. The master_ip will be disregarded.

:type masterIP: MasterAddress
:type arguments: list of string
:type memory: int or None
:type override_parameters: list of string or None
:type native_adam_path: string or None
:type run_local: boolean


if self != self.actual:<EOL><INDENT>add_host_option = '<STR_LIT>' + self.actual<EOL>if docker_parameters is None:<EOL><INDENT>docker_parameters = [add_host_option]<EOL><DEDENT>else:<EOL><INDENT>docker_parameters.append(add_host_option)<EOL><DEDENT><DEDENT>return docker_parameters<EOL>

Augment a list of "docker run" arguments with those needed to map the  notional Spark master address to the
real one, if they are different.


cmd = "<STR_LIT>""<STR_LIT>".format(work_dir, os.path.join(work_dir, bam_name))<EOL>process = subprocess.Popen(args=cmd, shell=True, stdout=subprocess.PIPE)<EOL>b_sum = <NUM_LIT:0.0><EOL>b_count = <NUM_LIT:0.0><EOL>while True:<EOL><INDENT>line = process.stdout.readline()<EOL>if not line:<EOL><INDENT>break<EOL><DEDENT>tmp = line.split("<STR_LIT:\t>")<EOL>if abs(int(tmp[<NUM_LIT:8>])) < <NUM_LIT>:<EOL><INDENT>b_sum += abs(int(tmp[<NUM_LIT:8>]))<EOL>b_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>process.wait()<EOL>try:<EOL><INDENT>mean = b_sum / b_count<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>mean = <NUM_LIT><EOL><DEDENT>print("<STR_LIT>" % mean)<EOL>return int(mean)<EOL>


def run_mutect(job, normal_bam, normal_bai, tumor_bam, tumor_bai, ref, ref_dict, fai, cosmic, dbsnp):

work_dir = job.fileStore.getLocalTempDir()<EOL>file_ids = [normal_bam, normal_bai, tumor_bam, tumor_bai, ref, fai, ref_dict, cosmic, dbsnp]<EOL>file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for file_store_id, name in zip(file_ids, file_names):<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(<NUM_LIT:10>),  <EOL>'<STR_LIT>', str(<NUM_LIT>),  <EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, workDir=work_dir, parameters=parameters,<EOL>tool='<STR_LIT>')<EOL>output_file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>output_file_paths = [os.path.join(work_dir, x) for x in output_file_names]<EOL>tarball_files('<STR_LIT>', file_paths=output_file_paths, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Calls MuTect to perform variant analysis

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str normal_bam: Normal BAM FileStoreID
:param str normal_bai: Normal BAM index FileStoreID
:param str tumor_bam: Tumor BAM FileStoreID
:param str tumor_bai: Tumor BAM Index FileStoreID
:param str ref: Reference genome FileStoreID
:param str ref_dict: Reference dictionary FileStoreID
:param str fai: Reference index FileStoreID
:param str cosmic: Cosmic VCF FileStoreID
:param str dbsnp: DBSNP VCF FileStoreID
:return: MuTect output (tarball) FileStoreID
:rtype: str


def run_muse(job, normal_bam, normal_bai, tumor_bam, tumor_bai, ref, ref_dict, fai, dbsnp):

work_dir = job.fileStore.getLocalTempDir()<EOL>file_ids = [normal_bam, normal_bai, tumor_bam, tumor_bai, ref, ref_dict, fai, dbsnp]<EOL>file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for file_store_id, name in zip(file_ids, file_names):<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(job.cores)]<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>tarball_files('<STR_LIT>', file_paths=[os.path.join(work_dir, '<STR_LIT>')], output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Calls MuSe to find variants

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str normal_bam: Normal BAM FileStoreID
:param str normal_bai: Normal BAM index FileStoreID
:param str tumor_bam: Tumor BAM FileStoreID
:param str tumor_bai: Tumor BAM Index FileStoreID
:param str tumor_bai: Tumor BAM Index FileStoreID
:param str ref: Reference genome FileStoreID
:param str ref_dict: Reference genome dictionary FileStoreID
:param str fai: Reference index FileStoreID
:param str dbsnp: DBSNP VCF FileStoreID
:return: MuSe output (tarball) FileStoreID
:rtype: str


def run_pindel(job, normal_bam, normal_bai, tumor_bam, tumor_bai, ref, fai):

work_dir = job.fileStore.getLocalTempDir()<EOL>file_ids = [normal_bam, normal_bai, tumor_bam, tumor_bai, ref, fai]<EOL>file_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for file_store_id, name in zip(file_ids, file_names):<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>with open(os.path.join(work_dir, '<STR_LIT>'), '<STR_LIT:w>') as f:<EOL><INDENT>for bam in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>f.write('<STR_LIT>'.format(bam + '<STR_LIT>', get_mean_insert_size(work_dir, bam + '<STR_LIT>'), bam))<EOL><DEDENT><DEDENT>parameters = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', str(job.cores),<EOL>'<STR_LIT>', '<STR_LIT:3>',<EOL>'<STR_LIT>', '<STR_LIT:true>',<EOL>'<STR_LIT>', '<STR_LIT:true>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>dockerCall(job=job, tool='<STR_LIT>',<EOL>workDir=work_dir, parameters=parameters)<EOL>output_files = glob(os.path.join(work_dir, '<STR_LIT>'))<EOL>tarball_files('<STR_LIT>', file_paths=output_files, output_dir=work_dir)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Calls Pindel to compute indels / deletions

:param JobFunctionWrappingJob job: Passed automatically by Toil
:param str normal_bam: Normal BAM FileStoreID
:param str normal_bai: Normal BAM index FileStoreID
:param str tumor_bam: Tumor BAM FileStoreID
:param str tumor_bai: Tumor BAM Index FileStoreID
:param str ref: Reference genome FileStoreID
:param str fai: Reference index FileStoreID
:return: Pindel output (tarball) FileStoreID
:rtype: str


def gatk_select_variants(job, mode, vcf_id, ref_fasta, ref_fai, ref_dict):

job.fileStore.logToMaster('<STR_LIT>' % mode)<EOL>inputs = {'<STR_LIT>': ref_fasta,<EOL>'<STR_LIT>': ref_fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': vcf_id}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>command = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', mode]<EOL>docker_parameters = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory)]<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>return job.fileStore.writeGlobalFile(os.path.join(work_dir, '<STR_LIT>'))<EOL>

Isolates a particular variant type from a VCF file using GATK SelectVariants

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str mode: variant type (i.e. SNP or INDEL)
:param str vcf_id: FileStoreID for input VCF file
:param str ref_fasta: FileStoreID for reference genome fasta
:param str ref_fai: FileStoreID for reference genome index file
:param str ref_dict: FileStoreID for reference genome sequence dictionary file
:return: FileStoreID for filtered VCF
:rtype: str


def gatk_variant_filtration(job, vcf_id, filter_name, filter_expression, ref_fasta, ref_fai, ref_dict):

inputs = {'<STR_LIT>': ref_fasta,<EOL>'<STR_LIT>': ref_fai,<EOL>'<STR_LIT>': ref_dict,<EOL>'<STR_LIT>': vcf_id}<EOL>work_dir = job.fileStore.getLocalTempDir()<EOL>for name, file_store_id in inputs.iteritems():<EOL><INDENT>job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))<EOL><DEDENT>command = ['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', filter_name,   <EOL>'<STR_LIT>', filter_expression,<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>job.fileStore.logToMaster('<STR_LIT>'<EOL>'<STR_LIT>'.format(name=filter_name, expression=filter_expression))<EOL>docker_parameters = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT:none>',<EOL>'<STR_LIT>', '<STR_LIT>'.format(job.memory)]<EOL>dockerCall(job=job, workDir=work_dir,<EOL>parameters=command,<EOL>tool='<STR_LIT>',<EOL>dockerParameters=docker_parameters)<EOL>malformed_header = os.path.join(work_dir, '<STR_LIT>')<EOL>fixed_header = os.path.join(work_dir, '<STR_LIT>')<EOL>filter_regex = re.escape('<STR_LIT>' % filter_expression)<EOL>with open(malformed_header, '<STR_LIT:r>') as f, open(fixed_header, '<STR_LIT:w>') as g:<EOL><INDENT>for line in f:<EOL><INDENT>g.write(re.sub(filter_regex, filter_expression, line))<EOL><DEDENT><DEDENT>return job.fileStore.writeGlobalFile(fixed_header)<EOL>

Filters VCF file using GATK VariantFiltration. Fixes extra pair of quotation marks in VCF header that
may interfere with other VCF tools.

:param JobFunctionWrappingJob job: passed automatically by Toil
:param str vcf_id: FileStoreID for input VCF file
:param str filter_name: Name of filter for VCF header
:param str filter_expression: JEXL filter expression
:param str ref_fasta: FileStoreID for reference genome fasta
:param str ref_fai: FileStoreID for reference genome index file
:param str ref_dict: FileStoreID for reference genome sequence dictionary file
:return: FileStoreID for filtered VCF file
:rtype: str