def getOnlyFields(self, pk, fields, cascadeFetch=False):

conn = self._get_connection()<EOL>key = self._get_key_for_id(pk)<EOL>res = conn.hmget(key, fields)<EOL>if type(res) != list or not len(res):<EOL><INDENT>return None<EOL><DEDENT>objDict = {}<EOL>numFields = len(fields)<EOL>i = <NUM_LIT:0><EOL>anyNotNone = False<EOL>while i < numFields:<EOL><INDENT>objDict[fields[i]] = res[i]<EOL>if res[i] != None:<EOL><INDENT>anyNotNone = True<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>if anyNotNone is False:<EOL><INDENT>return None<EOL><DEDENT>objDict['<STR_LIT>'] = pk<EOL>ret = self._redisResultToObj(objDict)<EOL>if cascadeFetch is True:<EOL><INDENT>self._doCascadeFetch(ret)<EOL><DEDENT>return ret<EOL>

getOnlyFields - Gets only certain fields from a paticular primary key. For working on entire filter set, see allOnlyFields

@param pk <int> - Primary Key

@param fields list<str> - List of fields

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

return - Partial objects with only fields applied


def getMultipleOnlyFields(self, pks, fields, cascadeFetch=False):

if type(pks) == set:<EOL><INDENT>pks = list(pks)<EOL><DEDENT>if len(pks) == <NUM_LIT:1>:<EOL><INDENT>return IRQueryableList([self.getOnlyFields(pks[<NUM_LIT:0>], fields, cascadeFetch=cascadeFetch)], mdl=self.mdl)<EOL><DEDENT>conn = self._get_connection()<EOL>pipeline = conn.pipeline()<EOL>for pk in pks:<EOL><INDENT>key = self._get_key_for_id(pk)<EOL>pipeline.hmget(key, fields)<EOL><DEDENT>res = pipeline.execute()<EOL>ret = IRQueryableList(mdl=self.mdl)<EOL>pksLen = len(pks)<EOL>i = <NUM_LIT:0><EOL>numFields = len(fields)<EOL>while i < pksLen:<EOL><INDENT>objDict = {}<EOL>anyNotNone = False<EOL>thisRes = res[i]<EOL>if thisRes is None or type(thisRes) != list:<EOL><INDENT>ret.append(None)<EOL>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>j = <NUM_LIT:0><EOL>while j < numFields:<EOL><INDENT>objDict[fields[j]] = thisRes[j]<EOL>if thisRes[j] != None:<EOL><INDENT>anyNotNone = True<EOL><DEDENT>j += <NUM_LIT:1><EOL><DEDENT>if anyNotNone is False:<EOL><INDENT>ret.append(None)<EOL>i += <NUM_LIT:1><EOL>continue<EOL><DEDENT>objDict['<STR_LIT>'] = pks[i]<EOL>obj = self._redisResultToObj(objDict)<EOL>ret.append(obj)<EOL>i += <NUM_LIT:1><EOL><DEDENT>if cascadeFetch is True:<EOL><INDENT>for obj in ret:<EOL><INDENT>self._doCascadeFetch(obj)<EOL><DEDENT><DEDENT>return ret<EOL>

getMultipleOnlyFields - Gets only certain fields from a list of  primary keys. For working on entire filter set, see allOnlyFields

@param pks list<str> - Primary Keys

@param fields list<str> - List of fields

@param cascadeFetch <bool> Default False, If True, all Foreign objects associated with this model
   will be fetched immediately. If False, foreign objects will be fetched on-access.

return - List of partial objects with only fields applied


return self.getOnlyFields(pk, self.indexedFields)<EOL>

getOnlyIndexedFields - Get only the indexed fields on an object. This is the minimum to delete.

@param pk - Primary key

@return - Object with only indexed fields fetched.


return self.getMultipleOnlyFields(pks, self.indexedFields)<EOL>

getMultipleOnlyIndexedFields - Get only the indexed fields on an object. This is the minimum to delete.

@param pks - List of primary keys

@return - List of objects with only indexed fields fetched


objs = self.all()<EOL>saver = IndexedRedisSave(self.mdl)<EOL>saver.reindex(objs)<EOL>

reindex - Reindexes the objects matching current filterset. Use this if you add/remove a field to INDEXED_FIELDS.

  NOTE - This will NOT remove entries from the old index if you change index type, or change decimalPlaces on a
    IRFixedPointField.  To correct these indexes, you'll need to run:

       Model.reset(Model.objects.all())

If you change the value of "hashIndex" on a field, you need to call #compat_convertHashedIndexes instead.


saver = IndexedRedisSave(self.mdl)<EOL>if fetchAll is True:<EOL><INDENT>objs = self.all()<EOL>saver.compat_convertHashedIndexes(objs)<EOL><DEDENT>else:<EOL><INDENT>didWarnOnce = False<EOL>pks = self.getPrimaryKeys()<EOL>for pk in pks:<EOL><INDENT>obj = self.get(pk)<EOL>if not obj:<EOL><INDENT>if didWarnOnce is False:<EOL><INDENT>sys.stderr.write('<STR_LIT>''<STR_LIT>''<STR_LIT>')<EOL>didWarnOnce = True<EOL><DEDENT>continue<EOL><DEDENT>saver.compat_convertHashedIndexes([obj])<EOL><DEDENT><DEDENT>

compat_convertHashedIndexes - Reindex fields, used for when you change the propery "hashIndex" on one or more fields.

For each field, this will delete both the hash and unhashed keys to an object,
  and then save a hashed or unhashed value, depending on that field's value for "hashIndex".

For an IndexedRedisModel class named "MyModel", call as "MyModel.objects.compat_convertHashedIndexes()"

NOTE: This works one object at a time (regardless of #fetchAll), so that an unhashable object does not trash all data.

This method is intended to be used while your application is offline,
  as it doesn't make sense to be changing your model while applications are actively using it.

@param fetchAll <bool>, Default True - If True, all objects will be fetched first, then converted.
  This is generally what you want to do, as it is more efficient. If you are memory contrainted,
  you can set this to "False", and it will fetch one object at a time, convert it, and save it back.


def save(self, obj, usePipeline=True, forceID=False, cascadeSave=True, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>if usePipeline is True:<EOL><INDENT>idConn = conn<EOL><DEDENT>else:<EOL><INDENT>idConn = self._get_new_connection()<EOL><DEDENT>if issubclass(obj.__class__, (list, tuple)):<EOL><INDENT>objs = obj<EOL><DEDENT>else:<EOL><INDENT>objs = [obj]<EOL><DEDENT>if usePipeline is True:<EOL><INDENT>pipeline = conn.pipeline()<EOL><DEDENT>else:<EOL><INDENT>pipeline = conn<EOL><DEDENT>oga = object.__getattribute__<EOL>if cascadeSave is True:<EOL>ignPipelines = OrderedDict()<EOL><INDENT>foreignSavers = {}<EOL>for thisObj in objs:<EOL><INDENT>if not thisObj.foreignFields:<EOL><INDENT>continue<EOL><DEDENT>foreignFields = thisObj.foreignFields<EOL>for foreignField in foreignFields:<EOL><INDENT>rawObj = oga(thisObj, str(foreignField))<EOL>if rawObj in (None, irNull) or not rawObj.isFetched():<EOL><INDENT>continue<EOL><DEDENT>foreignObjects = oga(thisObj, str(foreignField)).getObjs()<EOL>for foreignObject in foreignObjects:<EOL><INDENT>doSaveForeign = False<EOL>if getattr(foreignObject, '<STR_LIT>', None):<EOL><INDENT>if foreignObject.hasUnsavedChanges(cascadeObjects=True):<EOL><INDENT>doSaveForeign = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>doSaveForeign = True<EOL><DEDENT>if doSaveForeign is True:<EOL><INDENT>if foreignField not in foreignSavers:<EOL><DEDENT><DEDENT><DEDENT>foreignPipelines[foreignField] = self._get_new_connection().pipeline()<EOL><INDENT>foreignSavers[foreignField] = IndexedRedisSave(foreignObject.__class__)<EOL>

save - Save an object / objects associated with this model.

You probably want to just do object.save() instead of this, but to save multiple objects at once in a single transaction,
   you can use:

        MyModel.saver.save(myObjs)

@param obj <IndexedRedisModel or list<IndexedRedisModel> - The object to save, or a list of objects to save

@param usePipeline - Use a pipeline for saving. You should always want this, unless you are calling this function from within an existing pipeline.

@param forceID - if not False, force ID to this. If obj is list, this is also list. Forcing IDs also forces insert. Up to you to ensure ID will not clash.
@param cascadeSave <bool> Default True - If True, any Foreign models linked as attributes that have been altered
   or created will be saved with this object. If False, only this object (and the reference to an already-saved foreign model) will be saved.

@param conn - A connection or None

@note - if no ID is specified

@return - List of pks


<EOL>return self.save(objs)<EOL>

saveMultiple - Save a list of objects using a pipeline.

@param objs < list<IndexedRedisModel> > - List of objects to save


def _doSave(self, obj, isInsert, conn, pipeline=None):

if pipeline is None:<EOL><INDENT>pipeline = conn<EOL><DEDENT>newDict = obj.asDict(forStorage=True)<EOL>key = self._get_key_for_id(obj._id)<EOL>if isInsert is True:<EOL><INDENT>for thisField in self.fields:<EOL><INDENT>fieldValue = newDict.get(thisField, thisField.getDefaultValue())<EOL>pipeline.hset(key, thisField, fieldValue)<EOL>if fieldValue == IR_NULL_STR:<EOL><INDENT>obj._origData[thisField] = irNull<EOL><DEDENT>else:<EOL><INDENT>obj._origData[thisField] = object.__getattribute__(obj, str(thisField))<EOL><DEDENT><DEDENT>self._add_id_to_keys(obj._id, pipeline)<EOL>for indexedField in self.indexedFields:<EOL><INDENT>self._add_id_to_index(indexedField, obj._id, obj._origData[indexedField], pipeline)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>updatedFields = obj.getUpdatedFields()<EOL>for thisField, fieldValue in updatedFields.items():<EOL><INDENT>(oldValue, newValue) = fieldValue<EOL>oldValueForStorage = thisField.toStorage(oldValue)<EOL>newValueForStorage = thisField.toStorage(newValue)<EOL>pipeline.hset(key, thisField, newValueForStorage)<EOL>if thisField in self.indexedFields:<EOL><INDENT>self._rem_id_from_index(thisField, obj._id, oldValueForStorage, pipeline)<EOL>self._add_id_to_index(thisField, obj._id, newValueForStorage, pipeline)<EOL><DEDENT>obj._origData[thisField] = newValue<EOL><DEDENT><DEDENT>

_doSave - Internal function to save a single object. Don't call this directly.
            Use "save" instead.

  If a pipeline is provided, the operations (setting values, updating indexes, etc)
    will be queued into that pipeline.
  Otherwise, everything will be executed right away.

  @param obj - Object to save
  @param isInsert - Bool, if insert or update. Either way, obj._id is expected to be set.
  @param conn - Redis connection
  @param pipeline - Optional pipeline, if present the items will be queued onto it. Otherwise, go directly to conn.


if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>pipeline = conn.pipeline()<EOL>objDicts = [obj.asDict(True, forStorage=True) for obj in objs]<EOL>for indexedFieldName in self.indexedFields:<EOL><INDENT>for objDict in objDicts:<EOL><INDENT>self._rem_id_from_index(indexedFieldName, objDict['<STR_LIT>'], objDict[indexedFieldName], pipeline)<EOL>self._add_id_to_index(indexedFieldName, objDict['<STR_LIT>'], objDict[indexedFieldName], pipeline)<EOL><DEDENT><DEDENT>pipeline.execute()<EOL>

reindex - Reindexes a given list of objects. Probably you want to do Model.objects.reindex() instead of this directly.

@param objs list<IndexedRedisModel> - List of objects to reindex
@param conn <redis.Redis or None> - Specific Redis connection or None to reuse


def compat_convertHashedIndexes(self, objs, conn=None):

if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL><DEDENT>fields = []        <EOL>for indexedField in self.indexedFields:<EOL><INDENT>origField = self.fields[indexedField]<EOL>if '<STR_LIT>' not in origField.__class__.__new__.__code__.co_varnames:<EOL><INDENT>continue<EOL><DEDENT>if indexedField.hashIndex is True:<EOL><INDENT>hashingField = origField<EOL>regField = origField.copy()<EOL>regField.hashIndex = False<EOL><DEDENT>else:<EOL><INDENT>regField = origField<EOL>hashingField = origField.copy()<EOL>hashingField.hashIndex = True<EOL><DEDENT>fields.append ( (origField, regField, hashingField) )<EOL><DEDENT>objDicts = [obj.asDict(True, forStorage=True) for obj in objs]<EOL>for objDict in objDicts:<EOL><INDENT>pipeline = conn.pipeline()<EOL>pk = objDict['<STR_LIT>']<EOL>for origField, regField, hashingField in fields:<EOL><INDENT>val = objDict[indexedField]<EOL>self._rem_id_from_index(regField, pk, val, pipeline)<EOL>self._rem_id_from_index(hashingField, pk, val, pipeline)<EOL>self._add_id_to_index(origField, pk, val, pipeline)<EOL><DEDENT>pipeline.execute()<EOL><DEDENT>

compat_convertHashedIndexes - Reindex all fields for the provided objects, where the field value is hashed or not.
If the field is unhashable, do not allow.

NOTE: This works one object at a time. It is intended to be used while your application is offline,
  as it doesn't make sense to be changing your model while applications are actively using it.

@param objs <IndexedRedisModel objects to convert>
@param conn <redis.Redis or None> - Specific Redis connection or None to reuse.


if not getattr(obj, '<STR_LIT>', None):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if conn is None:<EOL><INDENT>conn = self._get_connection()<EOL>pipeline = conn.pipeline()<EOL>executeAfter = True<EOL><DEDENT>else:<EOL><INDENT>pipeline = conn <EOL>executeAfter = False<EOL><DEDENT>pipeline.delete(self._get_key_for_id(obj._id))<EOL>self._rem_id_from_keys(obj._id, pipeline)<EOL>for indexedFieldName in self.indexedFields:<EOL><INDENT>self._rem_id_from_index(indexedFieldName, obj._id, obj._origData[indexedFieldName], pipeline)<EOL><DEDENT>obj._id = None<EOL>if executeAfter is True:<EOL><INDENT>pipeline.execute()<EOL><DEDENT>return <NUM_LIT:1><EOL>

deleteOne - Delete one object

@param obj - object to delete
@param conn - Connection to reuse, or None

@return - number of items deleted (0 or 1)


obj = self.mdl.objects.getOnlyIndexedFields(pk)<EOL>if not obj:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>return self.deleteOne(obj)<EOL>

deleteByPk - Delete object associated with given primary key


conn = self._get_connection()<EOL>pipeline = conn.pipeline()<EOL>numDeleted = <NUM_LIT:0><EOL>for obj in objs:<EOL><INDENT>numDeleted += self.deleteOne(obj, pipeline)<EOL><DEDENT>pipeline.execute()<EOL>return numDeleted<EOL>

deleteMultiple - Delete multiple objects

@param objs - List of objects

@return - Number of objects deleted


if type(pks) == set:<EOL><INDENT>pks = list(pks)<EOL><DEDENT>if len(pks) == <NUM_LIT:1>:<EOL><INDENT>return self.deleteByPk(pks[<NUM_LIT:0>])<EOL><DEDENT>objs = self.mdl.objects.getMultipleOnlyIndexedFields(pks)<EOL>return self.deleteMultiple(objs)<EOL>

deleteMultipleByPks - Delete multiple objects given their primary keys

@param pks - List of primary keys

@return - Number of objects deleted


conn = self._get_connection()<EOL>pipeline = conn.pipeline()<EOL>pipeline.eval(

destroyModel - Destroy everything related to this model in one swoop.

    Same effect as Model.reset([]) - Except slightly more efficient.

    This function is called if you do Model.objects.delete() with no filters set.

@return - Number of keys deleted. Note, this is NOT number of models deleted, but total keys.


global __deprecatedMessagesEnabled<EOL>__deprecatedMessagesEnabled = enabled<EOL>

toggleDeprecatedMessages - Normally, a deprecated message will log up to once per message.
  Call toggleDeprecatedMessages(False) to turn them off altogether (like on a Production codebase)

@param enabled <bool> - False to disable deprecated messages, otherwise True.


def deprecatedMessage(msg, key=None, printStack=False):

if __deprecatedMessagesEnabled is False:<EOL><INDENT>return<EOL><DEDENT>if not _alreadyWarned:<EOL><INDENT>sys.stderr.write('<STR_LIT>')<EOL><DEDENT>if key is None:<EOL><INDENT>from .compat_str import tobytes<EOL>key = md5(tobytes(msg)).hexdigest()<EOL><DEDENT>if key not in _alreadyWarned:<EOL><INDENT>_alreadyWarned[key] = True<EOL>sys.stderr.write('<STR_LIT>' %(msg, ))<EOL>if printStack:<EOL><INDENT>sys.stderr.write('<STR_LIT>')<EOL>curStack = traceback.extract_stack()<EOL>sys.stderr.write('<STR_LIT:U+0020>' + '<STR_LIT>'.join(traceback.format_list(curStack[:-<NUM_LIT:2>])).replace('<STR_LIT:\t>', '<STR_LIT:U+0020>') + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT>

deprecatedMessage - Print a deprecated messsage (unless they are toggled off). Will print a message only once (based on "key")

@param msg <str> - Deprecated message to possibly print

@param key <anything> - A key that is specific to this message.
        If None is provided (default), one will be generated from the md5 of the message.
        However, better to save cycles and provide a unique key if at all possible.
        The decorator uses the function itself as the key.

@param printStack <bool> Default False, if True print a stack trace


keys = [str(x) for x in myDict.keys()]<EOL>keys.sort()<EOL>lst = []<EOL>for key in keys:<EOL><INDENT>lst.append(str(myDict[key]) + '<STR_LIT>')<EOL><DEDENT>return '<STR_LIT>'.join(lst).__hash__()<EOL>

A function which can generate a hash of a one-level
  dict containing strings (like REDIS_CONNECTION_PARAMS)

@param myDict <dict> - Dict with string keys and values

@return <long> - Hash of myDict


def _rindex(mylist: Sequence[T], x: T) -> int:

return len(mylist) - mylist[::-<NUM_LIT:1>].index(x) - <NUM_LIT:1><EOL>

Index of the last occurrence of x in the sequence.


def raw_to_delimited(header: Header, raw_payload: RawPayload) -> DelimitedMsg:

return tuple(header) + (b'<STR_LIT>',) + tuple(raw_payload)<EOL>

\
    Returns a message consisting of header frames, delimiter frame, and payload frames.
    The payload frames may be given as sequences of bytes (raw) or as `Message`s.


def to_delimited(header: Header, payload: Payload, side: CommSide) -> DelimitedMsg:

return raw_to_delimited(header, [side.serialize(msg) for msg in payload])<EOL>


def raw_from_delimited(msgs: DelimitedMsg) -> RawMsgs:

delim = _rindex(msgs, b'<STR_LIT>')<EOL>return tuple(msgs[:delim]), tuple(msgs[delim + <NUM_LIT:1>:])<EOL>

\
    From a message consisting of header frames, delimiter frame, and payload frames, return a tuple `(header, payload)`.
    The payload frames may be returned as sequences of bytes (raw) or as `Message`s.


def from_delimited(msgs: DelimitedMsg, side: CommSide) -> Msgs:

header, raw_payload = raw_from_delimited(msgs)<EOL>return header, tuple(side.parse(msg_raw) for msg_raw in raw_payload)<EOL>


def error(code: int, *args, **kwargs) -> HedgehogCommandError:

<EOL>if code == FAILED_COMMAND and len(args) >= <NUM_LIT:1> and args[<NUM_LIT:0>] == "<STR_LIT>":<EOL><INDENT>return EmergencyShutdown(*args, **kwargs)<EOL><DEDENT>return _errors[code](*args, **kwargs)<EOL>

Creates an error from the given code, and args and kwargs.

:param code: The acknowledgement code
:param args: Exception args
:param kwargs: Exception kwargs
:return: the error for the given acknowledgement code


from .messages import ack<EOL>return ack.Acknowledgement(self.code, self.args[<NUM_LIT:0>] if len(self.args) > <NUM_LIT:0> else '<STR_LIT>')<EOL>

Creates an error Acknowledgement message.
The message's code and message are taken from this exception.

:return: the message representing this exception


def parse(self, data: RawMessage) -> Message:

try:<EOL><INDENT>return self.receiver.parse(data)<EOL><DEDENT>except KeyError as err:<EOL><INDENT>raise UnknownCommandError from err<EOL><DEDENT>except DecodeError as err:<EOL><INDENT>raise UnknownCommandError(f"<STR_LIT>") from err<EOL><DEDENT>

\
        Parses a binary protobuf message into a Message object.


def serialize(self, msg: Message) -> RawMessage:

return self.sender.serialize(msg)<EOL>

\
        Serializes a Message object into a binary protobuf message.


product_url = '<STR_LIT>'<EOL>async with session.get(product_url) as response:<EOL><INDENT>data = await response.json()<EOL><DEDENT>return data['<STR_LIT>']<EOL>

Get URLs for LSST the Docs (LTD) products from the LTD Keeper API.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.

    Returns
    -------
    product_urls : `list`
        List of product URLs.


async def get_ltd_product(session, slug=None, url=None):

if url is None:<EOL><INDENT>url = '<STR_LIT>'.format(slug)<EOL><DEDENT>async with session.get(url) as response:<EOL><INDENT>data = await response.json()<EOL><DEDENT>return data<EOL>

Get the product resource (JSON document) from the LSST the Docs API.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    slug : `str`, optional
        Slug identfying the product. This is the same as the subdomain.
        For example, ``'ldm-151'`` is the slug for ``ldm-151.lsst.io``.
        A full product URL can be provided instead, see ``url``.
    url : `str`, optional
        The full LTD Keeper URL for the product resource. For example,
        ``'https://keeper.lsst.codes/products/ldm-151'``. The ``slug``
        can be provided instead.

    Returns
    -------
    product : `dict`
        Product dataset. See
        https://ltd-keeper.lsst.io/products.html#get--products-(slug)
        for fields.


logger = logging.getLogger(__name__)<EOL>@functools.wraps(func)<EOL>def _install_and_run(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>result = func(*args, **kwargs)<EOL><DEDENT>except OSError:<EOL><INDENT>message = "<STR_LIT>"<EOL>logger.warning(message)<EOL>pypandoc.download_pandoc(version='<STR_LIT>')<EOL>logger.debug("<STR_LIT>")<EOL>result = func(*args, **kwargs)<EOL><DEDENT>return result<EOL><DEDENT>return _install_and_run<EOL>

Decorate a function that uses pypandoc to ensure that pandoc is
    installed if necessary.


@ensure_pandoc<EOL>def convert_text(content, from_fmt, to_fmt, deparagraph=False, mathjax=False,<EOL>smart=True, extra_args=None):

logger = logging.getLogger(__name__)<EOL>if extra_args is not None:<EOL><INDENT>extra_args = list(extra_args)<EOL><DEDENT>else:<EOL><INDENT>extra_args = []<EOL><DEDENT>if mathjax:<EOL><INDENT>extra_args.append('<STR_LIT>')<EOL><DEDENT>if smart:<EOL><INDENT>extra_args.append('<STR_LIT>')<EOL><DEDENT>if deparagraph:<EOL><INDENT>extra_args.append('<STR_LIT>')<EOL><DEDENT>extra_args.append('<STR_LIT>')<EOL>extra_args = set(extra_args)<EOL>logger.debug('<STR_LIT>',<EOL>from_fmt, to_fmt, extra_args)<EOL>output = pypandoc.convert_text(content, to_fmt, format=from_fmt,<EOL>extra_args=extra_args)<EOL>return output<EOL>

Convert text from one markup format to another using pandoc.

    This function is a thin wrapper around `pypandoc.convert_text`.

    Parameters
    ----------
    content : `str`
        Original content.

    from_fmt : `str`
        Format of the original ``content``. Format identifier must be one of
        those known by Pandoc. See https://pandoc.org/MANUAL.html for details.

    to_fmt : `str`
        Output format for the content.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.


def convert_lsstdoc_tex(<EOL>content, to_fmt, deparagraph=False, mathjax=False,<EOL>smart=True, extra_args=None):

augmented_content = '<STR_LIT:\n>'.join((LSSTDOC_MACROS, content))<EOL>return convert_text(<EOL>augmented_content, '<STR_LIT>', to_fmt,<EOL>deparagraph=deparagraph, mathjax=mathjax,<EOL>smart=smart, extra_args=extra_args)<EOL>

Convert lsstdoc-class LaTeX to another markup format.

    This function is a thin wrapper around `convert_text` that automatically
    includes common lsstdoc LaTeX macros.

    Parameters
    ----------
    content : `str`
        Original content.

    to_fmt : `str`
        Output format for the content (see https://pandoc.org/MANUAL.html).
        For example, 'html5'.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.


if isinstance(element, Para):<EOL><INDENT>if element.next is not None:<EOL><INDENT>return element<EOL><DEDENT>elif element.prev is not None:<EOL><INDENT>return element<EOL><DEDENT>return Plain(*element.content)<EOL><DEDENT>

Panflute filter function that converts content wrapped in a Para to
    Plain.

    Use this filter with pandoc as::

        pandoc [..] --filter=lsstprojectmeta-deparagraph

    Only lone paragraphs are affected. Para elements with siblings (like a
    second Para) are left unaffected.

    This filter is useful for processing strings like titles or author names so
    that the output isn't wrapped in paragraph tags. For example, without
    this filter, pandoc converts a string ``"The title"`` to
    ``
The title
`` in HTML. These ``
`` tags aren't useful if you
    intend to put the title text in ``
`` tags using your own templating
    system.


toJSONFilter(deparagraph)<EOL>

Setuptools entrypoint for the deparagraph CLI.

    Use this filter as::

        pandoc [..] --filter=lsstprojectmeta-deparagraph


macros = {}<EOL>macros.update(get_def_macros(tex_source))<EOL>macros.update(get_newcommand_macros(tex_source))<EOL>return macros<EOL>

r"""Get all macro definitions from TeX source, supporting multiple
    declaration patterns.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    This function uses the following function to scrape macros of different
    types:

    - `get_def_macros`
    - `get_newcommand_macros`

    This macro scraping has the following caveats:

    - Macro definition (including content) must all occur on one line.
    - Macros with arguments are not supported.


macros = {}<EOL>for match in DEF_PATTERN.finditer(tex_source):<EOL><INDENT>macros[match.group('<STR_LIT:name>')] = match.group('<STR_LIT:content>')<EOL><DEDENT>return macros<EOL>

r"""Get all ``\def`` macro definition from TeX source.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    ``\def`` macros with arguments are not supported.


macros = {}<EOL>command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT:name>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'},<EOL>{'<STR_LIT:name>': '<STR_LIT:content>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>for macro in command.parse(tex_source):<EOL><INDENT>macros[macro['<STR_LIT:name>']] = macro['<STR_LIT:content>']<EOL><DEDENT>return macros<EOL>

r"""Get all ``\newcommand`` macro definition from TeX source.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.

    Returns
    -------
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros.

    Notes
    -----
    ``\newcommand`` macros with arguments are not supported.


for linker in self._linkers:<EOL><INDENT>tex_source = linker(tex_source)<EOL><DEDENT>return tex_source<EOL>

r"""Convert citations in LaTeX source to Hyperref links.

        Parameters
        ----------
        tex_source : `str`
            LaTeX document source.

        Returns
        -------
        processed_tex : `str`
            LaTeX document source with all citation commands converted to
            ``\hyperref`` commands.


while True:<EOL><INDENT>try:<EOL><INDENT>parsed = next(self.command.parse(tex_source))<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>tex_source = self._replace_command(tex_source, parsed)<EOL><DEDENT>return tex_source<EOL>

r"""Convert commands of type ``command`` in LaTeX source to Hyperref
        links.

        Parameters
        ----------
        tex_source : `str`
            LaTeX document source.

        Returns
        -------
        processed_tex : `str`
            LaTeX document source with commands of type ``command`` to
            ``\hyperref`` commands.


<EOL>return re.sub(r'<STR_LIT>', r'<STR_LIT>', tex_source, flags=re.M)<EOL>

Delete latex comments from TeX source.

    Parameters
    ----------
    tex_source : str
        TeX source content.

    Returns
    -------
    tex_source : str
        TeX source without comments.


<EOL>return re.sub(r'<STR_LIT>', '<STR_LIT>', tex_source, flags=re.M)<EOL>

Delete trailing whitespace from TeX source.

    Parameters
    ----------
    tex_source : str
        TeX source content.

    Returns
    -------
    tex_source : str
        TeX source without trailing whitespace.


with open(root_filepath, '<STR_LIT:r>') as f:<EOL><INDENT>tex_source = f.read()<EOL><DEDENT>if root_dir is None:<EOL><INDENT>root_dir = os.path.dirname(root_filepath)<EOL><DEDENT>tex_source = remove_comments(tex_source)<EOL>tex_source = remove_trailing_whitespace(tex_source)<EOL>tex_source = process_inputs(tex_source, root_dir=root_dir)<EOL>return tex_source<EOL>

r"""Read a TeX file, automatically processing and normalizing it
    (including other input files, removing comments, and deleting trailing
    whitespace).

    Parameters
    ----------
    root_filepath : `str`
        Filepath to a TeX file.
    root_dir : `str`
        Root directory of the TeX project. This only needs to be set when
        recursively reading in ``\input`` or ``\include`` files.

    Returns
    -------
    tex_source : `str`
        TeX source.


logger = logging.getLogger(__name__)<EOL>def _sub_line(match):<EOL><INDENT>"""<STR_LIT>"""<EOL>fname = match.group('<STR_LIT:filename>')<EOL>if not fname.endswith('<STR_LIT>'):<EOL><INDENT>full_fname = "<STR_LIT:.>".join((fname, '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>full_fname = fname<EOL><DEDENT>full_path = os.path.abspath(os.path.join(root_dir, full_fname))<EOL>try:<EOL><INDENT>included_source = read_tex_file(full_path, root_dir=root_dir)<EOL><DEDENT>except IOError:<EOL><INDENT>logger.error("<STR_LIT>".format(full_path))<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>return included_source<EOL><DEDENT><DEDENT>tex_source = input_include_pattern.sub(_sub_line, tex_source)<EOL>return tex_source<EOL>

r"""Insert referenced TeX file contents (from  ``\input`` and ``\include``
    commands) into the source.

    Parameters
    ----------
    tex_source : `str`
        TeX source where referenced source files will be found and inserted.
    root_dir : `str`, optional
        Name of the directory containing the TeX project's root file. Files
        referenced by TeX ``\input`` and ``\include`` commands are relative to
        this directory. If not set, the current working directory is assumed.

    Returns
    -------
    tex_source : `str`
        TeX source.

    See also
    --------
    `read_tex_file`
        Recommended API for reading a root TeX source file and inserting
        referenced files.


for macro_name, macro_content in macros.items():<EOL><INDENT>pattern = re.escape(macro_name) + r"<STR_LIT>"<EOL>tex_source = re.sub(pattern, lambda _: macro_content, tex_source)<EOL><DEDENT>return tex_source<EOL>

r"""Replace macros in the TeX source with their content.

    Parameters
    ----------
    tex_source : `str`
        TeX source content.
    macros : `dict`
        Keys are macro names (including leading ``\``) and values are the
        content (as `str`) of the macros. See
        `lsstprojectmeta.tex.scraper.get_macros`.

    Returns
    -------
    tex_source : `str`
        TeX source with known macros replaced.

    Notes
    -----
    Macros with arguments are not supported.

    Examples
    --------
    >>> macros = {r'\handle': 'LDM-nnn'}
    >>> sample = r'This is document \handle.'
    >>> replace_macros(sample, macros)
    'This is document LDM-nnn.'

    Any trailing slash after the macro command is also replaced by this
    function.

    >>> macros = {r'\product': 'Data Management'}
    >>> sample = r'\title    [Test Plan]  { \product\ Test Plan}'
    >>> replace_macros(sample, macros)
    '\\title    [Test Plan]  { Data Management Test Plan}'


<EOL>root_dir = os.path.dirname(root_tex_path)<EOL>tex_source = read_tex_file(root_tex_path)<EOL>tex_macros = get_macros(tex_source)<EOL>tex_source = replace_macros(tex_source, tex_macros)<EOL>return cls(tex_source, root_dir=root_dir)<EOL>

Construct an `LsstLatexDoc` instance by reading and parsing the
        LaTeX source.

        Parameters
        ----------
        root_tex_path : `str`
            Path to the LaTeX source on the filesystem. For multi-file LaTeX
            projects this should be the path to the root document.

        Notes
        -----
        This method implements the following pipeline:

        1. `lsstprojectmeta.tex.normalizer.read_tex_file`
        2. `lsstprojectmeta.tex.scraper.get_macros`
        3. `lsstprojectmeta.tex.normalizer.replace_macros`

        Thus ``input`` and ``includes`` are resolved along with simple macros.


return self.format_content(format='<STR_LIT>', mathjax=False, smart=True)<EOL>

Plain-text-formatted document content (`str`).


return self.format_title(format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True)<EOL>


Plain-text-formatted document title (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_title()<EOL><DEDENT>return self._title<EOL>


return self.format_short_title(format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True)<EOL>

HTML5-formatted document short title (`str`).


Plaintext-formatted document short title (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_title()<EOL><DEDENT>return self._short_title<EOL>

LaTeX-formatted document short title (`str`).


return self.format_authors(format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True)<EOL>

HTML5-formatted authors (`list` of `str`).


Plaintext-formatted authors (`list` of `str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_author()<EOL><DEDENT>return self._authors<EOL>

LaTeX-formatted authors (`list` of `str`).


return self.format_abstract(format='<STR_LIT>', deparagraph=False,<EOL>mathjax=False, smart=True)<EOL>

HTML5-formatted document abstract (`str`).


Plaintext-formatted document abstract (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_abstract()<EOL><DEDENT>return self._abstract<EOL>


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_doc_ref()<EOL><DEDENT>return self._handle<EOL>

LaTeX-formatted document handle (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_doc_ref()<EOL><DEDENT>return self._series<EOL>


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_doc_ref()<EOL><DEDENT>return self._serial<EOL>

Document serial number within series (`str`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_documentclass()<EOL><DEDENT>if '<STR_LIT>' in self._document_options:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Document is a draft if ``'lsstdoc'`` is included in the
        documentclass options (`bool`).


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_revision_date()<EOL><DEDENT>return self._datetime<EOL>

Current revision date of the document (`datetime.datetime`).

        The `revision_datetime_source` describes how the revision date
        is computed.

        This ``revision datetime`` is cached the first time you access it. This
        means that a datetime computed via ``now`` or ``git`` will not change
        during the lifetime of an `LsstLatexDoc` object.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._parse_revision_date()<EOL><DEDENT>return self._revision_datetime_source<EOL>

r"""Data source for the `revision_datetime` attribute (`str`).

        Possible string values are:

        - ``'tex'``: The document revision date is defined in the ``\date``
          command. ``YYYY-MM-DD`` dates are converted to UTC datetimes by
          assuming the document is released at the beginning of the day in the
          ``US/Pacific`` timezone. Note: the ``\date`` command is ignored
          for draft documents (`is_draft` is `True`) so that drafts always
          fall back to ``'git'`` or ``'now'``.

        - ``'git'``: The latest Git commit's timestamp that affected document
          content. Content is considered any file with a ``tex``, ``bib``,
          ``pdf``, ``jpg``, or ``png`` extension. Git timestamps are used when
          the ``\date`` command is missing or can't be parsed.

        - ``'now'``: The current date and time. This source is used as a
          fallback when the LaTeX and Git-based methods of determining a
          document's date fail.

        The `revision datetime` is cached the first time you access it. This
        means that a datetime computed via ``now`` or ``git`` will not change
        during the lifetime of an `LsstLatexDoc` object.


if self._bib_db is None:<EOL><INDENT>self._load_bib_db()<EOL><DEDENT>return self._bib_db<EOL>

Bibliography database referenced by the document
        (`pybtex.database.BibliographyData`).


def format_content(self, format='<STR_LIT>', mathjax=False,<EOL>smart=True, extra_args=None):

output_text = convert_lsstdoc_tex(<EOL>self._tex, format,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document content in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content.


def format_title(self, format='<STR_LIT>', deparagraph=True, mathjax=False,<EOL>smart=True, extra_args=None):

if self.title is None:<EOL><INDENT>return None<EOL><DEDENT>output_text = convert_lsstdoc_tex(<EOL>self.title, format,<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document title in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the title is not available in
            the document.


def format_short_title(self, format='<STR_LIT>', deparagraph=True,<EOL>mathjax=False, smart=True, extra_args=None):

if self.short_title is None:<EOL><INDENT>return None<EOL><DEDENT>output_text = convert_lsstdoc_tex(<EOL>self.short_title, '<STR_LIT>',<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document short title in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the short title is not available in
            the document.


def format_abstract(self, format='<STR_LIT>', deparagraph=False, mathjax=False,<EOL>smart=True, extra_args=None):

if self.abstract is None:<EOL><INDENT>return None<EOL><DEDENT>abstract_latex = self._prep_snippet_for_pandoc(self.abstract)<EOL>output_text = convert_lsstdoc_tex(<EOL>abstract_latex, format,<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>return output_text<EOL>

Get the document abstract in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `str`
            Converted content or `None` if the title is not available in
            the document.


def format_authors(self, format='<STR_LIT>', deparagraph=True, mathjax=False,<EOL>smart=True, extra_args=None):

formatted_authors = []<EOL>for latex_author in self.authors:<EOL><INDENT>formatted_author = convert_lsstdoc_tex(<EOL>latex_author, format,<EOL>deparagraph=deparagraph,<EOL>mathjax=mathjax,<EOL>smart=smart,<EOL>extra_args=extra_args)<EOL>formatted_author = formatted_author.strip()<EOL>formatted_authors.append(formatted_author)<EOL><DEDENT>return formatted_authors<EOL>

Get the document authors in the specified markup format.

        Parameters
        ----------
        format : `str`, optional
            Output format (such as ``'html5'`` or ``'plain'``).
        deparagraph : `bool`, optional
            Remove the paragraph tags from single paragraph content.
        mathjax : `bool`, optional
            Allow pandoc to use MathJax math markup.
        smart : `True`, optional
            Allow pandoc to create "smart" unicode punctuation.
        extra_args : `list`, optional
            Additional command line flags to pass to Pandoc. See
            `lsstprojectmeta.pandoc.convert.convert_text`.

        Returns
        -------
        output_text : `list` of `str`
            Sequence of author names in the specified output markup format.


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': False, '<STR_LIT>': '<STR_LIT:[>'},<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._document_options = []<EOL><DEDENT>try:<EOL><INDENT>content = parsed['<STR_LIT>']<EOL>self._document_options = [opt.strip()<EOL>for opt in content.split('<STR_LIT:U+002C>')]<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._document_options = []<EOL><DEDENT>

Parse documentclass options.

        Sets the the ``_document_options`` attribute.


command = LatexCommand(<EOL>'<STR_LIT:title>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': False, '<STR_LIT>': '<STR_LIT:[>'},<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._title = None<EOL>self._short_title = None<EOL><DEDENT>self._title = parsed['<STR_LIT>']<EOL>try:<EOL><INDENT>self._short_title = parsed['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._short_title = None<EOL><DEDENT>

Parse the title from TeX source.

        Sets these attributes:

        - ``_title``
        - ``_short_title``


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._handle = None<EOL>self._series = None<EOL>self._serial = None<EOL>return<EOL><DEDENT>self._handle = parsed['<STR_LIT>']<EOL>try:<EOL><INDENT>self._series, self._serial = self._handle.split('<STR_LIT:->', <NUM_LIT:1>)<EOL><DEDENT>except ValueError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>', self._handle)<EOL>self._series = None<EOL>self._serial = None<EOL><DEDENT>

Parse the document handle.

        Sets the ``_series``, ``_serial``, and ``_handle`` attributes.


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._authors = []<EOL>return<EOL><DEDENT>try:<EOL><INDENT>content = parsed['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._authors = []<EOL>return<EOL><DEDENT>content = content.replace('<STR_LIT:\n>', '<STR_LIT:U+0020>')<EOL>content = content.replace('<STR_LIT>', '<STR_LIT:U+0020>')<EOL>content = content.strip()<EOL>authors = []<EOL>for part in content.split('<STR_LIT:U+002C>'):<EOL><INDENT>part = part.strip()<EOL>for split_part in part.split('<STR_LIT>'):<EOL><INDENT>split_part = split_part.strip()<EOL>if len(split_part) > <NUM_LIT:0>:<EOL><INDENT>authors.append(split_part)<EOL><DEDENT><DEDENT><DEDENT>self._authors = authors<EOL>

r"""Parse the author from TeX source.

        Sets the ``_authors`` attribute.

        Goal is to parse::

           \author{
           A.~Author,
           B.~Author,
           and
           C.~Author}

        Into::

           ['A. Author', 'B. Author', 'C. Author']


command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._abstract = None<EOL>return<EOL><DEDENT>try:<EOL><INDENT>content = parsed['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>self._abstract = None<EOL>return<EOL><DEDENT>content = content.strip()<EOL>self._abstract = content<EOL>

Parse the abstract from the TeX source.

        Sets the ``_abstract`` attribute.


replace_cite = CitationLinker(self.bib_db)<EOL>latex_text = replace_cite(latex_text)<EOL>return latex_text<EOL>

Process a LaTeX snippet of content for better transformation
        with pandoc.

        Currently runs the CitationLinker to convert BibTeX citations to
        href links.


<EOL>command = LatexCommand(<EOL>'<STR_LIT>',<EOL>{'<STR_LIT:name>': '<STR_LIT>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(command.parse(self._tex))<EOL>bib_names = [n.strip() for n in parsed['<STR_LIT>'].split('<STR_LIT:U+002C>')]<EOL><DEDENT>except StopIteration:<EOL><INDENT>self._logger.warning('<STR_LIT>')<EOL>bib_names = []<EOL><DEDENT>custom_bib_names = [n for n in bib_names<EOL>if n not in KNOWN_LSSTTEXMF_BIB_NAMES]<EOL>custom_bibs = []<EOL>for custom_bib_name in custom_bib_names:<EOL><INDENT>custom_bib_path = os.path.join(<EOL>os.path.join(self._root_dir),<EOL>custom_bib_name + '<STR_LIT>'<EOL>)<EOL>if not os.path.exists(custom_bib_path):<EOL><INDENT>self._logger.warning('<STR_LIT>',<EOL>custom_bib_path)<EOL>continue<EOL><DEDENT>with open(custom_bib_path, '<STR_LIT:r>') as file_handle:<EOL><INDENT>custom_bibs.append(file_handle.read())<EOL><DEDENT><DEDENT>if len(custom_bibs) > <NUM_LIT:0>:<EOL><INDENT>custom_bibtex = '<STR_LIT>'.join(custom_bibs)<EOL><DEDENT>else:<EOL><INDENT>custom_bibtex = None<EOL><DEDENT>db = get_bibliography(bibtex=custom_bibtex)<EOL>self._bib_db = db<EOL>

r"""Load the BibTeX bibliography referenced by the document.

        This method triggered by the `bib_db` attribute and populates the
        `_bib_db` private attribute.

        The ``\bibliography`` command is parsed to identify the bibliographies
        referenced by the document.


doc_datetime = None<EOL>if not self.is_draft:<EOL><INDENT>date_command = LatexCommand(<EOL>'<STR_LIT:date>',<EOL>{'<STR_LIT:name>': '<STR_LIT:content>', '<STR_LIT>': True, '<STR_LIT>': '<STR_LIT:{>'})<EOL>try:<EOL><INDENT>parsed = next(date_command.parse(self._tex))<EOL>command_content = parsed['<STR_LIT:content>'].strip()<EOL><DEDENT>except StopIteration:<EOL><INDENT>command_content = None<EOL>self._logger.warning('<STR_LIT>')<EOL><DEDENT>if command_content is not None and command_content != r'<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>doc_datetime = datetime.datetime.strptime(command_content,<EOL>'<STR_LIT>')<EOL>project_tz = timezone('<STR_LIT>')<EOL>localized_datetime = project_tz.localize(doc_datetime)<EOL>doc_datetime = localized_datetime.astimezone(pytz.utc)<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>except ValueError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>command_content)<EOL><DEDENT><DEDENT><DEDENT>if doc_datetime is None:<EOL><INDENT>content_extensions = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>doc_datetime = get_content_commit_date(<EOL>content_extensions,<EOL>root_dir=self._root_dir)<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>except RuntimeError:<EOL><INDENT>self._logger.warning('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>self._root_dir)<EOL><DEDENT><DEDENT>if doc_datetime is None:<EOL><INDENT>doc_datetime = pytz.utc.localize(datetime.datetime.now())<EOL>self._revision_datetime_source = '<STR_LIT>'<EOL><DEDENT>self._datetime = doc_datetime<EOL>

r"""Parse the ``\date`` command, falling back to getting the
        most recent Git commit date and the current datetime.

        Result is available from the `revision_datetime` attribute.


def build_jsonld(self, url=None, code_url=None, ci_url=None,<EOL>readme_url=None, license_id=None):

jsonld = {<EOL>'<STR_LIT>': [<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"],<EOL>'<STR_LIT>': ['<STR_LIT>', '<STR_LIT>'],<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': self.handle,<EOL>'<STR_LIT:name>': self.plain_title,<EOL>'<STR_LIT:description>': self.plain_abstract,<EOL>'<STR_LIT>': [{'<STR_LIT>': '<STR_LIT>', '<STR_LIT:name>': author_name}<EOL>for author_name in self.plain_authors],<EOL>'<STR_LIT>': self.revision_datetime<EOL>}<EOL>try:<EOL><INDENT>jsonld['<STR_LIT>'] = self.plain_content<EOL>jsonld['<STR_LIT>'] = '<STR_LIT>'  <EOL><DEDENT>except RuntimeError:<EOL><INDENT>self._logger.exception('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>self._logger.warning('<STR_LIT>')<EOL>jsonld['<STR_LIT>'] = self._tex<EOL>jsonld['<STR_LIT>'] = '<STR_LIT>'  <EOL><DEDENT>if url is not None:<EOL><INDENT>jsonld['<STR_LIT>'] = url<EOL>jsonld['<STR_LIT:url>'] = url<EOL><DEDENT>else:<EOL><INDENT>jsonld['<STR_LIT>'] = self.handle<EOL><DEDENT>if code_url is not None:<EOL><INDENT>jsonld['<STR_LIT>'] = code_url<EOL><DEDENT>if ci_url is not None:<EOL><INDENT>jsonld['<STR_LIT>'] = ci_url<EOL><DEDENT>if readme_url is not None:<EOL><INDENT>jsonld['<STR_LIT>'] = readme_url<EOL><DEDENT>if license_id is not None:<EOL><INDENT>jsonld['<STR_LIT>'] = None<EOL><DEDENT>return jsonld<EOL>

Create a JSON-LD representation of this LSST LaTeX document.

        Parameters
        ----------
        url : `str`, optional
            URL where this document is published to the web. Prefer
            the LSST the Docs URL if possible.
            Example: ``'https://ldm-151.lsst.io'``.
        code_url : `str`, optional
            Path the the document's repository, typically on GitHub.
            Example: ``'https://github.com/lsst/LDM-151'``.
        ci_url : `str`, optional
            Path to the continuous integration service dashboard for this
            document's repository.
            Example: ``'https://travis-ci.org/lsst/LDM-151'``.
        readme_url : `str`, optional
            URL to the document repository's README file. Example:
            ``https://raw.githubusercontent.com/lsst/LDM-151/master/README.rst``.
        license_id : `str`, optional
            License identifier, if known. The identifier should be from the
            listing at https://spdx.org/licenses/. Example: ``CC-BY-4.0``.

        Returns
        -------
        jsonld : `dict`
            JSON-LD-formatted dictionary.


command_regex = self._make_command_regex(self.name)<EOL>for match in re.finditer(command_regex, source):<EOL><INDENT>self._logger.debug(match)<EOL>start_index = match.start(<NUM_LIT:0>)<EOL>yield self._parse_command(source, start_index)<EOL><DEDENT>

Parse command content from the LaTeX source.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.

        Yields
        ------
        parsed_command : `ParsedCommand`
            Yields parsed commands instances for each occurence of the command
            in the source.


return r'<STR_LIT:\\>' + name + r'<STR_LIT>'<EOL>

r"""Given a command name, build a regular expression to detect that
        command in TeX source.

        The regular expression is designed to discern "\title{..}" from
        "\titlename{..}". It does this by ensuring that the command is
        followed by a whitespace character, argument brackets, or a comment
        character.

        Parameters
        ----------
        name : `str`
            Name of the command (with a backslash prefix).

        Returns
        -------
        regex : `str`
            Regular expression pattern for detecting the command.


def _parse_command(self, source, start_index):

parsed_elements = []<EOL>running_index = start_index<EOL>for element in self.elements:<EOL><INDENT>opening_bracket = element['<STR_LIT>']<EOL>closing_bracket = self._brackets[opening_bracket]<EOL>element_start = None<EOL>element_end = None<EOL>for i, c in enumerate(source[running_index:], start=running_index):<EOL><INDENT>if c == element['<STR_LIT>']:<EOL><INDENT>element_start = i<EOL>break<EOL><DEDENT>elif c == '<STR_LIT:\n>':<EOL><INDENT>if element['<STR_LIT>'] is True:<EOL><INDENT>content = self._parse_whitespace_argument(<EOL>source[running_index:],<EOL>self.name)<EOL>return ParsedCommand(<EOL>self.name,<EOL>[{'<STR_LIT:index>': element['<STR_LIT:index>'],<EOL>'<STR_LIT:name>': element['<STR_LIT:name>'],<EOL>'<STR_LIT:content>': content.strip()}],<EOL>start_index,<EOL>source[start_index:i])<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>if element_start is None and element['<STR_LIT>'] is False:<EOL><INDENT>continue<EOL><DEDENT>elif element_start is None and element['<STR_LIT>'] is True:<EOL><INDENT>message = ('<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>self.name,<EOL>start_index,<EOL>element['<STR_LIT:index>']))<EOL>raise CommandParserError(message)<EOL><DEDENT>balance = <NUM_LIT:1><EOL>for i, c in enumerate(source[element_start + <NUM_LIT:1>:],<EOL>start=element_start + <NUM_LIT:1>):<EOL><INDENT>if c == opening_bracket:<EOL><INDENT>balance += <NUM_LIT:1><EOL><DEDENT>elif c == closing_bracket:<EOL><INDENT>balance -= <NUM_LIT:1><EOL><DEDENT>if balance == <NUM_LIT:0>:<EOL><INDENT>element_end = i<EOL>break<EOL><DEDENT><DEDENT>if balance > <NUM_LIT:0>:<EOL><INDENT>message = ('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>self.name,<EOL>start_index,<EOL>element['<STR_LIT:index>']))<EOL>raise CommandParserError(message)<EOL><DEDENT>element_content = source[element_start + <NUM_LIT:1>:element_end]<EOL>parsed_element = {<EOL>'<STR_LIT:index>': element['<STR_LIT:index>'],<EOL>'<STR_LIT:name>': element['<STR_LIT:name>'],<EOL>'<STR_LIT:content>': element_content.strip()<EOL>}<EOL>parsed_elements.append(parsed_element)<EOL>running_index = element_end + <NUM_LIT:1><EOL><DEDENT>command_source = source[start_index:running_index]<EOL>parsed_command = ParsedCommand(self.name, parsed_elements,<EOL>start_index, command_source)<EOL>return parsed_command<EOL>

Parse a single command.

        Parameters
        ----------
        source : `str`
            The full source of the tex document.
        start_index : `int`
            Character index in ``source`` where the command begins.

        Returns
        -------
        parsed_command : `ParsedCommand`
            The parsed command from the source at the given index.


<EOL>command_pattern = r'<STR_LIT>' + name + r'<STR_LIT>'<EOL>command_match = re.search(command_pattern, source)<EOL>if command_match is not None:<EOL><INDENT>source = source[command_match.end(<NUM_LIT:1>):]<EOL><DEDENT>pattern = r'<STR_LIT>'<EOL>match = re.search(pattern, source)<EOL>if match is None:<EOL><INDENT>message = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL>raise CommandParserError(message.format(name))<EOL><DEDENT>content = match.group('<STR_LIT:content>')<EOL>content.strip()<EOL>return content<EOL>

r"""Attempt to parse a single token on the first line of this source.

        This method is used for parsing whitespace-delimited arguments, like
        ``\input file``. The source should ideally contain `` file`` along
        with a newline character.

        >>> source = 'Line 1\n' r'\input test.tex' '\nLine 2'
        >>> LatexCommand._parse_whitespace_argument(source, 'input')
        'test.tex'

        Bracket delimited arguments (``\input{test.tex}``) are handled in
        the normal logic of `_parse_command`.


logger = logging.getLogger(__name__)<EOL>async with session.get(url) as response:<EOL><INDENT>logger.info('<STR_LIT>', url)<EOL>return await response.text()<EOL><DEDENT>

Asynchronously request a URL and get the encoded text content of the
    body.

    Parameters
    ----------
    url : `str`
        URL to download.
    session : `aiohttp.ClientSession`
        An open aiohttp session.

    Returns
    -------
    content : `str`
        Content downloaded from the URL.


blob_url_template = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL>urls = [blob_url_template.format(name=name) for name in bibtex_names]<EOL>tasks = []<EOL>async with ClientSession() as session:<EOL><INDENT>for url in urls:<EOL><INDENT>task = asyncio.ensure_future(_download_text(url, session))<EOL>tasks.append(task)<EOL><DEDENT>return await asyncio.gather(*tasks)<EOL><DEDENT>

Asynchronously download a set of lsst-texmf BibTeX bibliographies from
    GitHub.

    Parameters
    ----------
    bibtex_names : sequence of `str`
        Names of lsst-texmf BibTeX files to download. For example:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

    Returns
    -------
    bibtexs : `list` of `str`
        List of BibTeX file content, in the same order as ``bibtex_names``.


logger = logging.getLogger(__name__)<EOL>if bibtex_filenames is None:<EOL><INDENT>bibtex_names = KNOWN_LSSTTEXMF_BIB_NAMES<EOL><DEDENT>else:<EOL><INDENT>bibtex_names = []<EOL>for filename in bibtex_filenames:<EOL><INDENT>name = os.path.basename(os.path.splitext(filename)[<NUM_LIT:0>])<EOL>if name not in KNOWN_LSSTTEXMF_BIB_NAMES:<EOL><INDENT>logger.warning('<STR_LIT>',<EOL>name)<EOL>continue<EOL><DEDENT>bibtex_names.append(name)<EOL><DEDENT><DEDENT>uncached_names = [name for name in bibtex_names<EOL>if name not in _LSSTTEXMF_BIB_CACHE]<EOL>if len(uncached_names) > <NUM_LIT:0>:<EOL><INDENT>loop = asyncio.get_event_loop()<EOL>future = asyncio.ensure_future(_download_lsst_bibtex(uncached_names))<EOL>loop.run_until_complete(future)<EOL>for name, text in zip(bibtex_names, future.result()):<EOL><INDENT>_LSSTTEXMF_BIB_CACHE[name] = text<EOL><DEDENT><DEDENT>return {name: _LSSTTEXMF_BIB_CACHE[name] for name in bibtex_names}<EOL>

Get content of lsst-texmf bibliographies.

    BibTeX content is downloaded from GitHub (``master`` branch of
    https://github.com/lsst/lsst-texmf or retrieved from an in-memory cache.

    Parameters
    ----------
    bibtex_filenames : sequence of `str`, optional
        List of lsst-texmf BibTeX files to retrieve. These can be the filenames
        of lsst-bibtex files (for example, ``['lsst.bib', 'lsst-dm.bib']``)
        or names without an extension (``['lsst', 'lsst-dm']``). The default
        (recommended) is to get *all* lsst-texmf bibliographies:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

    Returns
    -------
    bibtex : `dict`
        Dictionary with keys that are bibtex file names (such as ``'lsst'``,
        ``'lsst-dm'``). Values are the corresponding bibtex file content
        (`str`).


bibtex_data = get_lsst_bibtex(bibtex_filenames=lsst_bib_names)<EOL>pybtex_data = [pybtex.database.parse_string(_bibtex, '<STR_LIT>')<EOL>for _bibtex in bibtex_data.values()]<EOL>if bibtex is not None:<EOL><INDENT>pybtex_data.append(pybtex.database.parse_string(bibtex, '<STR_LIT>'))<EOL><DEDENT>bib = pybtex_data[<NUM_LIT:0>]<EOL>if len(pybtex_data) > <NUM_LIT:1>:<EOL><INDENT>for other_bib in pybtex_data[<NUM_LIT:1>:]:<EOL><INDENT>for key, entry in other_bib.entries.items():<EOL><INDENT>bib.add_entry(key, entry)<EOL><DEDENT><DEDENT><DEDENT>return bib<EOL>

Make a pybtex BibliographyData instance from standard lsst-texmf
    bibliography files and user-supplied bibtex content.

    Parameters
    ----------
    lsst_bib_names : sequence of `str`, optional
        Names of lsst-texmf BibTeX files to include. For example:

        .. code-block:: python

           ['lsst', 'lsst-dm', 'refs', 'books', 'refs_ads']

        Default is `None`, which includes all lsst-texmf bibtex files.

    bibtex : `str`
        BibTeX source content not included in lsst-texmf. This can be content
        from a import ``local.bib`` file.

    Returns
    -------
    bibliography : `pybtex.database.BibliographyData`
        A pybtex bibliography database that includes all given sources:
        lsst-texmf bibliographies and ``bibtex``.


if '<STR_LIT:url>' in entry.fields:<EOL><INDENT>return entry.fields['<STR_LIT:url>']<EOL><DEDENT>elif entry.type.lower() == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>' + entry.fields['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in entry.fields:<EOL><INDENT>return entry.fields['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in entry.fields:<EOL><INDENT>return '<STR_LIT>' + entry.fields['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise NoEntryUrlError()<EOL><DEDENT>

Get a usable URL from a pybtex entry.

    Parameters
    ----------
    entry : `pybtex.database.Entry`
        A pybtex bibliography entry.

    Returns
    -------
    url : `str`
        Best available URL from the ``entry``.

    Raises
    ------
    NoEntryUrlError
        Raised when no URL can be made from the bibliography entry.

    Notes
    -----
    The order of priority is:

    1. ``url`` field
    2. ``ls.st`` URL from the handle for ``@docushare`` entries.
    3. ``adsurl``
    4. DOI


def _format_last(person):<EOL><INDENT>"""<STR_LIT>"""<EOL>return '<STR_LIT:U+0020>'.join([n.strip('<STR_LIT:{}>') for n in person.last_names])<EOL><DEDENT>if len(entry.persons['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>persons = entry.persons['<STR_LIT>']<EOL><DEDENT>elif len(entry.persons['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>persons = entry.persons['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise AuthorYearError<EOL><DEDENT>try:<EOL><INDENT>year = entry.fields['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AuthorYearError<EOL><DEDENT>if paren and len(persons) == <NUM_LIT:1>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) == <NUM_LIT:1>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif paren and len(persons) == <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author1=_format_last(persons[<NUM_LIT:0>]),<EOL>author2=_format_last(persons[<NUM_LIT:1>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) == <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author1=_format_last(persons[<NUM_LIT:0>]),<EOL>author2=_format_last(persons[<NUM_LIT:1>]),<EOL>year=year)<EOL><DEDENT>elif not paren and len(persons) > <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>elif paren and len(persons) > <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL>return template.format(author=_format_last(persons[<NUM_LIT:0>]),<EOL>year=year)<EOL><DEDENT>

Get and format author-year text from a pybtex entry to emulate
    natbib citations.

    Parameters
    ----------
    entry : `pybtex.database.Entry`
        A pybtex bibliography entry.
    parens : `bool`, optional
        Whether to add parentheses around the year. Default is `False`.

    Returns
    -------
    authoryear : `str`
        The author-year citation text.


api_root = '<STR_LIT>'<EOL>url = '<STR_LIT>'.format(<EOL>api_root=api_root,<EOL>id_=installation_id)<EOL>headers = {<EOL>'<STR_LIT>': '<STR_LIT>'.format(integration_jwt.decode('<STR_LIT:utf-8>')),<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>resp = requests.post(url, headers=headers)<EOL>resp.raise_for_status()<EOL>return resp.json()<EOL>

Create a GitHub token for an integration installation.

    Parameters
    ----------
    installation_id : `int`
        Installation ID. This is available in the URL of the integration's
        **installation** ID.
    integration_jwt : `bytes`
        The integration's JSON Web Token (JWT). You can create this with
        `create_jwt`.

    Returns
    -------
    token_obj : `dict`
        GitHub token object. Includes the fields:

        - ``token``: the token string itself.
        - ``expires_at``: date time string when the token expires.

    Example
    -------
    The typical workflow for authenticating to an integration installation is:

    .. code-block:: python

       from dochubadapter.github import auth
       jwt = auth.create_jwt(integration_id, private_key_path)
       token_obj = auth.get_installation_token(installation_id, jwt)
       print(token_obj['token'])

    Notes
    -----
    See
    https://developer.github.com/early-access/integrations/authentication/#as-an-installation
    for more information


integration_id = int(integration_id)<EOL>with open(private_key_path, '<STR_LIT:rb>') as f:<EOL><INDENT>cert_bytes = f.read()<EOL><DEDENT>now = datetime.datetime.now()<EOL>expiration_time = now + datetime.timedelta(minutes=<NUM_LIT:9>)<EOL>payload = {<EOL>'<STR_LIT>': int(now.timestamp()),<EOL>'<STR_LIT>': int(expiration_time.timestamp()),<EOL>'<STR_LIT>': integration_id<EOL>}<EOL>return jwt.encode(payload, cert_bytes, algorithm='<STR_LIT>')<EOL>

Create a JSON Web Token to authenticate a GitHub Integration or
    installation.

    Parameters
    ----------
    integration_id : `int`
        Integration ID. This is available from the GitHub integration's
        homepage.
    private_key_path : `str`
        Path to the integration's private key (a ``.pem`` file).

    Returns
    -------
    jwt : `bytes`
        JSON Web Token that is good for 9 minutes.

    Notes
    -----
    The JWT is encoded with the RS256 algorithm. It includes a payload with
    fields:

    - ``'iat'``: The current time, as an `int` timestamp.
    - ``'exp'``: Expiration time, as an `int timestamp. The expiration
      time is set of 9 minutes in the future (maximum allowance is 10 minutes).
    - ``'iss'``: The integration ID (`int`).

    For more information, see
    https://developer.github.com/early-access/integrations/authentication/.


match = GITHUB_SLUG_PATTERN.match(github_url)<EOL>if not match:<EOL><INDENT>message = '<STR_LIT>'.format(github_url)<EOL>raise RuntimeError(message)<EOL><DEDENT>_full = '<STR_LIT:/>'.join((match.group('<STR_LIT>'),<EOL>match.group('<STR_LIT:name>')))<EOL>return RepoSlug(_full, match.group('<STR_LIT>'), match.group('<STR_LIT:name>'))<EOL>

Get the slug, <owner>/<repo_name>, for a GitHub repository from
    its URL.

    Parameters
    ----------
    github_url : `str`
        URL of a GitHub repository.

    Returns
    -------
    repo_slug : `RepoSlug`
        Repository slug with fields ``full``, ``owner``, and ``repo``.
        See `RepoSlug` for details.

    Raises
    ------
    RuntimeError
        Raised if the URL cannot be parsed.


def make_raw_content_url(repo_slug, git_ref, file_path):

if isinstance(repo_slug, RepoSlug):<EOL><INDENT>slug_str = repo_slug.full<EOL><DEDENT>else:<EOL><INDENT>slug_str = repo_slug<EOL><DEDENT>if file_path.startswith('<STR_LIT:/>'):<EOL><INDENT>file_path = file_path.lstrip('<STR_LIT:/>')<EOL><DEDENT>template = '<STR_LIT>'<EOL>return template.format(<EOL>slug=slug_str,<EOL>git_ref=git_ref,<EOL>path=file_path)<EOL>

Make a raw content (raw.githubusercontent.com) URL to a file.

    Parameters
    ----------
    repo_slug : `str` or `RepoSlug`
        The repository slug, formatted as either a `str` (``'owner/name'``)
        or a `RepoSlug` object (created by `parse_repo_slug_from_url`).
    git_ref : `str`
        The git ref: a branch name, commit hash, or tag name.
    file_path : `str`
        The POSIX path of the file in the repository tree.


<EOL>if url.endswith('<STR_LIT>'):<EOL><INDENT>url = url[:-<NUM_LIT:4>]<EOL><DEDENT>return url<EOL>

Normalize a GitHub URL into the root repository URL.

    Parameters
    ----------
    url : `str`
        A GitHub URL

    Returns
    -------
    url : `str`
        Normalized URL of a GitHub repository.

    Examples
    --------
    >>> normalize_repo_root_url('https://github.com/lsst/LDM-151.git')
    'https://github.com/lsst/LDM-151'


async def github_request(session, api_token,<EOL>query=None, mutation=None, variables=None):

payload = {}<EOL>if query is not None:<EOL><INDENT>payload['<STR_LIT>'] = str(query)  <EOL><DEDENT>if mutation is not None:<EOL><INDENT>payload['<STR_LIT>'] = str(mutation)  <EOL><DEDENT>if variables is not None:<EOL><INDENT>payload['<STR_LIT>'] = variables<EOL><DEDENT>headers = {'<STR_LIT>': '<STR_LIT>'.format(api_token)}<EOL>url = '<STR_LIT>'<EOL>async with session.post(url, json=payload, headers=headers) as response:<EOL><INDENT>data = await response.json()<EOL><DEDENT>return data<EOL>

Send a request to the GitHub v4 (GraphQL) API.

    The request is asynchronous, with asyncio.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
    api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    query : `str` or `GitHubQuery`
        GraphQL query string. If provided, then the ``mutation`` parameter
        should not be set. For examples, see the `GitHub guide to query and
        mutation operations`_.
    mutation : `str` or `GitHubQuery`
        GraphQL mutation string. If provided, then the ``query`` parameter
        should not be set. For examples, see the `GitHub guide to query and
        mutation operations`_.
    variables : `dict`
        GraphQL variables, as a JSON-compatible dictionary. This is only
        required if the ``query`` or ``mutation`` uses GraphQL variables.

    Returns
    -------
    data : `dict`
        Parsed JSON as a `dict` object.

    .. `GitHub personal access token guide`: https://ls.st/41d
    .. `GitHub guide to query and mutation operations`: https://ls.st/9s7


template_path = os.path.join(<EOL>os.path.dirname(__file__),<EOL>'<STR_LIT>',<EOL>query_name + '<STR_LIT>')<EOL>with open(template_path) as f:<EOL><INDENT>query_data = f.read()<EOL><DEDENT>return cls(query_data, name=query_name)<EOL>

Load a pre-made query.

        These queries are distributed with lsstprojectmeta. See
        :file:`lsstrojectmeta/data/githubv4/README.rst` inside the
        package repository for details on available queries.

        Parameters
        ----------
        query_name : `str`
            Name of the query, such as ``'technote_repo'``.

        Returns
        -------
        github_query : `GitHubQuery
            A GitHub query or mutation object that you can pass to
            `github_request` to execute the request itself.


parser = argparse.ArgumentParser(<EOL>description='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>dest='<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>help='<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>default='<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>default='<STR_LIT>',<EOL>help='<STR_LIT>')<EOL>args = parser.parse_args()<EOL>stream_handler = logging.StreamHandler()<EOL>stream_formatter = logging.Formatter(<EOL>'<STR_LIT>')<EOL>stream_handler.setFormatter(stream_formatter)<EOL>root_logger = logging.getLogger()<EOL>root_logger.addHandler(stream_handler)<EOL>root_logger.setLevel(logging.WARNING)<EOL>app_logger = logging.getLogger('<STR_LIT>')<EOL>app_logger.setLevel(logging.DEBUG)<EOL>if args.mongodb_uri is not None:<EOL><INDENT>mongo_client = AsyncIOMotorClient(args.mongodb_uri, ssl=True)<EOL>collection = mongo_client[args.mongodb_db][args.mongodb_collection]<EOL><DEDENT>else:<EOL><INDENT>collection = None<EOL><DEDENT>loop = asyncio.get_event_loop()<EOL>if args.ltd_product_url is not None:<EOL><INDENT>loop.run_until_complete(run_single_ltd_doc(args.ltd_product_url,<EOL>args.github_token,<EOL>collection))<EOL><DEDENT>else:<EOL><INDENT>loop.run_until_complete(run_bulk_etl(args.github_token,<EOL>collection))<EOL><DEDENT>

Command line entrypoint to reduce technote metadata.


async def process_ltd_doc_products(session, product_urls, github_api_token,<EOL>mongo_collection=None):

tasks = [asyncio.ensure_future(<EOL>process_ltd_doc(session, github_api_token,<EOL>product_url,<EOL>mongo_collection=mongo_collection))<EOL>for product_url in product_urls]<EOL>await asyncio.gather(*tasks)<EOL>

Run a pipeline to process extract, transform, and load metadata for
    multiple LSST the Docs-hosted projects

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    product_urls : `list` of `str`
        List of LSST the Docs product URLs.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records.


async def process_ltd_doc(session, github_api_token, ltd_product_url,<EOL>mongo_collection=None):

logger = logging.getLogger(__name__)<EOL>ltd_product_data = await get_ltd_product(session, url=ltd_product_url)<EOL>product_name = ltd_product_data['<STR_LIT>']<EOL>doc_handle_match = DOCUMENT_HANDLE_PATTERN.match(product_name)<EOL>if doc_handle_match is None:<EOL><INDENT>logger.debug('<STR_LIT>', product_name)<EOL>return<EOL><DEDENT>try:<EOL><INDENT>return await process_sphinx_technote(session,<EOL>github_api_token,<EOL>ltd_product_data,<EOL>mongo_collection=mongo_collection)<EOL><DEDENT>except NotSphinxTechnoteError:<EOL><INDENT>logger.debug('<STR_LIT>', product_name)<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception('<STR_LIT>', product_name)<EOL>return<EOL><DEDENT>try:<EOL><INDENT>return await process_lander_page(session,<EOL>github_api_token,<EOL>ltd_product_data,<EOL>mongo_collection=mongo_collection)<EOL><DEDENT>except NotLanderPageError:<EOL><INDENT>logger.debug('<STR_LIT>',<EOL>product_name)<EOL><DEDENT>except Exception:<EOL><INDENT>logger.exception('<STR_LIT>', product_name)<EOL>return<EOL><DEDENT>

Ingest any kind of LSST document hosted on LSST the Docs from its
    source.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_url : `str`
        URL of the technote's product resource in the LTD Keeper API.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    .. `GitHub personal access token guide`: https://ls.st/41d


encoder = JsonLdEncoder(**kwargs)<EOL>return encoder.encode(jsonld_dataset)<EOL>

Encode a JSON-LD dataset into a string.

    Parameters
    ----------
    jsonld_dataset : `dict`
        A JSON-LD dataset.
    kwargs
        Keyword argument passed to the encoder. See `json.JSONEncoder`.

    Returns
    -------
    encoded_dataset : `str`
        The JSON-LD dataset encoded as a string.


decoder = json.JSONDecoder(object_pairs_hook=_decode_object_pairs)<EOL>return decoder.decode(jsonld_text)<EOL>

Decode a JSON-LD dataset, including decoding datetime
    strings into `datetime.datetime` objects.

    Parameters
    ----------
    encoded_dataset : `str`
        The JSON-LD dataset encoded as a string.

    Returns
    -------
    jsonld_dataset : `dict`
        A JSON-LD dataset.

    Examples
    --------

    >>> doc = '{"dt": "2018-01-01T12:00:00Z"}'
    >>> decode_jsonld(doc)
    {'dt': datetime.datetime(2018, 1, 1, 12, 0, tzinfo=datetime.timezone.utc)}


if isinstance(obj, datetime.datetime):<EOL><INDENT>return self._encode_datetime(obj)<EOL><DEDENT>return json.JSONEncoder.default(self, obj)<EOL>

Encode values as JSON strings.

        This method overrides the default implementation from
        `json.JSONEncoder`.


if dt.tzinfo is None:<EOL><INDENT>dt = dt.replace(tzinfo=datetime.timezone.utc)<EOL><DEDENT>dt = dt.astimezone(datetime.timezone.utc)<EOL>return dt.strftime('<STR_LIT>')<EOL>

Encode a datetime in the format '%Y-%m-%dT%H:%M:%SZ'.

        The datetime can be naieve (doesn't have timezone info) or aware
        (it does have a tzinfo attribute set). Regardless, the datetime
        is transformed into UTC.


async def process_sphinx_technote(session, github_api_token, ltd_product_data,<EOL>mongo_collection=None):

logger = logging.getLogger(__name__)<EOL>github_url = ltd_product_data['<STR_LIT>']<EOL>github_url = normalize_repo_root_url(github_url)<EOL>repo_slug = parse_repo_slug_from_url(github_url)<EOL>try:<EOL><INDENT>metadata_yaml = await download_metadata_yaml(session, github_url)<EOL><DEDENT>except aiohttp.ClientResponseError as err:<EOL><INDENT>logger.debug('<STR_LIT>',<EOL>ltd_product_data['<STR_LIT>'], err.code)<EOL>raise NotSphinxTechnoteError()<EOL><DEDENT>github_query = GitHubQuery.load('<STR_LIT>')<EOL>github_variables = {<EOL>"<STR_LIT>": repo_slug.owner,<EOL>"<STR_LIT>": repo_slug.repo<EOL>}<EOL>github_data = await github_request(session, github_api_token,<EOL>query=github_query,<EOL>variables=github_variables)<EOL>try:<EOL><INDENT>jsonld = reduce_technote_metadata(<EOL>github_url, metadata_yaml, github_data, ltd_product_data)<EOL><DEDENT>except Exception as exception:<EOL><INDENT>message = "<STR_LIT>"<EOL>logger.exception(message, github_url, exception)<EOL>raise<EOL><DEDENT>if mongo_collection is not None:<EOL><INDENT>await _upload_to_mongodb(mongo_collection, jsonld)<EOL><DEDENT>logger.info('<STR_LIT>', github_url)<EOL>return jsonld<EOL>

Extract, transform, and load Sphinx-based technote metadata.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_data : `dict`
        Contents of ``metadata.yaml``, obtained via `download_metadata_yaml`.
        Data for this technote from the LTD Keeper API
        (``GET /products/<slug>``). Usually obtained via
        `lsstprojectmeta.ltd.get_ltd_product`.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    Raises
    ------
    NotSphinxTechnoteError
        Raised when the LTD product cannot be interpreted as a Sphinx-based
        technote project because it's missing a metadata.yaml file in its
        GitHub repository. This implies that the LTD product *could* be of a
        different format.

    .. `GitHub personal access token guide`: https://ls.st/41d


def reduce_technote_metadata(github_url, metadata, github_data,<EOL>ltd_product_data):

repo_slug = parse_repo_slug_from_url(github_url)<EOL>jsonld = {<EOL>'<STR_LIT>': [<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"],<EOL>'<STR_LIT>': ['<STR_LIT>', '<STR_LIT>'],<EOL>'<STR_LIT>': github_url<EOL>}<EOL>if '<STR_LIT:url>' in metadata:<EOL><INDENT>url = metadata['<STR_LIT:url>']<EOL><DEDENT>elif '<STR_LIT>' in ltd_product_data:<EOL><INDENT>url = ltd_product_data['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'<EOL>'<STR_LIT:{}>'.format(github_url))<EOL><DEDENT>jsonld['<STR_LIT>'] = url<EOL>jsonld['<STR_LIT:url>'] = url<EOL>if '<STR_LIT>' in metadata and '<STR_LIT>' in metadata:<EOL><INDENT>jsonld['<STR_LIT>'] = '<STR_LIT>'.format(**metadata)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(github_url))<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>jsonld['<STR_LIT:name>'] = metadata['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT:description>' in metadata:<EOL><INDENT>jsonld['<STR_LIT:description>'] = metadata['<STR_LIT:description>']<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>jsonld['<STR_LIT>'] = [{'<STR_LIT>': '<STR_LIT>', '<STR_LIT:name>': author_name}<EOL>for author_name in metadata['<STR_LIT>']]<EOL><DEDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>jsonld['<STR_LIT>'] = datetime.datetime.strptime(<EOL>metadata['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>_repo_data = github_data['<STR_LIT:data>']['<STR_LIT>']<EOL>_master_data = _repo_data['<STR_LIT>']<EOL>jsonld['<STR_LIT>'] = datetime.datetime.strptime(<EOL>_master_data['<STR_LIT:target>']['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>_license_data = github_data['<STR_LIT:data>']['<STR_LIT>']['<STR_LIT>']<EOL>_spdxId = _license_data['<STR_LIT>']<EOL>if _spdxId is not None:<EOL><INDENT>_spdx_url = '<STR_LIT>'.format(_spdxId)<EOL>jsonld['<STR_LIT>'] = _spdx_url<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>_master_data = github_data['<STR_LIT:data>']['<STR_LIT>']['<STR_LIT>']<EOL>_files = _master_data['<STR_LIT:target>']['<STR_LIT>']['<STR_LIT>']<EOL>for _node in _files:<EOL><INDENT>filename = _node['<STR_LIT:name>']<EOL>normalized_filename = filename.lower()<EOL>if normalized_filename.startswith('<STR_LIT>'):<EOL><INDENT>readme_url = make_raw_content_url(repo_slug, '<STR_LIT>',<EOL>filename)<EOL>jsonld['<STR_LIT>'] = readme_url<EOL>break<EOL><DEDENT><DEDENT><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>travis_url = '<STR_LIT>'.format(repo_slug.full)<EOL>jsonld['<STR_LIT>'] = travis_url<EOL>return jsonld<EOL>

Reduce a technote project's metadata from multiple sources into a
    single JSON-LD resource.

    Parameters
    ----------
    github_url : `str`
        URL of the technote's GitHub repository.
    metadata : `dict`
        The parsed contents of ``metadata.yaml`` found in a technote's
        repository.
    github_data : `dict`
        The contents of the ``technote_repo`` GitHub GraphQL API query.
    ltd_product_data : `dict`
        JSON dataset for the technote corresponding to the
        ``/products/<product>`` of LTD Keeper.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    .. `GitHub personal access token guide`: https://ls.st/41d


async def download_metadata_yaml(session, github_url):

metadata_yaml_url = _build_metadata_yaml_url(github_url)<EOL>async with session.get(metadata_yaml_url) as response:<EOL><INDENT>response.raise_for_status()<EOL>yaml_data = await response.text()<EOL><DEDENT>return yaml.safe_load(yaml_data)<EOL>

Download the metadata.yaml file from a technote's GitHub repository.


repo_slug = parse_repo_slug_from_url(github_url)<EOL>return make_raw_content_url(repo_slug, '<STR_LIT>', '<STR_LIT>')<EOL>

Compute the URL to the raw metadata.yaml resource given the technote's
    GitHub repository URL.

    Parameters
    ----------
    github_url : `str`
        URL of the technote's GitHub repository.

    Returns
    -------
    metadata_yaml_url : `str`
        metadata.yaml URL (using the ``raw.githubusercontent.com`` domain).


async def _upload_to_mongodb(collection, jsonld):

document = {<EOL>'<STR_LIT:data>': jsonld<EOL>}<EOL>query = {<EOL>'<STR_LIT>': jsonld['<STR_LIT>']<EOL>}<EOL>await collection.update(query, document, upsert=True, multi=False)<EOL>

Upsert the technote resource into the projectmeta MongoDB collection.

    Parameters
    ----------
    collection : `motor.motor_asyncio.AsyncIOMotorCollection`
        The MongoDB collection.
    jsonld : `dict`
        The JSON-LD document that reprsents the technote resource.


async def process_lander_page(session, github_api_token, ltd_product_data,<EOL>mongo_collection=None):

logger = logging.getLogger(__name__)<EOL>published_url = ltd_product_data['<STR_LIT>']<EOL>jsonld_url = urljoin(published_url, '<STR_LIT>')<EOL>try:<EOL><INDENT>async with session.get(jsonld_url) as response:<EOL><INDENT>logger.debug('<STR_LIT>', jsonld_url, response.status)<EOL>response.raise_for_status()<EOL>json_data = await response.text()<EOL><DEDENT><DEDENT>except aiohttp.ClientResponseError as err:<EOL><INDENT>logger.debug('<STR_LIT>',<EOL>jsonld_url, err.code)<EOL>raise NotLanderPageError()<EOL><DEDENT>metadata = decode_jsonld(json_data)<EOL>if mongo_collection is not None:<EOL><INDENT>await _upload_to_mongodb(mongo_collection, metadata)<EOL><DEDENT>return metadata<EOL>

Extract, transform, and load metadata from Lander-based projects.

    Parameters
    ----------
    session : `aiohttp.ClientSession`
        Your application's aiohttp client session.
        See http://aiohttp.readthedocs.io/en/stable/client.html.
    github_api_token : `str`
        A GitHub personal API token. See the `GitHub personal access token
        guide`_.
    ltd_product_data : `dict`
        Contents of ``metadata.yaml``, obtained via `download_metadata_yaml`.
        Data for this technote from the LTD Keeper API
        (``GET /products/<slug>``). Usually obtained via
        `lsstprojectmeta.ltd.get_ltd_product`.
    mongo_collection : `motor.motor_asyncio.AsyncIOMotorCollection`, optional
        MongoDB collection. This should be the common MongoDB collection for
        LSST projectmeta JSON-LD records. If provided, ths JSON-LD is upserted
        into the MongoDB collection.

    Returns
    -------
    metadata : `dict`
        JSON-LD-formatted dictionary.

    Raises
    ------
    NotLanderPageError
        Raised when the LTD product cannot be interpreted as a Lander page
        because the ``/metadata.jsonld`` file is absent. This implies that
        the LTD product *could* be of a different format.

    .. `GitHub personal access token guide`: https://ls.st/41d


Upsert the technote resource into the projectmeta MongoDB collection.

    Parameters
    ----------
    collection : `motor.motor_asyncio.AsyncIOMotorCollection`
        The MongoDB collection.
    jsonld : `dict`
        The JSON-LD document that represents the document resource.


if repo is None:<EOL><INDENT>repo = git.repo.base.Repo(path=repo_path,<EOL>search_parent_directories=True)<EOL><DEDENT>head_commit = repo.head.commit<EOL>return head_commit.committed_datetime<EOL>

Obtain the timestamp from the current head commit of a Git repository.

    Parameters
    ----------
    repo_path : `str`, optional
        Path to the Git repository. Leave as `None` to use the current working
        directory.

    Returns
    -------
    commit_timestamp : `datetime.datetime`
        The datetime of the head commit.


def read_git_commit_timestamp_for_file(filepath, repo_path=None, repo=None):

logger = logging.getLogger(__name__)<EOL>if repo is None:<EOL><INDENT>repo = git.repo.base.Repo(path=repo_path,<EOL>search_parent_directories=True)<EOL><DEDENT>repo_path = repo.working_tree_dir<EOL>head_commit = repo.head.commit<EOL>logger.debug('<STR_LIT>', repo_path)<EOL>filepath = os.path.relpath(<EOL>os.path.abspath(filepath),<EOL>start=repo_path)<EOL>logger.debug('<STR_LIT>', filepath)<EOL>for commit in head_commit.iter_items(repo,<EOL>head_commit,<EOL>[filepath],<EOL>skip=<NUM_LIT:0>):<EOL><INDENT>return commit.committed_datetime<EOL><DEDENT>raise IOError('<STR_LIT>'.format(filepath))<EOL>

Obtain the timestamp for the most recent commit to a given file in a
    Git repository.

    Parameters
    ----------
    filepath : `str`
        Absolute or repository-relative path for a file.
    repo_path : `str`, optional
        Path to the Git repository. Leave as `None` to use the current working
        directory or if a ``repo`` argument is provided.
    repo : `git.Repo`, optional
        A `git.Repo` instance.

    Returns
    -------
    commit_timestamp : `datetime.datetime`
        The datetime of the most recent commit to the given file.

    Raises
    ------
    IOError
        Raised if the ``filepath`` does not exist in the Git repository.


logger = logging.getLogger(__name__)<EOL>def _null_callback(_):<EOL><INDENT>return True<EOL><DEDENT>if acceptance_callback is None:<EOL><INDENT>acceptance_callback = _null_callback<EOL><DEDENT>root_dir = os.path.abspath(root_dir)<EOL>repo = git.repo.base.Repo(path=root_dir, search_parent_directories=True)<EOL>newest_datetime = None<EOL>iters = [_iter_filepaths_with_extension(ext, root_dir=root_dir)<EOL>for ext in extensions]<EOL>for content_path in itertools.chain(*iters):<EOL><INDENT>content_path = os.path.abspath(os.path.join(root_dir, content_path))<EOL>if acceptance_callback(content_path):<EOL><INDENT>logger.debug('<STR_LIT>', content_path)<EOL>try:<EOL><INDENT>commit_datetime = read_git_commit_timestamp_for_file(<EOL>content_path, repo=repo)<EOL>logger.debug('<STR_LIT>',<EOL>content_path, commit_datetime)<EOL><DEDENT>except IOError:<EOL><INDENT>logger.warning(<EOL>'<STR_LIT>',<EOL>content_path)<EOL>continue<EOL><DEDENT>if not newest_datetime or commit_datetime > newest_datetime:<EOL><INDENT>newest_datetime = commit_datetime<EOL>logger.debug('<STR_LIT>', newest_datetime)<EOL><DEDENT><DEDENT>logger.debug('<STR_LIT>', newest_datetime)<EOL><DEDENT>if newest_datetime is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(root_dir))<EOL><DEDENT>return newest_datetime<EOL>

Get the datetime for the most recent commit to a project that
    affected certain types of content.

    Parameters
    ----------
    extensions : sequence of 'str'
        Extensions of files to consider in getting the most recent commit
        date. For example, ``('rst', 'svg', 'png')`` are content extensions
        for a Sphinx project. **Extension comparision is case sensitive.** add
        uppercase variants to match uppercase extensions.
    acceptance_callback : callable
        Callable function whose sole argument is a file path, and returns
        `True` or `False` depending on whether the file's commit date should
        be considered or not. This callback is only run on files that are
        included by ``extensions``. Thus this callback is a way to exclude
        specific files that would otherwise be included by their extension.
    root_dir : 'str`, optional
        Only content contained within this root directory is considered.
        This directory must be, or be contained by, a Git repository. This is
        the current working directory by default.

    Returns
    -------
    commit_date : `datetime.datetime`
        Datetime of the most recent content commit.

    Raises
    ------
    RuntimeError
        Raised if no content files are found.


<EOL>if not extname.startswith('<STR_LIT:.>'):<EOL><INDENT>extname = '<STR_LIT:.>' + extname<EOL><DEDENT>root_dir = os.path.abspath(root_dir)<EOL>for dirname, sub_dirnames, filenames in os.walk(root_dir):<EOL><INDENT>for filename in filenames:<EOL><INDENT>if os.path.splitext(filename)[-<NUM_LIT:1>] == extname:<EOL><INDENT>full_filename = os.path.join(dirname, filename)<EOL>rel_filepath = os.path.relpath(full_filename, start=root_dir)<EOL>yield rel_filepath<EOL><DEDENT><DEDENT><DEDENT>

Iterative over relative filepaths of files in a directory, and
    sub-directories, with the given extension.

    Parameters
    ----------
    extname : `str`
        Extension name (such as 'txt' or 'rst'). Extension comparison is
        case sensitive.
    root_dir : 'str`, optional
        Root directory. Current working directory by default.

    Yields
    ------
    filepath : `str`
        File path, relative to ``root_dir``, with the given extension.


self._path_to_tagger = path_to_tagger<EOL>self._dir_to_tagger = os.path.dirname(path_to_tagger)<EOL>self._tagger = subprocess.Popen('<STR_LIT>'+os.path.basename(path_to_tagger),<EOL>cwd=self._dir_to_tagger,<EOL>stdin=subprocess.PIPE, stdout=subprocess.PIPE)<EOL>


results = list()<EOL>for oneline in text.split('<STR_LIT:\n>'):<EOL><INDENT>self._tagger.stdin.write(oneline+'<STR_LIT:\n>')<EOL>while True:<EOL><INDENT>r = self._tagger.stdout.readline()[:-<NUM_LIT:1>]<EOL>if not r:<EOL><INDENT>break<EOL><DEDENT>results.append(tuple(r.split('<STR_LIT:\t>')))<EOL><DEDENT><DEDENT>return results<EOL>