def _from_wkt(string, wkttype=None, strict=False):

<EOL>if wkttype: wkttype = wkttype.lower()<EOL>assert wkttype in ("<STR_LIT>","<STR_LIT>",None)<EOL>string = "<STR_LIT:U+0020>".join(string.split())<EOL>def _consume_bracket(chars, char):<EOL><INDENT>"<STR_LIT>"<EOL>consumed = "<STR_LIT>"<EOL>depth = <NUM_LIT:1><EOL>while char and depth > <NUM_LIT:0>:<EOL><INDENT>consumed += char<EOL>char = next(chars, None)<EOL>if char == "<STR_LIT:[>":<EOL><INDENT>depth += <NUM_LIT:1><EOL><DEDENT>elif char == "<STR_LIT:]>":<EOL><INDENT>depth -= <NUM_LIT:1><EOL><DEDENT><DEDENT>consumed += char <EOL>return consumed<EOL><DEDENT>def _consume_quote(chars, char, quotechar):<EOL><INDENT>"<STR_LIT>"<EOL>consumed = "<STR_LIT>"<EOL>consumed += char <EOL>char = next(chars, None)<EOL>while char and char != quotechar:<EOL><INDENT>consumed += char<EOL>char = next(chars, None)<EOL><DEDENT>consumed += char <EOL>return consumed<EOL><DEDENT>def _next_elem(chars, char):<EOL><INDENT>"<STR_LIT>"<EOL>header = "<STR_LIT>"<EOL>while not char.isalpha():<EOL><INDENT>char = next(chars, None)<EOL><DEDENT>while char.isalpha():<EOL><INDENT>header += char<EOL>char = next(chars, None)<EOL><DEDENT>while char != "<STR_LIT:[>":<EOL><INDENT>char = next(chars, None)<EOL><DEDENT>if char == "<STR_LIT:[>":<EOL><INDENT>content = _consume_bracket(chars, char)<EOL><DEDENT>char = next(chars, None)<EOL>content = content[<NUM_LIT:1>:-<NUM_LIT:1>] <EOL>content = _split_except(content)<EOL>for i,item in enumerate(content):<EOL><INDENT>if isinstance(item, str) and "<STR_LIT:[>" in item:<EOL><INDENT>chars = (char for char in item)<EOL>char = next(chars)<EOL>item = _next_elem(chars, char)<EOL>content[i] = item<EOL><DEDENT><DEDENT>return header, content<EOL><DEDENT>def _clean_value(string):<EOL><INDENT>string = string.strip()<EOL>try: string = float(string)<EOL>except: pass<EOL>return string<EOL><DEDENT>def _split_except(string):<EOL><INDENT>"<STR_LIT>"<EOL>chars = (char for char in string)<EOL>char = next(chars)<EOL>items = []<EOL>consumed = "<STR_LIT>"<EOL>while char:<EOL><INDENT>if char in ("<STR_LIT:'>", '<STR_LIT:">'):<EOL><INDENT>consumed += _consume_quote(chars, char, char)<EOL><DEDENT>elif char == "<STR_LIT:[>":<EOL><INDENT>consumed += _consume_bracket(chars, char)<EOL><DEDENT>elif char == "<STR_LIT:U+002C>":<EOL><INDENT>consumed = _clean_value(consumed)<EOL>items.append(consumed)<EOL>consumed = "<STR_LIT>"<EOL><DEDENT>elif char:<EOL><INDENT>consumed += char<EOL><DEDENT>char = next(chars, None)<EOL><DEDENT>consumed = _clean_value(consumed)<EOL>items.append(consumed)<EOL>return items<EOL><DEDENT>crstuples = []<EOL>chars = (char for char in string)<EOL>char = next(chars)<EOL>while char:<EOL><INDENT>header,content = _next_elem(chars, char)<EOL>crstuples.append((header, content))<EOL>char = next(chars, None)<EOL><DEDENT>if not wkttype:<EOL><INDENT>topheader,topcontent = crstuples[<NUM_LIT:0>]<EOL>if topheader == "<STR_LIT>":<EOL><INDENT>geogcsheader,geogcscontent = topcontent[<NUM_LIT:1>]<EOL><DEDENT>elif topheader == "<STR_LIT>":<EOL><INDENT>geogcsheader,geogcscontent = topheader,topcontent<EOL><DEDENT>datumheader, datumcontent = geogcscontent[<NUM_LIT:1>]<EOL>datumname = datumcontent[<NUM_LIT:0>].upper().strip('<STR_LIT:">')<EOL>if datumname.startswith("<STR_LIT>"):<EOL><INDENT>wkttype = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>wkttype = "<STR_LIT>"<EOL><DEDENT><DEDENT>def _parse_top(header, content):<EOL><INDENT>"<STR_LIT>"<EOL>if header.upper() == "<STR_LIT>":<EOL><INDENT>csname = content[<NUM_LIT:0>].strip('<STR_LIT:">')<EOL>subheader, subcontent = content[<NUM_LIT:1>]<EOL>geogcs = _parse_top(subheader, subcontent)<EOL>for part in content:<EOL><INDENT>if isinstance(part, tuple):<EOL><INDENT>subheader,subcontent = part<EOL>if subheader == "<STR_LIT>":<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>projname = subcontent[<NUM_LIT:0>].strip('<STR_LIT:">')<EOL>projclass = projections.find(projname, "<STR_LIT>" % wkttype, strict)<EOL>if projclass:<EOL><INDENT>proj = projclass()<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>" % projname)<EOL><DEDENT>params = []<EOL>for part in content:<EOL><INDENT>if isinstance(part, tuple):<EOL><INDENT>subheader,subcontent = part<EOL>if subheader == "<STR_LIT>":<EOL><INDENT>name, value = subcontent[<NUM_LIT:0>].strip('<STR_LIT:">'), subcontent[<NUM_LIT:1>]<EOL>itemclass = parameters.find(name, "<STR_LIT>" % wkttype, strict)<EOL>if itemclass:<EOL><INDENT>item = itemclass(value)<EOL>params.append(item)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for part in content:<EOL><INDENT>if isinstance(part, tuple):<EOL><INDENT>subheader,subcontent = part<EOL>if subheader == "<STR_LIT>":<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>unitname,value = subcontent[<NUM_LIT:0>].strip('<STR_LIT:">'), subcontent[<NUM_LIT:1>]<EOL>unitclass = units.find(unitname, "<STR_LIT>" % wkttype, strict)<EOL>if unitclass:<EOL><INDENT>unit = unitclass()<EOL><DEDENT>else:<EOL><INDENT>unit = units.Unknown()<EOL><DEDENT>unit.unitmultiplier.value = value <EOL>linunit = unit<EOL><INDENT>if len(content) >= <NUM_LIT:6>:<EOL><INDENT>twinax = (parameters.Axis(<EOL>else:<EOL>twinax = None<EOL>projcs = containers.ProjCS(csname, geogcs, proj, params, linunit) <EOL>return projcs<EOL>elif header.upper() == "<STR_LIT>":<EOL>csname = content[<NUM_LIT:0>].strip('<STR_LIT:">')<EOL>subheader, subcontent = content[<NUM_LIT:1>]<EOL>datumname = subcontent[<NUM_LIT:0>].strip('<STR_LIT:">')<EOL>datumclass = datums.find(datumname, "<STR_LIT>" % wkttype, strict)<EOL>if datumclass:<EOL>datum = datumclass()<EOL>else:<EOL>datum = datums.Unknown()<EOL>subsubheader, subsubcontent = subcontent[<NUM_LIT:1>]<EOL>ellipsname = subsubcontent[<NUM_LIT:0>].strip('<STR_LIT:">')<EOL>ellipsclass = ellipsoids.find(ellipsname, "<STR_LIT>" % wkttype, strict)<EOL>if ellipsclass:<EOL>ellipsoid = ellipsclass()<EOL>else:<EOL>ellipsoid = ellipsoids.Unknown()<EOL>ellipsoid.semimaj_ax = parameters.SemiMajorRadius(subsubcontent[<NUM_LIT:1>])<EOL>if subsubcontent[<NUM_LIT:2>] == <NUM_LIT:0>:<EOL>ellipsoid.flat = parameters.Flattening(subsubcontent[<NUM_LIT:2>])<EOL>else:<EOL>ellipsoid.inv_flat = parameters.InverseFlattening(subsubcontent[<NUM_LIT:2>])<EOL>if wkttype == "<STR_LIT>":<EOL>for subsubheader,subsubcontent in subcontent[<NUM_LIT:1>:]:<EOL>if subsubheader == "<STR_LIT>":<EOL>datumshift = parameters.DatumShift(subsubcontent)<EOL>break<EOL>else:<EOL>datumshift = None<EOL>elif wkttype == "<STR_LIT>":<EOL>datumshift = None<EOL>datum.ellips = ellipsoid<EOL>datum.datumshift = datumshift<EOL>subheader, subcontent = content[<NUM_LIT:2>]<EOL>prime_mer = parameters.PrimeMeridian(subcontent[<NUM_LIT:1>])<EOL>subheader, subcontent = content[<NUM_LIT:3>]<EOL>unitname,value = subcontent[<NUM_LIT:0>].strip('<STR_LIT:">'), subcontent[<NUM_LIT:1>]<EOL>unitclass = units.find(unitname, "<STR_LIT>" % wkttype, strict)<EOL>if unitclass:<EOL>unit = unitclass()<EOL>else:<EOL>unit = units.Unknown()<EOL>unit.unitmultiplier.value = value <EOL>angunit = unit<EOL>geogcs = containers.GeogCS(csname, datum, prime_mer, angunit, twin_ax=None)<EOL>return geogcs<EOL>header, content = crstuples[<NUM_LIT:0>]<EOL>crs = _parse_top(header, content)<EOL>return crs<EOL>

Internal method for parsing wkt, with minor differences depending on ogc or esri style.

Arguments:

- *string*: The OGC or ESRI WKT representation as a string.
- *wkttype* (optional): How to parse the WKT string, as either 'ogc', 'esri', or None. If None, tries to autodetect the wkt type before parsing (default).
- *strict* (optional): When True, the parser is strict about names having to match
    exactly with upper and lowercases. Default is not strict (False).

Returns:

- A CS instance of the indicated type.


<EOL>params = []<EOL>if isinstance(proj4, dict):<EOL><INDENT>partdict = dict([('<STR_LIT:+>'+k,v) for k,v in proj4.items()])<EOL><DEDENT>else: <EOL><INDENT>partdict = dict([part.split("<STR_LIT:=>") for part in proj4.split()<EOL>if len(part.split("<STR_LIT:=>")) == <NUM_LIT:2> ])<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>codetype, code = partdict["<STR_LIT>"].split("<STR_LIT::>")<EOL>if codetype == "<STR_LIT>":<EOL><INDENT>initproj4 = utils.crscode_to_string("<STR_LIT>", code, "<STR_LIT>")<EOL><DEDENT>elif codetype == "<STR_LIT>":<EOL><INDENT>initproj4 = utils.crscode_to_string("<STR_LIT>", code, "<STR_LIT>")<EOL><DEDENT>initpartdict = dict([part.split("<STR_LIT:=>") for part in initproj4.split()<EOL>if len(part.split("<STR_LIT:=>")) == <NUM_LIT:2> ])<EOL>initpartdict.update(partdict)<EOL>del initpartdict["<STR_LIT>"]<EOL>string = "<STR_LIT:U+0020>".join("<STR_LIT>" % (key,val) for key,val in initpartdict.items())<EOL>return from_proj4(string)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>datumname = partdict["<STR_LIT>"]<EOL>datumclass = datums.find(datumname, "<STR_LIT>", strict)<EOL>if datumclass:<EOL><INDENT>datum = datumclass()<EOL><DEDENT>else:<EOL><INDENT>datum = datums.Unknown()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>datum = datums.Unknown()<EOL><DEDENT>ellips = None<EOL>if "<STR_LIT>" in partdict:<EOL><INDENT>ellipsname = partdict["<STR_LIT>"]<EOL>ellipsclass = ellipsoids.find(ellipsname, "<STR_LIT>", strict)<EOL>if ellipsclass:<EOL><INDENT>ellips = ellipsclass()<EOL><DEDENT><DEDENT>if not ellips:<EOL><INDENT>ellips = ellipsoids.Unknown()<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>coeffs = partdict["<STR_LIT>"].split("<STR_LIT:U+002C>")<EOL>datumshift = parameters.DatumShift(coeffs)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>ellips.semimaj_ax = parameters.SemiMajorRadius(partdict["<STR_LIT>"])        <EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>ellips.semimin_ax = parameters.SemiMinorRadius(partdict["<STR_LIT>"])<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>ellips.flat = parameters.Flattening(partdict["<STR_LIT>"])<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>ellips.inv_flat = parameters.InverseFlattening(partdict["<STR_LIT>"])<EOL><DEDENT>if ellips.semimaj_ax and ellips.semimin_ax:<EOL><INDENT>pass<EOL><DEDENT>elif ellips.semimaj_ax and ellips.inv_flat:<EOL><INDENT>pass<EOL><DEDENT>elif ellips.semimaj_ax and ellips.flat:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>raise FormatError("<STR_LIT>" % partdict)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>datum.ellips = ellips<EOL><DEDENT>elif "<STR_LIT>" in partdict:<EOL><INDENT>datum.ellips = ellips<EOL>datum.datumshift = datumshift<EOL><DEDENT>else:<EOL><INDENT>datum.ellips = ellips<EOL><DEDENT>prime_mer = parameters.PrimeMeridian(<NUM_LIT:0>)<EOL>if "<STR_LIT>" in partdict:<EOL><INDENT>prime_mer = parameters.PrimeMeridian(partdict["<STR_LIT>"])<EOL><DEDENT>angunit = units.Degree() <EOL>geogcs = containers.GeogCS("<STR_LIT>", datum, prime_mer, angunit) <EOL>if "<STR_LIT>" in partdict:<EOL><INDENT>projname = partdict["<STR_LIT>"]<EOL>projclass = projections.find(projname, "<STR_LIT>", strict)<EOL>if projclass:<EOL><INDENT>proj = projclass()<EOL><DEDENT>elif projname == "<STR_LIT>":<EOL><INDENT>proj = None<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>" % projname)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise FormatError("<STR_LIT>")<EOL><DEDENT>if proj:<EOL><INDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.CentralMeridian(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.FalseEasting(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.FalseNorthing(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict or "<STR_LIT>" in partdict:<EOL><INDENT>if "<STR_LIT>" in partdict: val = partdict["<STR_LIT>"]<EOL>elif "<STR_LIT>" in partdict: val = partdict["<STR_LIT>"]<EOL>obj = parameters.ScalingFactor(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.LatitudeOrigin(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.LatitudeTrueScale(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.LongitudeCenter(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.Azimuth(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.LatitudeFirstStndParallel(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.LatitudeSecondStndParallel(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.SatelliteHeight(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>val = partdict["<STR_LIT>"]<EOL>obj = parameters.TiltAngle(val)<EOL>params.append(obj)<EOL><DEDENT>if "<STR_LIT>" in partdict:<EOL><INDENT>unitname = partdict["<STR_LIT>"]<EOL>unitclass = units.find(unitname, "<STR_LIT>", strict)<EOL>if unitclass:<EOL><INDENT>unit = unitclass() <EOL><DEDENT>else:<EOL><INDENT>raise FormatError("<STR_LIT>" % unitname)<EOL><DEDENT><DEDENT>elif "<STR_LIT>" in partdict:<EOL><INDENT>unit = units.Unknown()<EOL>unit.metermultiplier.value = partdict["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>unit = units.Meter()<EOL><DEDENT>projcs = containers.ProjCS("<STR_LIT>", geogcs, proj, params, unit)<EOL>return projcs<EOL><DEDENT>else:<EOL><INDENT>return geogcs<EOL><DEDENT>

Parse crs as proj4 formatted string or dict and return the resulting crs object.

Arguments:

- *proj4*: The proj4 representation as a string or dict.
- *strict* (optional): When True, the parser is strict about names having to match
    exactly with upper and lowercases. Default is not strict (False).

Returns:

- A CS instance of the indicated type.


if text.startswith("<STR_LIT:+>"):<EOL><INDENT>crs = from_proj4(text, strict)<EOL><DEDENT>elif text.startswith(("<STR_LIT>","<STR_LIT>")):<EOL><INDENT>crs = from_unknown_wkt(text, strict)<EOL><DEDENT>elif text.startswith("<STR_LIT>"):<EOL><INDENT>crs = from_epsg_code(text.split("<STR_LIT::>")[<NUM_LIT:1>])<EOL><DEDENT>elif text.startswith("<STR_LIT>"):<EOL><INDENT>crs = from_esri_code(text.split("<STR_LIT::>")[<NUM_LIT:1>])<EOL><DEDENT>elif text.startswith("<STR_LIT>"):<EOL><INDENT>crs = from_sr_code(text.split("<STR_LIT::>")[<NUM_LIT:1>])<EOL><DEDENT>else: raise FormatError("<STR_LIT>")<EOL>return crs<EOL>

Detect crs string format and parse into crs object with appropriate function.

Arguments:

- *text*: The crs text representation of unknown type.
- *strict* (optional): When True, the parser is strict about names having to match
    exactly with upper and lowercases. Default is not strict (False).

Returns:

- CRS object.


<EOL>string = urllib2.urlopen(url).read()<EOL>if PY3 is True:<EOL><INDENT>string = string.decode('<STR_LIT:utf-8>')<EOL><DEDENT>if format:<EOL><INDENT>format = format.lower().replace("<STR_LIT:U+0020>", "<STR_LIT:_>")<EOL>func = parse.__getattr__("<STR_LIT>" % format)<EOL><DEDENT>else:<EOL><INDENT>func = parse.from_unknown_text<EOL><DEDENT>crs = func(string)<EOL>return crs<EOL>

Returns the crs object from a string interpreted as a specified format, located at a given url site.

Arguments:

- *url*: The url where the crs string is to be read from.
- *format* (optional): Which format to parse the crs string as. One of "ogc wkt", "esri wkt", or "proj4".
    If None, tries to autodetect the format for you (default).

Returns:

- CRS object.


if filepath.endswith("<STR_LIT>"):<EOL><INDENT>string = open(filepath, "<STR_LIT:r>").read()<EOL>return parse.from_unknown_wkt(string)<EOL><DEDENT>elif filepath.endswith(("<STR_LIT>","<STR_LIT>")):<EOL><INDENT>raw = open(filepath).read()<EOL>geoj = json.loads(raw)<EOL>if "<STR_LIT>" in geoj:<EOL><INDENT>crsinfo = geoj["<STR_LIT>"]<EOL>if crsinfo["<STR_LIT:type>"] == "<STR_LIT:name>":<EOL><INDENT>string = crsinfo["<STR_LIT>"]["<STR_LIT:name>"]<EOL>return parse.from_unknown_text(string)<EOL><DEDENT>elif crsinfo["<STR_LIT:type>"] == "<STR_LIT>":<EOL><INDENT>url = crsinfo["<STR_LIT>"]["<STR_LIT:name>"]<EOL>type = crsinfo["<STR_LIT>"].get("<STR_LIT:type>")<EOL>return from_url(url, format=type)<EOL><DEDENT>else: raise FormatError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return parse.from_epsg_code("<STR_LIT>")<EOL><DEDENT><DEDENT>

Returns the crs object from a file, with the format determined from the filename extension.

Arguments:

- *filepath*: filepath to be loaded, including extension.


res = OrderedDict()<EOL>for k, v in sorted(d.items()):<EOL><INDENT>if isinstance(v, dict):<EOL><INDENT>res[k] = sort_dict(v)<EOL><DEDENT>else:<EOL><INDENT>res[k] = v<EOL><DEDENT><DEDENT>return res<EOL>

Helper function; returns a recursively sorted OrderedDict (by key).
:param d: A dictionary to sort.
:return: An OrderedDict.


if not isinstance(data, str):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(data).__name__)<EOL><DEDENT>parsed = wrapper()<EOL>current_section = parsed<EOL>sections = []<EOL>lines = (line.strip() for line in data.splitlines())<EOL>for line in lines:<EOL><INDENT>try:<EOL><INDENT>key, value = line.split(None, <NUM_LIT:1>)<EOL>key = key.replace('<STR_LIT:">', '<STR_LIT>').lstrip()<EOL>value = value.replace('<STR_LIT:">', '<STR_LIT>').rstrip()<EOL><DEDENT>except ValueError:<EOL><INDENT>if line == SECTION_START:<EOL><INDENT>current_section = _prepare_subsection(parsed, sections, wrapper)<EOL><DEDENT>elif line == SECTION_END:<EOL><INDENT>sections.pop()<EOL><DEDENT>else:<EOL><INDENT>sections.append(line.replace('<STR_LIT:">', '<STR_LIT>'))<EOL><DEDENT>continue<EOL><DEDENT>current_section[key] = value<EOL><DEDENT>return parsed<EOL>

Loads ACF content into a Python object.
:param data: An UTF-8 encoded content of an ACF file.
:param wrapper: A wrapping object for key-value pairs.
:return: An Ordered Dictionary with ACF data.


return loads(fp.read(), wrapper=wrapper)<EOL>

Loads the contents of an ACF file into a Python object.
:param fp: A file object.
:param wrapper: A wrapping object for key-value pairs.
:return: An Ordered Dictionary with ACF data.


if not isinstance(obj, dict):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(obj).__name__)<EOL><DEDENT>return '<STR_LIT:\n>'.join(_dumps(obj, level=<NUM_LIT:0>)) + '<STR_LIT:\n>'<EOL>

Serializes a dictionary into ACF data.
:param obj: A dictionary to serialize.
:return: ACF data.


fp.write(dumps(obj))<EOL>

Serializes a dictionary into ACF data and writes it to a file.
:param obj: A dictionary to serialize.
:param fp: A file object.


lines = []<EOL>indent = '<STR_LIT:\t>' * level<EOL>for key, value in obj.items():<EOL><INDENT>if isinstance(value, dict):<EOL><INDENT>line = indent + '<STR_LIT>'.format(key) + indent + '<STR_LIT:{>'<EOL>lines.append(line)<EOL>lines.extend(_dumps(value, level+<NUM_LIT:1>))<EOL>lines.append(indent + '<STR_LIT:}>')<EOL><DEDENT>else:<EOL><INDENT>lines.append(indent + '<STR_LIT>'.format(key) + '<STR_LIT>' + '<STR_LIT>'.format(value))<EOL><DEDENT><DEDENT>return lines<EOL>

Does the actual serializing of data into an ACF format.
:param obj: A dictionary to serialize.
:param level: Nesting level.
:return: A List of strings.


def _prepare_subsection(data, sections, wrapper):

current = data<EOL>for i in sections[:-<NUM_LIT:1>]:<EOL><INDENT>current = current[i]<EOL><DEDENT>current[sections[-<NUM_LIT:1>]] = wrapper()<EOL>return current[sections[-<NUM_LIT:1>]]<EOL>

Creates a subsection ready to be filled.
:param data: Semi-parsed dictionary.
:param sections: A list of sections.
:param wrapper: A wrapping object for key-value pairs.
:return: A newly created subsection.


Loads the contents of an Appinfo file into a Python object.
:param fp: A file object.
:param wrapper: A wrapping object for key-value pairs.
:return: An Ordered Dictionary with Appinfo data.


if not isinstance(data, (bytes, bytearray)):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(data).__name__)<EOL><DEDENT>return AppinfoDecoder(data, wrapper=wrapper).decode()<EOL>

Loads Appinfo content into a Python object.
:param data: A byte-like object with the contents of an Appinfo file.
:param wrapper: A wrapping object for key-value pairs.
:return: An Ordered Dictionary with Appinfo data.


fp.write(dumps(obj))<EOL>

Serializes a dictionary into Appinfo data and writes it to a file.
:param obj: A dictionary to serialize.
:param fp: A file object.


if not isinstance(obj, dict):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(obj).__name__)<EOL><DEDENT>return b'<STR_LIT>'.join(AppinfoEncoder(obj).iter_encode())<EOL>

Serializes a dictionary into Appinfo data.
:param obj: A dictionary to serialize.
:return:


Loads the contents of a Manifest file into a Python object.
:param fp: A file object.
:param wrapper: A wrapping object for key-value pairs.
:return: A dictionary with Manifest data.


if not isinstance(data, (bytes, bytearray)):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(data).__name__)<EOL><DEDENT>offset = <NUM_LIT:0><EOL>parsed = wrapper()<EOL>int32 = struct.Struct('<STR_LIT>')<EOL>while True:<EOL><INDENT>msg_id, = int32.unpack_from(data, offset)<EOL>offset += int32.size<EOL>if msg_id == MSG_EOF:<EOL><INDENT>break<EOL><DEDENT>msg_size, = int32.unpack_from(data, offset)<EOL>offset += int32.size<EOL>msg_data = data[offset:offset + msg_size]<EOL>offset += msg_size<EOL>message = MessageClass[msg_id]()<EOL>message.ParseFromString(msg_data)<EOL>parsed[MSG_NAMES[msg_id]] = wrapper(protobuf_to_dict(message))<EOL><DEDENT>return parsed<EOL>

Loads Manifest content into a Python object.
:param data: A byte-like object with the contents of an Appinfo file.
:param wrapper: A wrapping object for key-value pairs.
:return: A dictionary with Manifest data.


fp.write(dumps(obj))<EOL>

Serializes a dictionary into Manifest data and writes it to a file.
:param obj: A dictionary to serialize.
:param fp: A file object.


if not isinstance(obj, dict):<EOL><INDENT>raise TypeError('<STR_LIT>' + type(obj).__name__)<EOL><DEDENT>data = []<EOL>int32 = struct.Struct('<STR_LIT>')<EOL>for message_name in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>message_data = obj[message_name]<EOL>message_id = MSG_IDS[message_name]<EOL>message_class = MessageClass[message_id]<EOL>message = dict_to_protobuf(message_class, message_data)<EOL>message_bytes = message.SerializeToString()<EOL>message_size = len(message_bytes)<EOL>data.append(int32.pack(message_id))<EOL>data.append(int32.pack(message_size))<EOL>data.append(message_bytes)<EOL><DEDENT>data.append(int32.pack(MSG_EOF))<EOL>return b'<STR_LIT>'.join(data)<EOL>

Serializes a dictionary into Manifest data.
:param obj: A dictionary to serialize.
:return: A file object.


for i in range(<NUM_LIT:0>, len(l), n):<EOL><INDENT>yield l[i:i + n]<EOL><DEDENT>


@classmethod<EOL><INDENT>def list_dataset_uris(cls, base_uri, config_path):<DEDENT>

uri_list = []<EOL>parse_result = generous_parse_uri(base_uri)<EOL>bucket_name = parse_result.netloc<EOL>bucket = boto3.resource('<STR_LIT>').Bucket(bucket_name)<EOL>for obj in bucket.objects.filter(Prefix='<STR_LIT>').all():<EOL><INDENT>uuid = obj.key.split('<STR_LIT:->', <NUM_LIT:1>)[<NUM_LIT:1>]<EOL>uri = cls.generate_uri(None, uuid, base_uri)<EOL>storage_broker = cls(uri, config_path)<EOL>if storage_broker.has_admin_metadata():<EOL><INDENT>uri_list.append(uri)<EOL><DEDENT><DEDENT>return uri_list<EOL>

Return list containing URIs with base URI.


try:<EOL><INDENT>self.get_admin_metadata()<EOL>return True<EOL><DEDENT>except ClientError:<EOL><INDENT>return False<EOL><DEDENT>

Return True if the administrative metadata exists.

        This is the definition of being a "dataset".


admin_metadata = self.get_admin_metadata()<EOL>uuid = admin_metadata["<STR_LIT>"]<EOL>dataset_cache_abspath = os.path.join(self._s3_cache_abspath, uuid)<EOL>mkdir_parents(dataset_cache_abspath)<EOL>bucket_fpath = self.data_key_prefix + identifier<EOL>obj = self.s3resource.Object(self.bucket, bucket_fpath)<EOL>relpath = obj.get()['<STR_LIT>']['<STR_LIT>']<EOL>_, ext = os.path.splitext(relpath)<EOL>local_item_abspath = os.path.join(<EOL>dataset_cache_abspath,<EOL>identifier + ext<EOL>)<EOL>if not os.path.isfile(local_item_abspath):<EOL><INDENT>tmp_local_item_abspath = local_item_abspath + "<STR_LIT>"<EOL>self.s3resource.Bucket(self.bucket).download_file(<EOL>bucket_fpath,<EOL>tmp_local_item_abspath<EOL>)<EOL>os.rename(tmp_local_item_abspath, local_item_abspath)<EOL><DEDENT>return local_item_abspath<EOL>

Return absolute path at which item content can be accessed.

        :param identifier: item identifier
        :returns: absolute path from which the item content can be accessed


bucket = self.s3resource.Bucket(self.bucket)<EOL>overlay_names = []<EOL>for obj in bucket.objects.filter(<EOL>Prefix=self.overlays_key_prefix<EOL>).all():<EOL><INDENT>overlay_file = obj.key.rsplit('<STR_LIT:/>', <NUM_LIT:1>)[-<NUM_LIT:1>]<EOL>overlay_name, ext = overlay_file.split('<STR_LIT:.>')<EOL>overlay_names.append(overlay_name)<EOL><DEDENT>return overlay_names<EOL>


def add_item_metadata(self, handle, key, value):

identifier = generate_identifier(handle)<EOL>suffix = '<STR_LIT>'.format(identifier, key)<EOL>bucket_fpath = self.fragments_key_prefix + suffix<EOL>self.s3resource.Object(self.bucket, bucket_fpath).put(<EOL>Body=json.dumps(value)<EOL>)<EOL>

Store the given key:value pair for the item associated with handle.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :param key: metadata key
        :param value: metadata value


bucket = self.s3resource.Bucket(self.bucket)<EOL>for obj in bucket.objects.filter(Prefix=self.data_key_prefix).all():<EOL><INDENT>relpath = obj.get()['<STR_LIT>']['<STR_LIT>']<EOL>yield relpath<EOL><DEDENT>


bucket = self.s3resource.Bucket(self.bucket)<EOL>metadata = {}<EOL>identifier = generate_identifier(handle)<EOL>prefix = self.fragments_key_prefix + '<STR_LIT:{}>'.format(identifier)<EOL>for obj in bucket.objects.filter(Prefix=prefix).all():<EOL><INDENT>metadata_key = obj.key.split('<STR_LIT:.>')[-<NUM_LIT:2>]<EOL>response = obj.get()<EOL>value_as_string = response['<STR_LIT>'].read().decode('<STR_LIT:utf-8>')<EOL>value = json.loads(value_as_string)<EOL>metadata[metadata_key] = value<EOL><DEDENT>return metadata<EOL>

Return dictionary containing all metadata associated with handle.

        In other words all the metadata added using the ``add_item_metadata``
        method.

        :param handle: handle for accessing an item before the dataset is
                       frozen
        :returns: dictionary containing item metadata


def ch(option, parsed_option):<EOL><INDENT>return lambda self: self.check_valid_option(option, parsed_option)<EOL><DEDENT>for i, items in enumerate(options):<EOL><INDENT>prefix_tmp = "<STR_LIT>" % (items[<NUM_LIT:0>], i)<EOL>setattr(TestOptions, "<STR_LIT>" % prefix_tmp, ch(items[<NUM_LIT:1>], items[<NUM_LIT:2>]))<EOL><DEDENT>

Loop over list of options and dynamically create tests


def ch(pubkey, bits, fingerprint_md5, fingerprint_sha256, options, comment, **kwargs):  <EOL><INDENT>return lambda self: self.check_key(pubkey, bits, fingerprint_md5, fingerprint_sha256, options, comment, **kwargs)<EOL><DEDENT>for items in keyset:<EOL><INDENT>modes = items.pop()<EOL>prefix_tmp = "<STR_LIT>" % (prefix, items.pop())<EOL>for mode in modes:<EOL><INDENT>if mode == "<STR_LIT:strict>":<EOL><INDENT>kwargs = {"<STR_LIT:strict>": True}<EOL><DEDENT>else:<EOL><INDENT>kwargs = {"<STR_LIT:strict>": False}<EOL><DEDENT>if len(items) == <NUM_LIT:4>:<EOL><INDENT>pubkey, bits, fingerprint_md5, fingerprint_sha256 = items<EOL>options = comment = None<EOL><DEDENT>else:<EOL><INDENT>pubkey, bits, fingerprint_md5, fingerprint_sha256, options, comment = items<EOL><DEDENT>setattr(<EOL>TestKeys, "<STR_LIT>" % (prefix_tmp, mode),<EOL>ch(pubkey, bits, fingerprint_md5, fingerprint_sha256, options, comment, **kwargs)<EOL>)<EOL><DEDENT><DEDENT>

Loop over list of valid keys and dynamically create tests


def ch(pubkey, expected_error, **kwargs):<EOL><INDENT>return lambda self: self.check_fail(pubkey, expected_error, **kwargs)<EOL><DEDENT>for items in keyset:<EOL><INDENT>modes = items.pop()<EOL>prefix_tmp = "<STR_LIT>" % (prefix, items.pop())<EOL>for mode in modes:<EOL><INDENT>if mode == "<STR_LIT:strict>":<EOL><INDENT>kwargs = {"<STR_LIT:strict>": True}<EOL><DEDENT>else:<EOL><INDENT>kwargs = {"<STR_LIT:strict>": False}<EOL><DEDENT>pubkey, expected_error = items<EOL>setattr(TestKeys, "<STR_LIT>" % (prefix_tmp, mode), ch(pubkey, expected_error, **kwargs))<EOL><DEDENT><DEDENT>

Loop over list of invalid keys and dynamically create tests


def check_key(self, pubkey, bits, fingerprint_md5, fingerprint_sha256, options, comment, **kwargs):

ssh = SSHKey(pubkey, **kwargs)<EOL>ssh.parse()<EOL>self.assertEqual(ssh.bits, bits)<EOL>self.assertEqual(ssh.hash_md5(), fingerprint_md5)<EOL>self.assertEqual(ssh.options_raw, options)<EOL>self.assertEqual(ssh.comment, comment)<EOL>if fingerprint_sha256 is not None:<EOL><INDENT>self.assertEqual(ssh.hash_sha256(), fingerprint_sha256)<EOL><DEDENT>


def check_fail(self, pubkey, expected_error, **kwargs):

<EOL>ssh_key = SSHKey(pubkey, **kwargs)<EOL>self.assertRaises(expected_error, ssh_key.parse)<EOL>

Checks that key check raises specified exception


for field in self.FIELDS:<EOL><INDENT>setattr(self, field, None)<EOL><DEDENT>


warnings.warn("<STR_LIT>")<EOL>return self.hash_md5().replace(b"<STR_LIT>", b"<STR_LIT>")<EOL>

Calculate md5 fingerprint.

        Deprecated, use .hash_md5() instead.


fp_plain = hashlib.md5(self._decoded_key).hexdigest()<EOL>return "<STR_LIT>" + '<STR_LIT::>'.join(a + b for a, b in zip(fp_plain[::<NUM_LIT:2>], fp_plain[<NUM_LIT:1>::<NUM_LIT:2>]))<EOL>

Calculate md5 fingerprint.

        Shamelessly copied from http://stackoverflow.com/questions/6682815/deriving-an-ssh-fingerprint-from-a-public-key-in-python

        For specification, see RFC4716, section 4.


fp_plain = hashlib.sha256(self._decoded_key).digest()<EOL>return (b"<STR_LIT>" + base64.b64encode(fp_plain).replace(b"<STR_LIT:=>", b"<STR_LIT>")).decode("<STR_LIT:utf-8>")<EOL>


fp_plain = hashlib.sha512(self._decoded_key).digest()<EOL>return (b"<STR_LIT>" + base64.b64encode(fp_plain).replace(b"<STR_LIT:=>", b"<STR_LIT>")).decode("<STR_LIT:utf-8>")<EOL>


def _unpack_by_int(self, data, current_position):

<EOL>try:<EOL><INDENT>requested_data_length = struct.unpack('<STR_LIT>', data[current_position:current_position + self.INT_LEN])[<NUM_LIT:0>]<EOL><DEDENT>except struct.error:<EOL><INDENT>raise MalformedDataError("<STR_LIT>" % self.INT_LEN)<EOL><DEDENT>current_position += self.INT_LEN<EOL>remaining_data_length = len(data[current_position:])<EOL>if remaining_data_length < requested_data_length:<EOL><INDENT>raise MalformedDataError(<EOL>"<STR_LIT>" % (requested_data_length, remaining_data_length)<EOL>)<EOL><DEDENT>next_data = data[current_position:current_position + requested_data_length]<EOL>current_position += requested_data_length<EOL>return current_position, next_data<EOL>

Returns a tuple with (location of next data field, contents of requested data field).


if sys.version < '<STR_LIT:3>':<EOL><INDENT>ret = long(<NUM_LIT:0>)  <EOL>for byte in data:<EOL><INDENT>ret = (ret << <NUM_LIT:8>) + ord(byte)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ret = <NUM_LIT:0><EOL>for byte in data:<EOL><INDENT>ret = (ret << <NUM_LIT:8>) + byte<EOL><DEDENT><DEDENT>return ret<EOL>


try:<EOL><INDENT>decoded_key = base64.b64decode(pubkey_content.encode("<STR_LIT:ascii>"))<EOL><DEDENT>except (TypeError, binascii.Error):<EOL><INDENT>raise MalformedDataError("<STR_LIT>")<EOL><DEDENT>return decoded_key<EOL>


quote_open = False<EOL>parsed_options = {}<EOL>def parse_add_single_option(opt):<EOL><INDENT>"""<STR_LIT>"""<EOL>if "<STR_LIT:=>" in opt:<EOL><INDENT>opt_name, opt_value = opt.split("<STR_LIT:=>", <NUM_LIT:1>)<EOL>opt_value = opt_value.replace('<STR_LIT:">', '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>opt_name = opt<EOL>opt_value = True<EOL><DEDENT>if "<STR_LIT:U+0020>" in opt_name or not self.OPTION_NAME_RE.match(opt_name):<EOL><INDENT>raise InvalidOptionNameError("<STR_LIT>" % opt_name)<EOL><DEDENT>if self.strict_mode:<EOL><INDENT>for valid_opt_name, value_required in self.OPTIONS_SPEC:<EOL><INDENT>if opt_name.lower() == valid_opt_name:<EOL><INDENT>if value_required and opt_value is True:<EOL><INDENT>raise MissingMandatoryOptionValueError("<STR_LIT>" % opt_name)<EOL><DEDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise UnknownOptionNameError("<STR_LIT>" % opt_name)<EOL><DEDENT><DEDENT>if opt_name not in parsed_options:<EOL><INDENT>parsed_options[opt_name] = []<EOL><DEDENT>parsed_options[opt_name].append(opt_value)<EOL><DEDENT>start_of_current_opt = <NUM_LIT:0><EOL>i = <NUM_LIT:1>  <EOL>for i, character in enumerate(options):<EOL><INDENT>if character == '<STR_LIT:">':  <EOL><INDENT>quote_open = not quote_open<EOL><DEDENT>if quote_open:<EOL><INDENT>continue<EOL><DEDENT>if character == "<STR_LIT:U+002C>":<EOL><INDENT>opt = options[start_of_current_opt:i]<EOL>parse_add_single_option(opt)<EOL>start_of_current_opt = i + <NUM_LIT:1><EOL><DEDENT><DEDENT>if start_of_current_opt + <NUM_LIT:1> != i:<EOL><INDENT>opt = options[start_of_current_opt:]<EOL>parse_add_single_option(opt)<EOL><DEDENT>if quote_open:<EOL><INDENT>raise InvalidOptionsError("<STR_LIT>")<EOL><DEDENT>return parsed_options<EOL>


current_position, raw_e = self._unpack_by_int(data, <NUM_LIT:0>)<EOL>current_position, raw_n = self._unpack_by_int(data, current_position)<EOL>unpacked_e = self._parse_long(raw_e)<EOL>unpacked_n = self._parse_long(raw_n)<EOL>self.rsa = RSAPublicNumbers(unpacked_e, unpacked_n).public_key(default_backend())<EOL>self.bits = self.rsa.key_size<EOL>if self.strict_mode:<EOL><INDENT>min_length = self.RSA_MIN_LENGTH_STRICT<EOL>max_length = self.RSA_MAX_LENGTH_STRICT<EOL><DEDENT>else:<EOL><INDENT>min_length = self.RSA_MIN_LENGTH_LOOSE<EOL>max_length = self.RSA_MAX_LENGTH_LOOSE<EOL><DEDENT>if self.bits < min_length:<EOL><INDENT>raise TooShortKeyError(<EOL>"<STR_LIT>" % (self.key_type, min_length, self.bits)<EOL>)<EOL><DEDENT>if self.bits > max_length:<EOL><INDENT>raise TooLongKeyError(<EOL>"<STR_LIT>" % (self.key_type, max_length, self.bits)<EOL>)<EOL><DEDENT>return current_position<EOL>


data_fields = {}<EOL>current_position = <NUM_LIT:0><EOL>for item in ("<STR_LIT:p>", "<STR_LIT:q>", "<STR_LIT:g>", "<STR_LIT:y>"):<EOL><INDENT>current_position, value = self._unpack_by_int(data, current_position)<EOL>data_fields[item] = self._parse_long(value)<EOL><DEDENT>q_bits = self._bits_in_number(data_fields["<STR_LIT:q>"])<EOL>p_bits = self._bits_in_number(data_fields["<STR_LIT:p>"])<EOL>if q_bits != self.DSA_N_LENGTH:<EOL><INDENT>raise InvalidKeyError("<STR_LIT>" % (self.bits, q_bits))<EOL><DEDENT>if self.strict_mode:<EOL><INDENT>min_length = self.DSA_MIN_LENGTH_STRICT<EOL>max_length = self.DSA_MAX_LENGTH_STRICT<EOL><DEDENT>else:<EOL><INDENT>min_length = self.DSA_MIN_LENGTH_LOOSE<EOL>max_length = self.DSA_MAX_LENGTH_LOOSE<EOL><DEDENT>if p_bits < min_length:<EOL><INDENT>raise TooShortKeyError("<STR_LIT>" % (self.key_type, min_length, p_bits))<EOL><DEDENT>if p_bits > max_length:<EOL><INDENT>raise TooLongKeyError(<EOL>"<STR_LIT>" % (self.key_type, max_length, p_bits)<EOL>)<EOL><DEDENT>dsa_parameters = DSAParameterNumbers(data_fields["<STR_LIT:p>"], data_fields["<STR_LIT:q>"], data_fields["<STR_LIT:g>"])<EOL>self.dsa = DSAPublicNumbers(data_fields["<STR_LIT:y>"], dsa_parameters).public_key(default_backend())<EOL>self.bits = self.dsa.key_size<EOL>return current_position<EOL>


current_position, curve_information = self._unpack_by_int(data, <NUM_LIT:0>)<EOL>if curve_information not in self.ECDSA_CURVE_DATA:<EOL><INDENT>raise NotImplementedError("<STR_LIT>" % curve_information)<EOL><DEDENT>curve, hash_algorithm = self.ECDSA_CURVE_DATA[curve_information]<EOL>current_position, key_data = self._unpack_by_int(data, current_position)<EOL>try:<EOL><INDENT>ecdsa_key = ecdsa.VerifyingKey.from_string(key_data[<NUM_LIT:1>:], curve, hash_algorithm)<EOL><DEDENT>except AssertionError:<EOL><INDENT>raise InvalidKeyError("<STR_LIT>")<EOL><DEDENT>self.bits = int(curve_information.replace(b"<STR_LIT>", b"<STR_LIT>"))<EOL>self.ecdsa = ecdsa_key<EOL>return current_position<EOL>


current_position, verifying_key = self._unpack_by_int(data, <NUM_LIT:0>)<EOL>verifying_key_length = len(verifying_key) * <NUM_LIT:8><EOL>verifying_key = self._parse_long(verifying_key)<EOL>if verifying_key < <NUM_LIT:0>:<EOL><INDENT>raise InvalidKeyError("<STR_LIT>")<EOL><DEDENT>self.bits = verifying_key_length<EOL>if self.bits != <NUM_LIT>:<EOL><INDENT>raise InvalidKeyLengthError("<STR_LIT>" % self.bits)<EOL><DEDENT>return current_position<EOL>

Parses ed25516 keys.

        There is no (apparent) way to validate ed25519 keys. This only
        checks data length (256 bits), but does not try to validate
        the key in any way.


if keydata is None:<EOL><INDENT>if self.keydata is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>keydata = self.keydata<EOL><DEDENT>else:<EOL><INDENT>self.reset()<EOL>self.keydata = keydata<EOL><DEDENT>if keydata.startswith("<STR_LIT>"):<EOL><INDENT>key_type = None  <EOL>pubkey_content = "<STR_LIT>".join([line for line in keydata.split("<STR_LIT:\n>") if "<STR_LIT::>" not in line and "<STR_LIT>" not in line])<EOL><DEDENT>else:<EOL><INDENT>key_parts = self._split_key(keydata)<EOL>key_type = key_parts[<NUM_LIT:0>]<EOL>pubkey_content = key_parts[<NUM_LIT:1>]<EOL><DEDENT>self._decoded_key = self.decode_key(pubkey_content)<EOL>current_position, unpacked_key_type = self._unpack_by_int(self._decoded_key, <NUM_LIT:0>)<EOL>if key_type is not None and key_type != unpacked_key_type.decode():<EOL><INDENT>raise InvalidTypeError("<STR_LIT>" % (key_type, unpacked_key_type))<EOL><DEDENT>self.key_type = unpacked_key_type<EOL>key_data_length = self._process_key(self._decoded_key[current_position:])<EOL>current_position = current_position + key_data_length<EOL>if current_position != len(self._decoded_key):<EOL><INDENT>raise MalformedDataError("<STR_LIT>" % (len(self._decoded_key) - current_position))<EOL><DEDENT>if self.disallow_options and self.options:<EOL><INDENT>raise InvalidOptionsError("<STR_LIT>")<EOL><DEDENT>

Validates SSH public key.

        Throws exception for invalid keys. Otherwise returns None.

        Populates key_type, bits and bits fields.

        For rsa keys, see field "rsa" for raw public key data.
        For dsa keys, see field "dsa".
        For ecdsa keys, see field "ecdsa".


if isinstance(exc, exceptions.APIException):<EOL><INDENT>headers = {}<EOL>if getattr(exc, '<STR_LIT>', None):<EOL><INDENT>headers['<STR_LIT>'] = exc.auth_header<EOL><DEDENT>if getattr(exc, '<STR_LIT>', None):<EOL><INDENT>headers['<STR_LIT>'] = '<STR_LIT>' % exc.wait<EOL><DEDENT>return Response({'<STR_LIT:error>': exc.detail},<EOL>status=exc.status_code,<EOL>headers=headers)<EOL><DEDENT>elif isinstance(exc, Http404):<EOL><INDENT>return Response({'<STR_LIT:error>': '<STR_LIT>'},<EOL>status=status.HTTP_404_NOT_FOUND)<EOL><DEDENT>elif isinstance(exc, PermissionDenied):<EOL><INDENT>return Response({'<STR_LIT:error>': '<STR_LIT>'},<EOL>status=status.HTTP_403_FORBIDDEN)<EOL><DEDENT>return None<EOL>

Returns the response that should be used for any given exception.

By default we handle the REST framework `APIException`, and also
Django's builtin `Http404` and `PermissionDenied` exceptions.

Any unhandled exceptions may return `None`, which will cause a 500 error
to be raised.


def write_output(filepath, pack, filename, content):

if (filepath and filepath != '<STR_LIT:.>'<EOL>and not os.path.exists(os.path.join(filepath, pack))):<EOL><INDENT>os.makedirs(os.path.join(filepath, pack))<EOL><DEDENT>destination = os.path.join(filepath, pack, filename)<EOL>with io.open(destination, '<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>f.write(content)<EOL><DEDENT>return destination<EOL>

Write content to filepath+pack+filename, create filepath if it does not
allready exists.

This an helper to automatically rewrite HTML outputs when needed during
development.

Beware, using this will lose every templating instructions previously
written in output attempts.


context_kwargs = {<EOL>'<STR_LIT>': DJANGO110_COMPAT,<EOL>}<EOL>context_kwargs.update(**kwargs)<EOL>context = Context(context_kwargs)<EOL>with io.open(path, '<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>template = Template(f.read())<EOL><DEDENT>return template.render(context)<EOL>

Return compiled template from given path with given context

It's a little hack to be able to use Django template stuff to conditionnate
some HTML differences on Django versions.

Template is readed as a simple file without using Django template engine
loader select.

Template context will contains some variables about Django versions
compatibilities. Actually this is only about the 'required' input attribute
behavior that is different from 1.9 (``required=""``) to
1.10 (``required``).


context_kwargs = {<EOL>"<STR_LIT>": form,<EOL>}<EOL>context_kwargs.update(**kwargs)<EOL>context = Context(context_kwargs)<EOL>tpl = """<STR_LIT>"""<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>tpl += """<STR_LIT>"""<EOL><DEDENT>else:<EOL><INDENT>tpl += """<STR_LIT>"""<EOL><DEDENT>tpl += """<STR_LIT>"""<EOL>template = Template(tpl)<EOL>return template.render(context)<EOL>

Return compiled template with given context where only 'form' is required.
If crispy helper is given, it must be given as 'helper' named argument.

Template is different depending helper is given or not in kwargs.


kwargs = super(FormContainersMixin, self).get_form_kwargs()<EOL>kwargs.update({<EOL>'<STR_LIT>': "<STR_LIT>".format(self.kwargs.get('<STR_LIT>'))<EOL>})<EOL>return kwargs<EOL>


token = token.split_contents()<EOL>field = token.pop(<NUM_LIT:1>)<EOL>attrs = {}<EOL>token.pop(<NUM_LIT:0>)<EOL>for attribute_name, value in pairwise(token):<EOL><INDENT>attrs[attribute_name] = value<EOL><DEDENT>return CrispyFoundationFieldNode(field, attrs)<EOL>


try:<EOL><INDENT>return pkg_resources.get_distribution(package_name).version<EOL><DEDENT>except pkg_resources.DistributionNotFound:<EOL><INDENT>_conf = read_configuration(os.path.join(PROJECT_DIR, "<STR_LIT>"))<EOL><DEDENT>return _conf["<STR_LIT>"]["<STR_LIT:version>"]<EOL>

Get package version from installed distribution or configuration file if not
installed


def render(self, form, form_style, context, template_pack=TEMPLATE_PACK):

links, content = '<STR_LIT>', '<STR_LIT>'<EOL>if not self.css_id:<EOL><INDENT>self.css_id = "<STR_LIT:->".join(["<STR_LIT>",<EOL>text_type(randint(<NUM_LIT:1000>, <NUM_LIT>))])<EOL><DEDENT>for tab in self.fields:<EOL><INDENT>tab.active = False<EOL><DEDENT>self.open_target_group_for_form(form)<EOL>for tab in self.fields:<EOL><INDENT>content += render_field(<EOL>tab, form, form_style, context, template_pack=template_pack<EOL>)<EOL>links += tab.render_link(form, template_pack)<EOL><DEDENT>context.update({<EOL>'<STR_LIT>': self,<EOL>'<STR_LIT>': links,<EOL>'<STR_LIT:content>': content<EOL>})<EOL>template = self.get_template_name(template_pack)<EOL>return render_to_string(template, context.flatten())<EOL>

Re-implement almost the same code from crispy_forms but passing
``form`` instance to item ``render_link`` method.


return any([fieldname_error for fieldname_error in form.errors.keys()<EOL>if fieldname_error in self])<EOL>


def render_link(self, form, template_pack=TEMPLATE_PACK, **kwargs):

link_template = self.link_template % template_pack<EOL>return render_to_string(link_template,<EOL>{<EOL>'<STR_LIT>': self,<EOL>'<STR_LIT>': self.has_errors(form)<EOL>})<EOL>

Render the link for the tab-pane. It must be called after render so
``css_class`` is updated with ``active`` class name if needed.


def render(self, form, form_style, context, template_pack=TEMPLATE_PACK,<EOL>**kwargs):

content = '<STR_LIT>'<EOL>if not self.css_id:<EOL><INDENT>self.css_id = "<STR_LIT:->".join(["<STR_LIT>",<EOL>text_type(randint(<NUM_LIT:1000>, <NUM_LIT>))])<EOL><DEDENT>self.open_target_group_for_form(form)<EOL>for group in self.fields:<EOL><INDENT>group.data_parent = self.css_id<EOL>group.item_has_errors = any([fieldname_error for fieldname_error in<EOL>form.errors.keys()<EOL>if fieldname_error in group])<EOL>content += render_field(<EOL>group, form, form_style, context, template_pack=template_pack,<EOL>**kwargs<EOL>)<EOL><DEDENT>template = self.get_template_name(template_pack)<EOL>context.update({'<STR_LIT>': self, '<STR_LIT:content>': content})<EOL>return render_to_string(template, context.flatten())<EOL>

Re-implement almost the same code from crispy_forms but using
``form`` instance to catch field errors.


self.ser = serial.Serial(serial_port, baudrate=<NUM_LIT>)<EOL>self.lock = threading.Lock()<EOL>


def exec_command(self, domain, function, operator, value=None):

if operator in CMDS[domain][function]['<STR_LIT>']:<EOL><INDENT>if operator is '<STR_LIT:=>' and value is None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if value is None:<EOL><INDENT>cmd = '<STR_LIT>'.join([CMDS[domain][function]['<STR_LIT>'], operator])<EOL><DEDENT>else:<EOL><INDENT>cmd = '<STR_LIT>'.join(<EOL>[CMDS[domain][function]['<STR_LIT>'], operator, str(value)])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % operator)<EOL><DEDENT>if not self.ser.is_open:<EOL><INDENT>self.ser.open()<EOL><DEDENT>try:<EOL><INDENT>self.lock.acquire()<EOL>self.ser.write('<STR_LIT>'.join(['<STR_LIT:\r>', cmd, '<STR_LIT:\r>']).encode('<STR_LIT:utf-8>'))<EOL>time.sleep(<NUM_LIT:0.1>)<EOL>msg = self.ser.read(self.ser.in_waiting)<EOL>try:<EOL><INDENT>msg = msg.decode()[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>msg = msg.split('<STR_LIT:=>')[<NUM_LIT:1>]<EOL>return msg<EOL><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>self.lock.release()<EOL><DEDENT>

Write a command to the receiver and read the value it returns.

The receiver will always return a value, also when setting a value.


def main_dimmer(self, operator, value=None):

return self.exec_command('<STR_LIT>', '<STR_LIT>', operator, value)<EOL>


def main_mute(self, operator, value=None):


def main_power(self, operator, value=None):


def main_volume(self, operator, value=None):

try:<EOL><INDENT>res = int(self.exec_command('<STR_LIT>', '<STR_LIT>', operator, value))<EOL>return res<EOL><DEDENT>except (ValueError, TypeError):<EOL><INDENT>pass<EOL><DEDENT>return None<EOL>


def main_ir(self, operator, value=None):


def main_listeningmode(self, operator, value=None):


def main_sleep(self, operator, value=None):


def main_source(self, operator, value=None):

try:<EOL><INDENT>source = int(self.exec_command('<STR_LIT>', '<STR_LIT:source>', operator, value))<EOL>return source<EOL><DEDENT>except (ValueError, TypeError):<EOL><INDENT>pass<EOL><DEDENT>return None<EOL>


def main_version(self, operator, value=None):

return self.exec_command('<STR_LIT>', '<STR_LIT:version>', operator, value)<EOL>


def tuner_am_frequency(self, operator, value=None):


def tuner_am_preset(self, operator, value=None):


def tuner_band(self, operator, value=None):


def tuner_fm_frequency(self, operator, value=None):


def tuner_fm_mute(self, operator, value=None):


def tuner_fm_preset(self, operator, value=None):


self._host = host<EOL>


def _send(self, message, read_reply=False):

sock = None<EOL>for tries in range(<NUM_LIT:0>, <NUM_LIT:3>):<EOL><INDENT>try:<EOL><INDENT>sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)<EOL>sock.connect((self._host, self.PORT))<EOL>break<EOL><DEDENT>except (ConnectionError, BrokenPipeError):<EOL><INDENT>if tries == <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>sock.send(codecs.decode(message, '<STR_LIT>'))<EOL>if read_reply:<EOL><INDENT>sleep(<NUM_LIT:0.1>)<EOL>reply = '<STR_LIT>'<EOL>tries = <NUM_LIT:0><EOL>max_tries = <NUM_LIT:20><EOL>while len(reply) < len(message) and tries < max_tries:<EOL><INDENT>try:<EOL><INDENT>reply += codecs.encode(sock.recv(self.BUFFERSIZE), '<STR_LIT>').decode("<STR_LIT:utf-8>")<EOL><DEDENT>except (ConnectionError, BrokenPipeError):<EOL><INDENT>pass<EOL><DEDENT>tries += <NUM_LIT:1><EOL><DEDENT>sock.close()<EOL>if tries >= max_tries:<EOL><INDENT>return<EOL><DEDENT>return reply<EOL><DEDENT>sock.close()<EOL>


nad_reply = self._send(self.POLL_VOLUME +<EOL>self.POLL_POWER +<EOL>self.POLL_MUTED +<EOL>self.POLL_SOURCE, read_reply=True)<EOL>if nad_reply is None:<EOL><INDENT>return<EOL><DEDENT>num_chars = <NUM_LIT:10><EOL>nad_status = [nad_reply[i:i + num_chars]<EOL>for i in range(<NUM_LIT:0>, len(nad_reply), num_chars)]<EOL>return {'<STR_LIT>': int(nad_status[<NUM_LIT:0>][-<NUM_LIT:2>:], <NUM_LIT:16>),<EOL>'<STR_LIT>': nad_status[<NUM_LIT:1>][-<NUM_LIT:2>:] == '<STR_LIT>',<EOL>'<STR_LIT>': nad_status[<NUM_LIT:2>][-<NUM_LIT:2>:] == '<STR_LIT>',<EOL>'<STR_LIT:source>': self.SOURCES_REVERSED[nad_status[<NUM_LIT:3>][-<NUM_LIT:2>:]]}<EOL>

Return the status of the device.

Returns a dictionary with keys 'volume' (int 0-200) , 'power' (bool),
 'muted' (bool) and 'source' (str).


status = self.status()<EOL>if status['<STR_LIT>']:  <EOL><INDENT>self._send(self.CMD_POWERSAVE + self.CMD_OFF)<EOL><DEDENT>


status = self.status()<EOL>if not status['<STR_LIT>']:<EOL><INDENT>self._send(self.CMD_ON, read_reply=True)<EOL>sleep(<NUM_LIT:0.5>)<EOL><DEDENT>


if <NUM_LIT:0> <= volume <= <NUM_LIT:200>:<EOL><INDENT>volume = format(volume, "<STR_LIT>")  <EOL>self._send(self.CMD_VOLUME + volume)<EOL><DEDENT>

Set volume level of the device. Accepts integer values 0-200.


self._send(self.CMD_MUTE, read_reply=True)<EOL>


status = self.status()<EOL>if status['<STR_LIT>']:  <EOL><INDENT>if status['<STR_LIT:source>'] != source:  <EOL><INDENT>if source in self.SOURCES:<EOL><INDENT>self._send(self.CMD_SOURCE + self.SOURCES[source], read_reply=True)<EOL><DEDENT><DEDENT><DEDENT>

Select a source from the list of sources.


return list(self.SOURCES.keys())<EOL>


if self.telnet:<EOL><INDENT>self.telnet.close()<EOL><DEDENT>


def __init__(self, host, port=<NUM_LIT>, timeout=DEFAULT_TIMEOUT):

self.telnet = None<EOL>self.host = host<EOL>self.port = port<EOL>self.timeout = timeout<EOL>


def exec_command(self, domain, function, operator, value=None):

if operator in CMDS[domain][function]['<STR_LIT>']:<EOL><INDENT>if operator is '<STR_LIT:=>' and value is None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if value is None:<EOL><INDENT>cmd = '<STR_LIT>'.join([CMDS[domain][function]['<STR_LIT>'], operator])<EOL><DEDENT>else:<EOL><INDENT>cmd = '<STR_LIT>'.join(<EOL>[CMDS[domain][function]['<STR_LIT>'], operator, str(value)])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % operator)<EOL><DEDENT>if self._open_connection():<EOL><INDENT>self.telnet.write(('<STR_LIT>'.join(['<STR_LIT:\r>', cmd, '<STR_LIT:\n>']).encode()))<EOL>loop = <NUM_LIT:3><EOL>while loop:<EOL><INDENT>msg = self.telnet.read_until('<STR_LIT:\n>'.encode(), self.timeout)<EOL>if msg == "<STR_LIT>":<EOL><INDENT>loop -= <NUM_LIT:1><EOL>continue<EOL><DEDENT>msg = msg.decode().strip('<STR_LIT:\r\n>')<EOL>if msg.strip().split('<STR_LIT:=>')[<NUM_LIT:0>].lower() == '<STR_LIT:.>'.join([domain, function]).lower():<EOL><INDENT>return msg.strip().split('<STR_LIT:=>')[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>raise RuntimeError('<STR_LIT>')<EOL>

Write a command to the receiver and read the value it returns.


ver = '<STR_LIT:.>'.join(map(str, VERSION[:<NUM_LIT:3>]))<EOL>if not VERSION[<NUM_LIT:3>]:<EOL><INDENT>return ver<EOL><DEDENT>hyphen = '<STR_LIT>'<EOL>suffix = hyphen.join(map(str, VERSION[-<NUM_LIT:2>:]))<EOL>if VERSION[<NUM_LIT:3>] in [VERSION_SUFFIX_DEV, VERSION_SUFFIX_POST]:<EOL><INDENT>hyphen = '<STR_LIT:.>'<EOL><DEDENT>ver = hyphen.join([ver, suffix])<EOL>return ver<EOL>

获取打包使用的版本号，符合 PYPI 官方推荐的版本号方案

:return: PYPI 打包版本号
:rtype: str


directory = os.path.dirname(os.path.abspath(__file__))<EOL>version_path = os.path.join(directory, '<STR_LIT>')<EOL>if os.path.exists(version_path):<EOL><INDENT>with open(version_path) as f:<EOL><INDENT>ver = f.read()<EOL><DEDENT>return ver<EOL><DEDENT>return get_setup_version()<EOL>

获取终端命令版本号，若存在VERSION文件则使用其中的版本号，
否则使用 :meth:`.get_setup_version`

:return: 终端命令版本号
:rtype: str


hand[func.__name__] = func<EOL>return func<EOL>

将被装饰函数注册到 hand 单例字典中

:param object func: 被装饰函数
:return: 被装饰的函数本身（不进行封装）
:rtype: function


invoked = bool(len(dargs) == <NUM_LIT:1> and not dkwargs and callable(dargs[<NUM_LIT:0>]))<EOL>if invoked:<EOL><INDENT>func = dargs[<NUM_LIT:0>]<EOL><DEDENT>def wrapper(func):<EOL><INDENT>@hand._click.command(<EOL>name=func.__name__.lower(),<EOL>help=func.__doc__)<EOL>def _wrapper(*args, **kwargs):<EOL><INDENT>log_level = dkwargs.pop('<STR_LIT>', logging.INFO)<EOL>log.setLevel(log_level)<EOL>log.debug("<STR_LIT>".format(dargs, dkwargs))<EOL>log.debug("<STR_LIT>".format(args, kwargs))<EOL>func(*args, **kwargs)<EOL><DEDENT>return _wrapper<EOL><DEDENT>return wrapper if not invoked else wrapper(func)<EOL>

将被装饰函数封装为一个 :class:`click.core.Command` 类，
此装饰器并不提供额外的复杂功能，仅提供将被装饰方法注册为一个 ``mohand`` 子命令的功能

该装饰器作为一个一般装饰器使用（如： ``@hand.general`` ）

.. note::

    该装饰器会在插件系统加载外部插件前辈注册到 :data:`.hands.hand` 中。

    此处的 ``general`` 装饰器同时兼容有参和无参调用方式

:param int log_level: 当前子命令的日志输出等级，默认为： ``logging.INFO``
:return: 被封装后的函数
:rtype: function


<EOL>import mohand.decorator  <EOL>mgr = stevedore.ExtensionManager(<EOL>namespace=env.plugin_namespace,<EOL>invoke_on_load=True)<EOL>def register_hand(ext):<EOL><INDENT>_hand = ext.obj.register()<EOL>if hasattr(hand, _hand.__name__):<EOL><INDENT>raise HandDuplicationOfNameError(_hand.__name__)<EOL><DEDENT>hand[_hand.__name__] = _hand<EOL>_pkg, _ver = ext.obj.version()<EOL>env.version[_pkg] = _ver<EOL><DEDENT>try:<EOL><INDENT>mgr.map(register_hand)<EOL><DEDENT>except stevedore.exception.NoMatches:<EOL><INDENT>pass<EOL><DEDENT>return hand<EOL>

加载hand扩展插件

:return: 返回hand注册字典(单例)
:rtype: HandDict


@abc.abstractmethod<EOL><INDENT>def register(self, *args, **kwargs):<DEDENT>

注册hand任务

:return: 返回待注册的hand函数
:rtype: function


版本信息

:return: 返回包名，版本号元组
:rtype: (str, str)


@click.group()<EOL>@click.option(<EOL>'<STR_LIT>', is_flag=True, callback=print_author,<EOL>expose_value=False, is_eager=True, help='<STR_LIT>')<EOL>@click.option(<EOL>'<STR_LIT>', is_flag=True, callback=print_version,<EOL>expose_value=False, is_eager=True, help='<STR_LIT>')<EOL>def cli(*args, **kwargs):

log.debug('<STR_LIT>'.format(args, kwargs))<EOL>env.update(kwargs)<EOL>

通用自动化处理工具

详情参考 `GitHub <https://github.com/littlemo/mohand>`_


try:<EOL><INDENT>return self[key]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AttributeError(key)<EOL><DEDENT>

重载获取属性方法，使字典支持点语法取值


self[key] = value<EOL>

重载设置属性方法，使字典支持点语法赋值


for name in names:<EOL><INDENT>value = self.get(name)<EOL>if value:<EOL><INDENT>return value<EOL><DEDENT><DEDENT>

返回列表key在字典中第一个不为空的值


class_name, super_cls, dict_ = args<EOL>dict_['<STR_LIT>'] = None<EOL>cls_ = super(Singleton, mcs).__new__(<EOL>mcs, class_name, super_cls, dict_, **kwargs)<EOL>return cls_<EOL>

元类msc通过__new__组建类对象，其中msc指Singleton

:param list args: 可以包含类构建所需要三元素， ``类名`` ， ``父类`` ，
    ``命名空间``, 其中命名空间中 __qualname__ 和函数的 __qualname__
    均含有 classname 做为前缀，在这里，如果想替换类名，需要把以上全部替换才可以。
:param dict kwargs: 可以自定义传递一些参数
:return: 返回类对象,通过super(Singleton, mcs).__new__此时已经组装好了类
:rtype: class


with cls.lock:<EOL><INDENT>cls._instance = cls._instance orsuper(Singleton, cls).__call__(*args, **kwargs)<EOL><DEDENT>return cls._instance<EOL>

在调用时在线程锁中实现单例实例化


def _exists(s):<EOL><INDENT>return os.path.exists(os.path.join(path, s))<EOL><DEDENT>return (<EOL>os.path.isdir(path) and<EOL>(_exists('<STR_LIT>') or _exists('<STR_LIT>'))<EOL>)<EOL>

判断传入的路径是否为一个 Python 模块包

:param str path: 待判断的路径
:return: 返回是，则传入 path 为一个 Python 包，否则不是
:rtype: bool


<EOL>names = names or [env.handfile]<EOL>if not names[<NUM_LIT:0>].endswith('<STR_LIT>'):<EOL><INDENT>names += [names[<NUM_LIT:0>] + '<STR_LIT>']<EOL><DEDENT>if os.path.dirname(names[<NUM_LIT:0>]):<EOL><INDENT>for name in names:<EOL><INDENT>expanded = os.path.expanduser(name)<EOL>if os.path.exists(expanded):<EOL><INDENT>if name.endswith('<STR_LIT>') or _is_package(expanded):<EOL><INDENT>return os.path.abspath(expanded)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>path = '<STR_LIT:.>'<EOL>while os.path.split(os.path.abspath(path))[<NUM_LIT:1>]:<EOL><INDENT>for name in names:<EOL><INDENT>joined = os.path.join(path, name)<EOL>if os.path.exists(joined):<EOL><INDENT>if name.endswith('<STR_LIT>') or _is_package(joined):<EOL><INDENT>return os.path.abspath(joined)<EOL><DEDENT><DEDENT><DEDENT>path = os.path.join('<STR_LIT:..>', path)<EOL><DEDENT><DEDENT>return None<EOL>

尝试定位 ``handfile`` 文件，明确指定或逐级搜索父路径

:param str names: 可选，待查找的文件名，主要用于调试，默认使用终端传入的配置
:return: ``handfile`` 文件所在的绝对路径，默认为 None
:rtype: str


<EOL>imported_vars = vars(imported)<EOL>if "<STR_LIT>" in imported_vars:<EOL><INDENT>imported_vars = [<EOL>(name, imported_vars[name]) for name in<EOL>imported_vars if name in imported_vars["<STR_LIT>"]]<EOL><DEDENT>else:<EOL><INDENT>imported_vars = imported_vars.items()<EOL><DEDENT>cmd_dict = extract_commands(imported_vars)<EOL>return imported.__doc__, cmd_dict<EOL>

从传入的 ``imported`` 中获取所有 ``click.core.Command``

:param module imported: 导入的Python包
:return: 包描述文档，仅含终端命令函数的对象字典
:rtype: (str, dict(str, object))


return isinstance(obj, click.core.Command)<EOL>

验证传入的 obj 是否为一个 CLI 命令对象

:param object obj: 待判断对象
:return: 是否为 ``click.core.Command`` 命令对象
:rtype: bool


commands = dict()<EOL>for tup in imported_vars:<EOL><INDENT>name, obj = tup<EOL>if is_command_object(obj):<EOL><INDENT>commands.setdefault(name, obj)<EOL><DEDENT><DEDENT>return commands<EOL>

从传入的变量列表中提取命令( ``click.core.Command`` )对象

:param dict_items imported_vars: 字典的键值条目列表
:return: 判定为终端命令的对象字典
:rtype: dict(str, object)


if importer is None:<EOL><INDENT>importer = __import__<EOL><DEDENT>directory, handfile = os.path.split(path)<EOL>added_to_path = False<EOL>index = None<EOL>if directory not in sys.path:<EOL><INDENT>sys.path.insert(<NUM_LIT:0>, directory)<EOL>added_to_path = True<EOL><DEDENT>else:<EOL><INDENT>i = sys.path.index(directory)<EOL>if i != <NUM_LIT:0>:<EOL><INDENT>index = i<EOL>sys.path.insert(<NUM_LIT:0>, directory)<EOL>del sys.path[i + <NUM_LIT:1>]<EOL><DEDENT><DEDENT>sys_byte_code_bak = sys.dont_write_bytecode<EOL>sys.dont_write_bytecode = True<EOL>imported = importer(os.path.splitext(handfile)[<NUM_LIT:0>])<EOL>sys.dont_write_bytecode = sys_byte_code_bak<EOL>if added_to_path:<EOL><INDENT>del sys.path[<NUM_LIT:0>]<EOL><DEDENT>if index is not None:<EOL><INDENT>sys.path.insert(index + <NUM_LIT:1>, directory)<EOL>del sys.path[<NUM_LIT:0>]<EOL><DEDENT>docstring, commands = get_commands_from_module(imported)<EOL>return docstring, commands<EOL>

导入传入的 ``handfile`` 文件路径，并返回(docstring, callables)

也就是 handfile 包的 ``__doc__`` 属性 (字符串) 和一个 ``{'name': callable}``
的字典，包含所有通过 mohand 的 command 测试的 callables

:param str path: 待导入的 handfile 文件路径
:param function importer: 可选，包导入函数，默认为 ``__import__``
:return: 包描述文档，仅含终端命令函数的对象字典
:rtype: (str, dict(str, object))


def __init__(<EOL>self,<EOL>version: str,<EOL>frequency: int,<EOL>timestamp: int,<EOL>seed_value: str,<EOL>previous_output_value: str,<EOL>signature_value: str,<EOL>output_value: str,<EOL>status_code: str,<EOL>):

self._version = version<EOL>self._frequency = frequency<EOL>self._timestamp = timestamp<EOL>self._seed_value = seed_value<EOL>self._previous_output_value = previous_output_value<EOL>self._signature_value = signature_value<EOL>self._output_value = output_value<EOL>self._status_code = status_code<EOL>self._json = json.dumps(<EOL>{<EOL>self._KEY_VERSION: self.version,<EOL>self._KEY_FREQUENCY: self.frequency,<EOL>self._KEY_TIMESTAMP: self.timestamp,<EOL>self._KEY_SEED_VALUE: self.seed_value,<EOL>self._KEY_PREVIOUS_OUTPUT_VALUE: self.previous_output_value,<EOL>self._KEY_SIGNATURE_VALUE: self.signature_value,<EOL>self._KEY_OUTPUT_VALUE: self.output_value,<EOL>self._KEY_STATUS_CODE: self.status_code,<EOL>},<EOL>sort_keys=True,<EOL>)<EOL>self._xml = self._xml_template.format(<EOL>self.version,<EOL>self.frequency,<EOL>self.timestamp,<EOL>self.seed_value,<EOL>self.previous_output_value,<EOL>self.signature_value,<EOL>self.output_value,<EOL>self.status_code,<EOL>)<EOL>sha512_hash = NistBeaconCrypto.get_hash(<EOL>self.version,<EOL>self.frequency,<EOL>self.timestamp,<EOL>self.seed_value,<EOL>self.previous_output_value,<EOL>self.status_code,<EOL>)<EOL>sig_check_result = NistBeaconCrypto.verify(<EOL>timestamp=self.timestamp,<EOL>message_hash=sha512_hash,<EOL>signature=binascii.a2b_hex(self.signature_value)[::-<NUM_LIT:1>],<EOL>)<EOL>expected_signature = hashlib.sha512(<EOL>binascii.a2b_hex(self.signature_value)<EOL>).hexdigest().upper()<EOL>sig_hash_check = (expected_signature == self.output_value)<EOL>self._valid_signature = sig_check_result and sig_hash_check<EOL>self._pseudo_random = Random(self.output_value)<EOL>

:param version:
    Reported NIST randomness beacon version

:param frequency:
    The time interval, in seconds, between expected records

:param timestamp:
    The time the seed value was generated as the number of
    seconds since January 1, 1970

:param seed_value:
    A seed value represented as a 64 byte (512-bit) hex string
    value

:param previous_output_value:
    The SHA-512 hash value for the previous record - 64 byte hex
    string

:param signature_value:
    A digital signature (RSA) computed over (in order): version,
    frequency, timeStamp, seedValue, previousHashValue, statusCode

    Note: Except for version, the hash is on the byte
    representations and not the string representations of the data
    values

:param output_value:
    The SHA-512 hash of the signatureValue as a 64 byte hex string

:param status_code:
    The status code value:
        0 - Chain intact, values all good
        1 - Start of a new chain of values, previous hash value
            will be all zeroes
        2 - Time between values is greater than the frequency, but
            the chain is still intact


@property<EOL><INDENT>def frequency(self) -> int:<DEDENT>

return self._frequency<EOL>

:return: The time interval, in seconds, between expected records


@property<EOL><INDENT>def json(self) -> str:<DEDENT>

return self._json<EOL>

:return: The JSON representation of the beacon, as a string


@property<EOL><INDENT>def output_value(self) -> str:<DEDENT>

return self._output_value<EOL>

:return: The SHA-512 hash of the signatureValue as a 64 byte hex string


@property<EOL><INDENT>def previous_output_value(self) -> str:<DEDENT>

return self._previous_output_value<EOL>

:return:
    The SHA-512 hash value for the previous record - 64 byte hex
    string


@property<EOL><INDENT>def pseudo_random(self) -> Random:<DEDENT>

return self._pseudo_random<EOL>

:return:
    A python `random.Random` object that has been seeded with
    the value's `output_value`. This is a pseudo-random
    number generator


@property<EOL><INDENT>def seed_value(self) -> str:<DEDENT>

return self._seed_value<EOL>

:return:
    A seed value represented as a 64 byte (512-bit) hex string
    value


@property<EOL><INDENT>def signature_value(self) -> str:<DEDENT>

return self._signature_value<EOL>

:return:
    A digital signature (RSA) computed over (in order): version,
    frequency, timeStamp, seedValue, previousHashValue, statusCode

    Note: Except for version, the hash is on the byte
    representations and not the string representations of the data
    values


@property<EOL><INDENT>def status_code(self) -> str:<DEDENT>

return self._status_code<EOL>

:return:
    The status code value:
        0 - Chain intact, values all good
        1 - Start of a new chain of values, previous hash value
            will be all zeroes
        2 - Time between values is greater than the frequency, but
            the chain is still intact


@property<EOL><INDENT>def timestamp(self) -> int:<DEDENT>

return self._timestamp<EOL>

:return:
    The time the seed value was generated as the number of
    seconds since January 1, 1970


@property<EOL><INDENT>def valid_signature(self) -> bool:<DEDENT>

return self._valid_signature<EOL>

Shows the result of signature verification

First, required records (version, frequency, timestamp,
seed_value, previous_output_value) are packed together to form
a message. This message is then checked against the record's reported
signature field WITH the known NIST public key.

Second, the signature value is independently ran through a SHA512
hash. The result of this operation SHOULD equal the record's reported
output_value field.

As long as the result of the 'First' step and'ed with the 'Second'
step, the record is considered valid.

:return: 'True' if this record is valid. 'False' otherwise


@property<EOL><INDENT>def version(self) -> str:<DEDENT>

return self._version<EOL>

:return: Reported NIST randomness beacon version


@property<EOL><INDENT>def xml(self) -> str:<DEDENT>

return self._xml<EOL>

:return: The XML representation of the beacon, as a string


@classmethod<EOL><INDENT>def from_json(cls, input_json: str) -> '<STR_LIT>':<DEDENT>

try:<EOL><INDENT>data_dict = json.loads(input_json)<EOL><DEDENT>except ValueError:<EOL><INDENT>return None<EOL><DEDENT>required_values = {<EOL>cls._KEY_FREQUENCY: None,<EOL>cls._KEY_OUTPUT_VALUE: None,<EOL>cls._KEY_PREVIOUS_OUTPUT_VALUE: None,<EOL>cls._KEY_SEED_VALUE: None,<EOL>cls._KEY_SIGNATURE_VALUE: None,<EOL>cls._KEY_STATUS_CODE: None,<EOL>cls._KEY_TIMESTAMP: None,<EOL>cls._KEY_VERSION: None,<EOL>}<EOL>for key in required_values:<EOL><INDENT>if key in data_dict:<EOL><INDENT>required_values[key] = data_dict[key]<EOL><DEDENT><DEDENT>if None in required_values.values():<EOL><INDENT>return None<EOL><DEDENT>return cls(<EOL>version=required_values[cls._KEY_VERSION],<EOL>frequency=int(required_values[cls._KEY_FREQUENCY]),<EOL>timestamp=int(required_values[cls._KEY_TIMESTAMP]),<EOL>seed_value=required_values[cls._KEY_SEED_VALUE],<EOL>previous_output_value=required_values[<EOL>cls._KEY_PREVIOUS_OUTPUT_VALUE<EOL>],<EOL>signature_value=required_values[cls._KEY_SIGNATURE_VALUE],<EOL>output_value=required_values[cls._KEY_OUTPUT_VALUE],<EOL>status_code=required_values[cls._KEY_STATUS_CODE],<EOL>)<EOL>

Convert a string of JSON which represents a NIST randomness beacon
value into a 'NistBeaconValue' object.

:param input_json: JSON to build a 'Nist RandomnessBeaconValue' from
:return: A 'NistBeaconValue' object, 'None' otherwise


@classmethod<EOL><INDENT>def from_xml(cls, input_xml: str) -> '<STR_LIT>':<DEDENT>

invalid_result = None<EOL>understood_namespaces = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>required_values = {<EOL>cls._KEY_FREQUENCY: None,<EOL>cls._KEY_OUTPUT_VALUE: None,<EOL>cls._KEY_PREVIOUS_OUTPUT_VALUE: None,<EOL>cls._KEY_SEED_VALUE: None,<EOL>cls._KEY_SIGNATURE_VALUE: None,<EOL>cls._KEY_STATUS_CODE: None,<EOL>cls._KEY_TIMESTAMP: None,<EOL>cls._KEY_VERSION: None,<EOL>}<EOL>try:<EOL><INDENT>tree = ElementTree.ElementTree(ElementTree.fromstring(input_xml))<EOL><DEDENT>except ElementTree.ParseError:<EOL><INDENT>return invalid_result<EOL><DEDENT>for key in required_values:<EOL><INDENT>discovered_element = tree.find(<EOL>"<STR_LIT>".format('<STR_LIT>', key),<EOL>namespaces=understood_namespaces,<EOL>)<EOL>if not isinstance(discovered_element, ElementTree.Element):<EOL><INDENT>continue<EOL><DEDENT>required_values[key] = discovered_element.text<EOL><DEDENT>if None in required_values.values():<EOL><INDENT>return invalid_result<EOL><DEDENT>return cls(<EOL>version=required_values[cls._KEY_VERSION],<EOL>frequency=int(required_values[cls._KEY_FREQUENCY]),<EOL>timestamp=int(required_values[cls._KEY_TIMESTAMP]),<EOL>seed_value=required_values[cls._KEY_SEED_VALUE],<EOL>previous_output_value=required_values[<EOL>cls._KEY_PREVIOUS_OUTPUT_VALUE<EOL>],<EOL>signature_value=required_values[cls._KEY_SIGNATURE_VALUE],<EOL>output_value=required_values[cls._KEY_OUTPUT_VALUE],<EOL>status_code=required_values[cls._KEY_STATUS_CODE],<EOL>)<EOL>

Convert a string of XML which represents a NIST Randomness Beacon value
into a 'NistBeaconValue' object.

:param input_xml: XML to build a 'NistBeaconValue' from
:return: A 'NistBeaconValue' object, 'None' otherwise


@classmethod<EOL><INDENT>def chain_check(cls, timestamp: int) -> bool:<DEDENT>

<EOL>record = cls.get_record(timestamp)<EOL>if isinstance(record, NistBeaconValue) is False:<EOL><INDENT>return False<EOL><DEDENT>prev_record = cls.get_previous(record.timestamp)<EOL>next_record = cls.get_next(record.timestamp)<EOL>if prev_record is None and next_record is None:<EOL><INDENT>return False<EOL><DEDENT>if (<EOL>isinstance(prev_record, NistBeaconValue) and<EOL>isinstance(next_record, NistBeaconValue)<EOL>):<EOL><INDENT>return (<EOL>record.valid_signature and<EOL>prev_record.valid_signature and<EOL>next_record.valid_signature and<EOL>record.previous_output_value == prev_record.output_value and<EOL>next_record.previous_output_value == record.output_value<EOL>)<EOL><DEDENT>if (<EOL>prev_record is None and<EOL>isinstance(next_record, NistBeaconValue)<EOL>):<EOL><INDENT>return (<EOL>record.valid_signature and<EOL>next_record.valid_signature and<EOL>cls._INIT_RECORD == record and<EOL>next_record.previous_output_value == record.output_value<EOL>)<EOL><DEDENT>if (<EOL>isinstance(prev_record, NistBeaconValue) and<EOL>next_record is None<EOL>):<EOL><INDENT>return (<EOL>record.valid_signature and<EOL>prev_record.valid_signature and<EOL>record.previous_output_value == prev_record.output_value<EOL>)<EOL><DEDENT>

Given a record timestamp, verify the chain integrity.

:param timestamp: UNIX time / POSIX time / Epoch time
:return: 'True' if the timestamp fits the chain. 'False' otherwise.


@classmethod<EOL><INDENT>def get_first_record(<EOL>cls,<EOL>download: bool=True<EOL>) -> NistBeaconValue:<DEDENT>

if download:<EOL><INDENT>return NistBeacon.get_record(cls._INIT_RECORD.timestamp)<EOL><DEDENT>else:<EOL><INDENT>return NistBeaconValue.from_json(cls._INIT_RECORD.json)<EOL><DEDENT>

Get the first (oldest) record available. Since the first record
IS a known value in the system we can load it from constants.

:param download: 'True' will always reach out to NIST to get the
                 first record. 'False' returns a local copy.
:return: The first beacon value. 'None' otherwise.


@classmethod<EOL><INDENT>def get_last_record(cls) -> NistBeaconValue:<DEDENT>

return cls._query_nist("<STR_LIT>")<EOL>

Get the last (newest) record available.

:return: The last beacon value. 'None' otherwise.


@classmethod<EOL><INDENT>def get_next(cls, timestamp: int) -> NistBeaconValue:<DEDENT>

return cls._query_nist("<STR_LIT>".format(timestamp))<EOL>

Given a record timestamp, get the next record available.

:param timestamp: UNIX time / POSIX time / Epoch time
:return: The next beacon value if available. 'None' otherwise.


@classmethod<EOL><INDENT>def get_previous(cls, timestamp: int) -> NistBeaconValue:<DEDENT>

Given a record timestamp, get the previous record available.

:param timestamp: UNIX time / POSIX time / Epoch time
:return: The previous beacon value if available. 'None; otherwise


@classmethod<EOL><INDENT>def get_record(cls, timestamp: int) -> NistBeaconValue:<DEDENT>

return cls._query_nist(str(timestamp))<EOL>

Get a specific record (or next closest)

:param timestamp: UNIX time / POSIX time / Epoch time
:return: The requested beacon value if available. 'None' otherwise.


@classmethod<EOL><INDENT>def get_hash(<EOL>cls,<EOL>version: str,<EOL>frequency: int,<EOL>timestamp: int,<EOL>seed_value: str,<EOL>prev_output: str,<EOL>status_code: str,<EOL>) -> SHA512Hash:<DEDENT>

Given required properties from a NistBeaconValue,
compute the SHA512Hash object.

:param version: NistBeaconValue.version
:param frequency: NistBeaconValue.frequency
:param timestamp: NistBeaconValue.timestamp
:param seed_value: NistBeaconValue.seed_value
:param prev_output: NistBeaconValue.previous_output_value
:param status_code: NistBeaconValue.status_code

:return: SHA512 Hash for NistBeaconValue signature verification


@classmethod<EOL><INDENT>def verify(<EOL>cls,<EOL>timestamp: int,<EOL>message_hash: SHA512Hash,<EOL>signature: bytes,<EOL>) -> bool:<DEDENT>

<EOL>if timestamp < <NUM_LIT>:<EOL><INDENT>verifier = cls._VERIFIER_20130905<EOL><DEDENT>elif timestamp < <NUM_LIT>:<EOL><INDENT>verifier = None<EOL><DEDENT>else:<EOL><INDENT>verifier = cls._VERIFIER_20170808<EOL><DEDENT>if verifier:<EOL><INDENT>result = verifier.verify(<EOL>message_hash,<EOL>signature,<EOL>)<EOL><DEDENT>else:<EOL><INDENT>result = False<EOL><DEDENT>if isinstance(result, int):<EOL><INDENT>result = True if result == <NUM_LIT:1> else False<EOL><DEDENT>return result<EOL>

Verify a given NIST message hash and signature for a beacon value.

:param timestamp: The timestamp of the record being verified.
:param message_hash:
    The hash that was carried out over the message.
    This is an object belonging to the `Crypto.Hash` module.
:param signature: The signature that needs to be validated.
:return: True if verification is correct. False otherwise.


def get_versions() -> FileVersionResult:

version_counter = Counter()<EOL>versions_match = False<EOL>version_str = None<EOL>versions_discovered = {}<EOL>for version_obj in version_objects:<EOL><INDENT>discovered = version_obj.get_version()<EOL>versions_discovered[version_obj.key_name] = discovered<EOL>version_counter.update([discovered])<EOL><DEDENT>if len(version_counter) == <NUM_LIT:1>:<EOL><INDENT>versions_match = True<EOL>version_str = list(version_counter.keys())[<NUM_LIT:0>]<EOL><DEDENT>return FileVersionResult(<EOL>uniform=versions_match,<EOL>version_details=versions_discovered,<EOL>version_result=version_str,<EOL>)<EOL>

Search specific project files and extract versions to check.

:return: A FileVersionResult object for reporting.


for version_obj in version_objects:<EOL><INDENT>version_obj.set_version(new_version)<EOL><DEDENT>

Update all known version objects with a new version string.

:param new_version: The new version string to set in the project.


def __init__(<EOL>self,<EOL>key_name: str,<EOL>file_path: str,<EOL>magic_line: str,<EOL>strip_end_chars: int = <NUM_LIT:0>,<EOL>):

self.key_name = key_name<EOL>self.file_path = file_path<EOL>self.magic_line = magic_line<EOL>self.strip_end_chars = strip_end_chars<EOL>

A simple python object that is used to extract and
(eventually) update version values across the project

The 'magic_line' is the first part of a file line that
is before the version number. So for example, if the version
is stored on a line such as '__version__=1' your magic line
will be '__version__='. If you need to remove any extra characters
at the end of the version line, increase the 'strip_end_chars'
property on this object.

:param key_name: The key name, or reference name, for this file
:param file_path: The path to the file
:param magic_line: The "magic line" to search for
:param strip_end_chars: The number of characters to strip off the end


try:<EOL><INDENT>f = open(self.file_path, '<STR_LIT:r>')<EOL>lines = f.readlines()<EOL>f.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>return str(e)<EOL><DEDENT>result = '<STR_LIT>'<EOL>for line in lines:<EOL><INDENT>if self.magic_line in line:<EOL><INDENT>start = len(self.magic_line)<EOL>end = len(line) - self.strip_end_chars<EOL>result = line[start:end]<EOL>break<EOL><DEDENT><DEDENT>return result<EOL>

Open the file referenced in this object, and scrape the version.

:return:
    The version as a string, an empty string if there is no match
    to the magic_line, or any file exception messages encountered.


def set_version(self, new_version: str):

try:<EOL><INDENT>f = open(self.file_path, '<STR_LIT:r>')<EOL>lines = f.readlines()<EOL>f.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>print(str(e))<EOL>return<EOL><DEDENT>for idx, line in enumerate(lines):<EOL><INDENT>if self.magic_line in line:<EOL><INDENT>start = len(self.magic_line)<EOL>end = len(line) - self.strip_end_chars<EOL>start_str = line[<NUM_LIT:0>:start]<EOL>end_str = line[end:]<EOL>lines[idx] = start_str + new_version + end_str<EOL><DEDENT><DEDENT>try:<EOL><INDENT>f = open(self.file_path, '<STR_LIT:w>')<EOL>f.writelines(lines)<EOL>f.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>print(str(e))<EOL>return<EOL><DEDENT>

Set the version for this given file.

:param new_version: The new version string to set.


def __init__(<EOL>self,<EOL>uniform: bool,<EOL>version_details: dict,<EOL>version_result: str<EOL>):

self.uniform = uniform<EOL>self.version_details = version_details<EOL>self.version_result = version_result<EOL>

Object to contain results about the project's version

:param uniform: True if versions are the same, False otherwise
:param version_details: Full details on each file and reported version
:param version_result: The project's version as a single string.


def purge_key(surrogate_key, service_id, api_key):

logger = logging.getLogger(__name__)<EOL>api_root = '<STR_LIT>'<EOL>path = '<STR_LIT>'.format(<EOL>service=service_id,<EOL>surrogate_key=surrogate_key)<EOL>logger.info('<STR_LIT>'.format(path))<EOL>r = requests.post(api_root + path,<EOL>headers={'<STR_LIT>': api_key,<EOL>'<STR_LIT>': '<STR_LIT:application/json>'})<EOL>if r.status_code != <NUM_LIT:200>:<EOL><INDENT>raise FastlyError(r.json)<EOL><DEDENT>

Instant purge URLs with a given surrogate key from the Fastly caches.

    Parameters
    ----------
    surrogate_key : `str`
        Surrogate key header (``x-amz-meta-surrogate-key``) value of objects
        to purge from the Fastly cache.
    service_id : `str`
        Fastly service ID.
    api_key : `str`
        Fastly API key.

    Raises
    ------
    FastlyError
       Error with the Fastly API usage.

    Notes
    -----
    This function uses Fastly's ``/service/{service}/purge/{key}`` endpoint.
    See the `Fastly Purge documentation <http://ls.st/jxg>`_ for more
    information.

    For other Fastly APIs, consider using `fastly-py
    <https://github.com/fastly/fastly-py>`_.


logger = logging.getLogger(__name__)<EOL>logger.debug('<STR_LIT>'.format(<EOL>bucket_name, path_prefix, source_dir))<EOL>session = boto3.session.Session(<EOL>profile_name=aws_profile,<EOL>aws_access_key_id=aws_access_key_id,<EOL>aws_secret_access_key=aws_secret_access_key)<EOL>s3 = session.resource('<STR_LIT>')<EOL>bucket = s3.Bucket(bucket_name)<EOL>metadata = {}<EOL>if surrogate_key is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_key<EOL><DEDENT>if surrogate_control is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_control<EOL><DEDENT>manager = ObjectManager(session, bucket_name, path_prefix)<EOL>for (rootdir, dirnames, filenames) in os.walk(source_dir):<EOL><INDENT>bucket_root = os.path.relpath(rootdir, start=source_dir)<EOL>if bucket_root in ('<STR_LIT:.>', '<STR_LIT:/>'):<EOL><INDENT>bucket_root = '<STR_LIT>'<EOL><DEDENT>bucket_dirnames = manager.list_dirnames_in_directory(bucket_root)<EOL>for bucket_dirname in bucket_dirnames:<EOL><INDENT>if bucket_dirname not in dirnames:<EOL><INDENT>logger.debug(('<STR_LIT>'.format(<EOL>bucket_dirname)))<EOL>manager.delete_directory(bucket_dirname)<EOL><DEDENT><DEDENT>bucket_filenames = manager.list_filenames_in_directory(bucket_root)<EOL>for bucket_filename in bucket_filenames:<EOL><INDENT>if bucket_filename not in filenames:<EOL><INDENT>bucket_filename = os.path.join(bucket_root, bucket_filename)<EOL>logger.debug(<EOL>'<STR_LIT>'.format(bucket_filename))<EOL>manager.delete_file(bucket_filename)<EOL><DEDENT><DEDENT>for filename in filenames:<EOL><INDENT>local_path = os.path.join(rootdir, filename)<EOL>bucket_path = os.path.join(path_prefix, bucket_root, filename)<EOL>logger.debug('<STR_LIT>'.format(bucket_path))<EOL>upload_file(local_path, bucket_path, bucket,<EOL>metadata=metadata, acl=acl,<EOL>cache_control=cache_control)<EOL><DEDENT>if upload_dir_redirect_objects is True:<EOL><INDENT>bucket_dir_path = os.path.join(path_prefix, bucket_root)<EOL>create_dir_redirect_object(<EOL>bucket_dir_path,<EOL>bucket,<EOL>metadata=metadata,<EOL>acl=acl,<EOL>cache_control=cache_control)<EOL><DEDENT><DEDENT>

Upload a directory of files to S3.

    This function places the contents of the Sphinx HTML build directory
    into the ``/path_prefix/`` directory of an *existing* S3 bucket.
    Existing files on S3 are overwritten; files that no longer exist in the
    ``source_dir`` are deleted from S3.

    Parameters
    ----------
    bucket_name : `str`
        Name of the S3 bucket where documentation is uploaded.
    path_prefix : `str`
        The root directory in the bucket where documentation is stored.
    source_dir : `str`
        Path of the Sphinx HTML build directory on the local file system.
        The contents of this directory are uploaded into the ``/path_prefix/``
        directory of the S3 bucket.
    upload_dir_redirect_objects : `bool`, optional
        A feature flag to enable uploading objects to S3 for every directory.
        These objects contain ``x-amz-meta-dir-redirect=true`` HTTP headers
        that tell Fastly to issue a 301 redirect from the directory object to
        the `index.html`` in that directory.
    surrogate_key : `str`, optional
        The surrogate key to insert in the header of all objects
        in the ``x-amz-meta-surrogate-key`` field. This key is used to purge
        builds from the Fastly CDN when Editions change.
        If `None` then no header will be set.
    cache_control : `str`, optional
        This sets the ``Cache-Control`` header on the uploaded
        files. The ``Cache-Control`` header specifically dictates how content
        is cached by the browser (if ``surrogate_control`` is also set).
    surrogate_control : `str`, optional
        This sets the ``x-amz-meta-surrogate-control`` header
        on the uploaded files. The ``Surrogate-Control``
        or ``x-amz-meta-surrogate-control`` header is used in priority by
        Fastly to givern it's caching. This caching policy is *not* passed
        to the browser.
    acl : `str`, optional
        The pre-canned AWS access control list to apply to this upload.
        Can be ``'public-read'``, which allow files to be downloaded
        over HTTP by the public. See
        https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        for an overview of S3's pre-canned ACL lists. Note that ACL settings
        are not validated locally. Default is `None`, meaning that no ACL
        is applied to an individual object. In this case, use ACLs applied
        to the bucket itself.
    aws_access_key_id : `str`, optional
        The access key for your AWS account. Also set
        ``aws_secret_access_key``.
    aws_secret_access_key : `str`, optional
        The secret key for your AWS account.
    aws_profile : `str`, optional
        Name of AWS profile in :file:`~/.aws/credentials`. Use this instead
        of ``aws_access_key_id`` and ``aws_secret_access_key`` for file-based
        credentials.

    Notes
    -----
    ``cache_control`` and  ``surrogate_control`` can be used together.
    ``surrogate_control`` takes priority in setting Fastly's POP caching,
    while ``cache_control`` then sets the browser's caching. For example:

    - ``cache_control='no-cache'``
    - ``surrogate_control='max-age=31536000'``

    together will ensure that the browser always does an ETAG server query,
    but that Fastly will cache the content for one year (or until purged).
    This configuration is good for files that are frequently changed in place.

    For immutable uploads simply using ``cache_control`` is more efficient
    since it allows the browser to also locally cache content.

    .. seelso:

       - `Fastly: Cache control tutorial
         <https://docs.fastly.com/guides/tutorials/cache-control-tutorial>`_.
       - `Google: HTTP caching <http://ls.st/39v>`_.


def upload_file(local_path, bucket_path, bucket,<EOL>metadata=None, acl=None, cache_control=None):

logger = logging.getLogger(__name__)<EOL>extra_args = {}<EOL>if acl is not None:<EOL><INDENT>extra_args['<STR_LIT>'] = acl<EOL><DEDENT>if metadata is not None and len(metadata) > <NUM_LIT:0>:  <EOL><INDENT>extra_args['<STR_LIT>'] = metadata<EOL><DEDENT>if cache_control is not None:<EOL><INDENT>extra_args['<STR_LIT>'] = cache_control<EOL><DEDENT>content_type, content_encoding = mimetypes.guess_type(local_path,<EOL>strict=False)<EOL>if content_type is not None:<EOL><INDENT>extra_args['<STR_LIT>'] = content_type<EOL><DEDENT>logger.debug(str(extra_args))<EOL>obj = bucket.Object(bucket_path)<EOL>obj.upload_file(local_path, ExtraArgs=extra_args)<EOL>

Upload a file to the S3 bucket.

    This function uses the mimetypes module to guess and then set the
    Content-Type and Encoding-Type headers.

    Parameters
    ----------
    local_path : `str`
        Full path to a file on the local file system.
    bucket_path : `str`
        Destination path (also known as the key name) of the file in the
        S3 bucket.
    bucket : boto3 Bucket instance
        S3 bucket.
    metadata : `dict`, optional
        Header metadata values. These keys will appear in headers as
        ``x-amz-meta-*``.
    acl : `str`, optional
        A pre-canned access control list. See
        https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        Default is `None`, mean that no ACL is applied to the object.
    cache_control : `str`, optional
        The cache-control header value. For example, ``'max-age=31536000'``.


def upload_object(bucket_path, bucket, content='<STR_LIT>',<EOL>metadata=None, acl=None, cache_control=None,<EOL>content_type=None):

obj = bucket.Object(bucket_path)<EOL>args = {}<EOL>if metadata is not None and len(metadata) > <NUM_LIT:0>:  <EOL><INDENT>args['<STR_LIT>'] = metadata<EOL><DEDENT>if acl is not None:<EOL><INDENT>args['<STR_LIT>'] = acl<EOL><DEDENT>if cache_control is not None:<EOL><INDENT>args['<STR_LIT>'] = cache_control<EOL><DEDENT>if content_type is not None:<EOL><INDENT>args['<STR_LIT>'] = content_type<EOL><DEDENT>obj.put(Body=content, **args)<EOL>

Upload an arbitrary object to an S3 bucket.

    Parameters
    ----------
    bucket_path : `str`
        Destination path (also known as the key name) of the file in the
        S3 bucket.
    content : `str` or `bytes`, optional
        Object content.
    bucket : boto3 Bucket instance
        S3 bucket.
    metadata : `dict`, optional
        Header metadata values. These keys will appear in headers as
        ``x-amz-meta-*``.
    acl : `str`, optional
        A pre-canned access control list. See
        https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        Default is `None`, meaning that no ACL is applied to the object.
    cache_control : `str`, optional
        The cache-control header value. For example, ``'max-age=31536000'``.
    content_type : `str`, optional
        The object's content type (such as ``text/html``). If left unset,
        no MIME type is passed to boto3 (which defaults to
        ``binary/octet-stream``).


def create_dir_redirect_object(bucket_dir_path, bucket, metadata=None,<EOL>acl=None, cache_control=None):

<EOL>bucket_dir_path = bucket_dir_path.rstrip('<STR_LIT:/>')<EOL>if metadata is not None:<EOL><INDENT>metadata = dict(metadata)<EOL><DEDENT>else:<EOL><INDENT>metadata = {}<EOL><DEDENT>metadata['<STR_LIT>'] = '<STR_LIT:true>'<EOL>upload_object(bucket_dir_path,<EOL>content='<STR_LIT>',<EOL>bucket=bucket,<EOL>metadata=metadata,<EOL>acl=acl,<EOL>cache_control=cache_control)<EOL>

Create an S3 object representing a directory that's designed to
    redirect a browser (via Fastly) to the ``index.html`` contained inside
    that directory.

    Parameters
    ----------
    bucket_dir_path : `str`
        Full name of the object in the S3, which is equivalent to a POSIX
        directory path, like ``dir1/dir2``.
    bucket : boto3 Bucket instance
        S3 bucket.
    metadata : `dict`, optional
        Header metadata values. These keys will appear in headers as
        ``x-amz-meta-*``.
    acl : `str`, optional
        A pre-canned access control list. See
        https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        Default is None, meaning that no ACL is applied to the object.
    cache_control : `str`, optional
        The cache-control header value. For example, ``'max-age=31536000'``.


prefix = self._create_prefix(dirname)<EOL>filenames = []<EOL>for obj in self._bucket.objects.filter(Prefix=prefix):<EOL><INDENT>if obj.key.endswith('<STR_LIT:/>'):<EOL><INDENT>continue<EOL><DEDENT>obj_dirname = os.path.dirname(obj.key)<EOL>if obj_dirname == prefix:<EOL><INDENT>filenames.append(os.path.relpath(obj.key,<EOL>start=prefix))<EOL><DEDENT><DEDENT>return filenames<EOL>

List all file-type object names that exist at the root of this
        bucket directory.

        Parameters
        ----------
        dirname : `str`
            Directory name in the bucket relative to ``bucket_root/``.

        Returns
        -------
        filenames : `list`
            List of file names (`str`), relative to ``bucket_root/``, that
            exist at the root of ``dirname``.


prefix = self._create_prefix(dirname)<EOL>dirnames = []<EOL>for obj in self._bucket.objects.filter(Prefix=prefix):<EOL><INDENT>dirname = os.path.dirname(obj.key)<EOL>if dirname == '<STR_LIT>':<EOL><INDENT>dirname = obj.key + '<STR_LIT:/>'<EOL><DEDENT>rel_dirname = os.path.relpath(dirname, start=prefix)<EOL>dir_parts = rel_dirname.split('<STR_LIT:/>')<EOL>if len(dir_parts) == <NUM_LIT:1>:<EOL><INDENT>dirnames.append(dir_parts[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>dirnames = list(set(dirnames))<EOL>for filtered_dir in ('<STR_LIT:.>', '<STR_LIT:..>'):<EOL><INDENT>if filtered_dir in dirnames:<EOL><INDENT>dirnames.remove(filtered_dir)<EOL><DEDENT><DEDENT>return dirnames<EOL>

List all names of directories that exist at the root of this
        bucket directory.

        Note that *directories* don't exist in S3; rather directories are
        inferred from path names.

        Parameters
        ----------
        dirname : `str`
            Directory name in the bucket relative to ``bucket_root``.

        Returns
        -------
        dirnames : `list`
            List of directory names (`str`), relative to ``bucket_root/``,
            that exist at the root of ``dirname``.


if dirname in ('<STR_LIT:.>', '<STR_LIT:/>'):<EOL><INDENT>dirname = '<STR_LIT>'<EOL><DEDENT>prefix = os.path.join(self._bucket_root, dirname)<EOL>prefix = prefix.rstrip('<STR_LIT:/>')<EOL>return prefix<EOL>

Make an absolute directory path in the bucker for dirname,
        which is is assumed relative to the self._bucket_root prefix directory.


key = os.path.join(self._bucket_root, filename)<EOL>objects = list(self._bucket.objects.filter(Prefix=key))<EOL>for obj in objects:<EOL><INDENT>obj.delete()<EOL><DEDENT>

Delete a file from the bucket.

        Parameters
        ----------
        filename : `str`
            Name of the file, relative to ``bucket_root/``.


key = os.path.join(self._bucket_root, dirname)<EOL>if not key.endswith('<STR_LIT:/>'):<EOL><INDENT>key += '<STR_LIT:/>'<EOL><DEDENT>key_objects = [{'<STR_LIT>': obj.key}<EOL>for obj in self._bucket.objects.filter(Prefix=key)]<EOL>if len(key_objects) == <NUM_LIT:0>:<EOL><INDENT>msg = '<STR_LIT>'.format(dirname)<EOL>raise RuntimeError(msg)<EOL><DEDENT>delete_keys = {'<STR_LIT>': key_objects}<EOL>s3 = self._session.resource('<STR_LIT>')<EOL>r = s3.meta.client.delete_objects(Bucket=self._bucket.name,<EOL>Delete=delete_keys)<EOL>self._logger.debug(r)<EOL>if '<STR_LIT>' in r:<EOL><INDENT>raise S3Error('<STR_LIT>'.format(key))<EOL><DEDENT>

Delete a directory (and contents) from the bucket.

        Parameters
        ----------
        dirname : `str`
            Name of the directory, relative to ``bucket_root/``.

        Raises
        ------
        RuntimeError
            Raised when there are no objects to delete (directory
            does not exist).


if not src_path.endswith('<STR_LIT:/>'):<EOL><INDENT>src_path += '<STR_LIT:/>'<EOL><DEDENT>if not dest_path.endswith('<STR_LIT:/>'):<EOL><INDENT>dest_path += '<STR_LIT:/>'<EOL><DEDENT>common_prefix = os.path.commonprefix([src_path, dest_path])<EOL>if common_prefix == src_path:<EOL><INDENT>msg = '<STR_LIT>'.format(<EOL>common_prefix, src_path)<EOL>raise RuntimeError(msg)<EOL><DEDENT>if common_prefix == dest_path:<EOL><INDENT>msg = '<STR_LIT>'.format(<EOL>common_prefix, dest_path)<EOL>raise RuntimeError(msg)<EOL><DEDENT>delete_dir(bucket_name, dest_path,<EOL>aws_access_key_id, aws_secret_access_key)<EOL>session = boto3.session.Session(<EOL>aws_access_key_id=aws_access_key_id,<EOL>aws_secret_access_key=aws_secret_access_key,<EOL>profile_name=aws_profile)<EOL>s3 = session.resource('<STR_LIT>')<EOL>bucket = s3.Bucket(bucket_name)<EOL>for src_obj in bucket.objects.filter(Prefix=src_path):<EOL><INDENT>src_rel_path = os.path.relpath(src_obj.key, start=src_path)<EOL>dest_key_path = os.path.join(dest_path, src_rel_path)<EOL>head = s3.meta.client.head_object(Bucket=bucket_name,<EOL>Key=src_obj.key)<EOL>metadata = head['<STR_LIT>']<EOL>content_type = head['<STR_LIT>']<EOL>if cache_control is None and '<STR_LIT>' in head:<EOL><INDENT>cache_control = head['<STR_LIT>']<EOL><DEDENT>if surrogate_control is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_control<EOL><DEDENT>if surrogate_key is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_key<EOL><DEDENT>s3.meta.client.copy_object(<EOL>Bucket=bucket_name,<EOL>Key=dest_key_path,<EOL>CopySource={'<STR_LIT>': bucket_name, '<STR_LIT>': src_obj.key},<EOL>MetadataDirective='<STR_LIT>',<EOL>Metadata=metadata,<EOL>ACL='<STR_LIT>',<EOL>CacheControl=cache_control,<EOL>ContentType=content_type)<EOL><DEDENT>if create_directory_redirect_object:<EOL><INDENT>dest_dirname = dest_path.rstrip('<STR_LIT:/>')<EOL>obj = bucket.Object(dest_dirname)<EOL>metadata = {'<STR_LIT>': '<STR_LIT:true>'}<EOL>obj.put(Body='<STR_LIT>',<EOL>ACL='<STR_LIT>',<EOL>Metadata=metadata,<EOL>CacheControl=cache_control)<EOL><DEDENT>

Copy objects from one directory in a bucket to another directory in
    the same bucket.

    Object metadata is preserved while copying, with the following exceptions:

    - If a new surrogate key is provided it will replace the original one.
    - If ``cache_control`` and ``surrogate_control`` values are provided they
      will replace the old one.

    Parameters
    ----------
    bucket_name : `str`
        Name of an S3 bucket.
    src_path : `str`
        Source directory in the S3 bucket. The ``src_path`` should ideally end
        in a trailing `'/'`. E.g. `'dir/dir2/'`.
    dest_path : `str`
        Destination directory in the S3 bucket. The ``dest_path`` should
        ideally end in a trailing `'/'`. E.g. `'dir/dir2/'`. The destination
        path cannot contain the source path.
    aws_access_key_id : `str`
        The access key for your AWS account. Also set
        ``aws_secret_access_key``.
    aws_secret_access_key : `str`
        The secret key for your AWS account.
    aws_profile : `str`, optional
        Name of AWS profile in :file:`~/.aws/credentials`. Use this instead
        of ``aws_access_key_id`` and ``aws_secret_access_key`` for file-based
        credentials.
    surrogate_key : `str`, optional
        The surrogate key to insert in the header of all objects in the
        ``x-amz-meta-surrogate-key`` field. This key is used to purge
        builds from the Fastly CDN when Editions change.
        If `None` then no header will be set.
        If the object already has a ``x-amz-meta-surrogate-key`` header then
        it will be replaced.
    cache_control : `str`, optional
        This sets (and overrides) the ``Cache-Control`` header on the copied
        files. The ``Cache-Control`` header specifically dictates how content
        is cached by the browser (if ``surrogate_control`` is also set).
    surrogate_control : `str`, optional
        This sets (and overrides) the ``x-amz-meta-surrogate-control`` header
        on the copied files. The ``Surrogate-Control``
        or ``x-amz-meta-surrogate-control`` header is used in priority by
        Fastly to givern it's caching. This caching policy is *not* passed
        to the browser.
    create_directory_redirect_object : `bool`, optional
        Create a directory redirect object for the root directory. The
        directory redirect object is an empty S3 object named after the
        directory (without a trailing slash) that contains a
        ``x-amz-meta-dir-redirect=true`` HTTP header. LSST the Docs' Fastly
        VCL is configured to redirect requests for a directory path to the
        directory's ``index.html`` (known as *courtesy redirects*).

    Raises
    ------
    ltdconveyor.s3.S3Error
        Thrown by any unexpected faults from the S3 API.
    RuntimeError
        Thrown when the source and destination directories are the same.


def delete_dir(bucket_name, root_path,<EOL>aws_access_key_id=None, aws_secret_access_key=None,<EOL>aws_profile=None):

logger = logging.getLogger(__name__)<EOL>session = boto3.session.Session(<EOL>aws_access_key_id=aws_access_key_id,<EOL>aws_secret_access_key=aws_secret_access_key)<EOL>s3 = session.resource('<STR_LIT>')<EOL>client = s3.meta.client<EOL>if not root_path.endswith('<STR_LIT:/>'):<EOL><INDENT>root_path.rstrip('<STR_LIT:/>')<EOL><DEDENT>paginator = client.get_paginator('<STR_LIT>')<EOL>pages = paginator.paginate(Bucket=bucket_name, Prefix=root_path)<EOL>keys = dict(Objects=[])<EOL>for item in pages.search('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>keys['<STR_LIT>'].append({'<STR_LIT>': item['<STR_LIT>']})<EOL><DEDENT>except TypeError:  <EOL><INDENT>continue<EOL><DEDENT>if len(keys['<STR_LIT>']) >= <NUM_LIT:1000>:<EOL><INDENT>try:<EOL><INDENT>client.delete_objects(Bucket=bucket_name, Delete=keys)<EOL><DEDENT>except Exception:<EOL><INDENT>message = '<STR_LIT>' % root_path<EOL>logger.exception(message)<EOL>raise S3Error(message)<EOL><DEDENT>keys = dict(Objects=[])<EOL><DEDENT><DEDENT>if len(keys['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>try:<EOL><INDENT>client.delete_objects(Bucket=bucket_name, Delete=keys)<EOL><DEDENT>except Exception:<EOL><INDENT>message = '<STR_LIT>' % root_path<EOL>logger.exception(message)<EOL>raise S3Error(message)<EOL><DEDENT><DEDENT>

Delete all objects in the S3 bucket named ``bucket_name`` that are
    found in the ``root_path`` directory.

    Parameters
    ----------
    bucket_name : `str`
        Name of an S3 bucket.
    root_path : `str`
        Directory in the S3 bucket that will be deleted.
    aws_access_key_id : `str`
        The access key for your AWS account. Also set
        ``aws_secret_access_key``.
    aws_secret_access_key : `str`
        The secret key for your AWS account.
    aws_profile : `str`, optional
        Name of AWS profile in :file:`~/.aws/credentials`. Use this instead
        of ``aws_access_key_id`` and ``aws_secret_access_key`` for file-based
        credentials.

    Raises
    ------
    ltdconveyor.s3.S3Error
        Thrown by any unexpected faults from the S3 API.