domain = self.get_clean_domain()<EOL>if domain in list(self.custom_site_mapping.keys()):<EOL><INDENT>classes = self.custom_site_mapping.get(domain).split('<STR_LIT:|>')<EOL>for classname in classes:<EOL><INDENT>KNOWN_IMG_DOM_NAMES.append(classname)<EOL><DEDENT><DEDENT>image = None<EOL>doc = self.article.raw_doc<EOL>def _check_elements(elements):<EOL><INDENT>for element in elements:<EOL><INDENT>tag = self.parser.getTag(element)<EOL>if tag == '<STR_LIT>':<EOL><INDENT>return element<EOL><DEDENT>images = self.parser.getElementsByTag(element, tag='<STR_LIT>')<EOL>if images:<EOL><INDENT>return images[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>return None<EOL><DEDENT>for css in KNOWN_IMG_DOM_NAMES:<EOL><INDENT>elements = self.parser.getElementsByTag(doc, attr="<STR_LIT:id>", value=css)<EOL>image = _check_elements(elements)<EOL>if image is not None:<EOL><INDENT>src = self.parser.getAttribute(image, attr='<STR_LIT:src>')<EOL>if src:<EOL><INDENT>return self.get_image(src, score=<NUM_LIT>, extraction_type='<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>for css in KNOWN_IMG_DOM_NAMES:<EOL><INDENT>elements = self.parser.getElementsByTag(doc, attr='<STR_LIT:class>', value=css)<EOL>image = _check_elements(elements)<EOL>if image is not None:<EOL><INDENT>src = self.parser.getAttribute(image, attr='<STR_LIT:src>')<EOL>if src:<EOL><INDENT>return self.get_image(src, score=<NUM_LIT>, extraction_type='<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

\
        in here we check for known image contains from sites
        we've checked out like yahoo, techcrunch, etc... that have
        * known  places to look for good images.
        * TODO: enable this to use a series of settings files
          so people can define what the image ids/classes
          are on specific sites


o = urlparse(src)<EOL>if o.netloc != '<STR_LIT>':<EOL><INDENT>return o.geturl()<EOL><DEDENT>return urljoin(self.article.final_url, src)<EOL>

\
        This method will take an image path and build
        out the absolute path to that image
        * using the initial url we crawled
          so we can find a link to the image
          if they use relative urls like ../myimage.jpg


kwargs = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>', '<STR_LIT:value>': '<STR_LIT>'}<EOL>meta = self.parser.getElementsByTag(self.article.doc, **kwargs)<EOL>if meta:<EOL><INDENT>favicon = self.parser.getAttribute(meta[<NUM_LIT:0>], '<STR_LIT>')<EOL>return favicon<EOL><DEDENT>return '<STR_LIT>'<EOL>

Extract the favicon from a website
http://en.wikipedia.org/wiki/Favicon
<link rel="shortcut icon" type="image/png" href="favicon.png" />
<link rel="icon" type="image/png" href="favicon.png" />


if self.article.final_url:<EOL><INDENT>kwargs = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>', '<STR_LIT:value>': '<STR_LIT>'}<EOL>meta = self.parser.getElementsByTag(self.article.doc, **kwargs)<EOL>if meta is not None and len(meta) > <NUM_LIT:0>:<EOL><INDENT>href = self.parser.getAttribute(meta[<NUM_LIT:0>], '<STR_LIT>')<EOL>if href:<EOL><INDENT>href = href.strip()<EOL>o = urlparse(href)<EOL>if not o.hostname:<EOL><INDENT>tmp = urlparse(self.article.final_url)<EOL>domain = '<STR_LIT>' % (tmp.scheme, tmp.hostname)<EOL>href = urljoin(domain, href)<EOL><DEDENT>return href<EOL><DEDENT><DEDENT><DEDENT>return self.article.final_url<EOL>

if the article has meta canonical link set in the url


<EOL>attr = self.parser.getAttribute(self.article.doc, attr='<STR_LIT>')<EOL>if attr is None:<EOL><INDENT>items = [<EOL>{'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>', '<STR_LIT:value>': '<STR_LIT>'},<EOL>{'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT:name>', '<STR_LIT:value>': '<STR_LIT>'}<EOL>]<EOL>for item in items:<EOL><INDENT>meta = self.parser.getElementsByTag(self.article.doc, **item)<EOL>if meta:<EOL><INDENT>attr = self.parser.getAttribute(meta[<NUM_LIT:0>], attr='<STR_LIT:content>')<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if attr:<EOL><INDENT>value = attr[:<NUM_LIT:2>]<EOL>if re.search(RE_LANG, value):<EOL><INDENT>return value.lower()<EOL><DEDENT><DEDENT>return None<EOL>


meta = self.parser.css_select(self.article.doc, meta_name)<EOL>content = None<EOL>if meta is not None and len(meta) > <NUM_LIT:0>:<EOL><INDENT>content = self.parser.getAttribute(meta[<NUM_LIT:0>], '<STR_LIT:content>')<EOL><DEDENT>if content:<EOL><INDENT>return content.strip()<EOL><DEDENT>return '<STR_LIT>'<EOL>

Extract a given meta content form document


return self.get_meta_content("<STR_LIT>")<EOL>

if the article has meta description set in the source, use that


if the article has meta keywords set in the source, use that


encoding = get_encodings_from_content(self.article.raw_html)<EOL>return encoding and encoding[<NUM_LIT:0>] or None<EOL>


return self._top_image_node<EOL>

etree: The most likely top image element node

            Note:
                Read only


return self._src<EOL>

str: Source URL for the image

            Note:
                Read only


return self._confidence_score<EOL>

float: The confidence score that this is the main image

            Note:
                Read only


return self._height<EOL>

int: The image height in pixels

            Note:
                Read only


return self._width<EOL>

int: The image width in pixels

            Note:
                Read only


return self._extraction_type<EOL>

str: The extraction type used

            Note:
                Read only


return self._bytes<EOL>

int: The size of the image in bytes

            Note:
                Read only


<EOL>if self.config.use_meta_language:<EOL><INDENT>if self.article.meta_lang:<EOL><INDENT>return self.article.meta_lang[:<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return self.config.target_language<EOL>

\
        Returns the language is by the article or
        the configuration language


self.parser.stripTags(self.get_top_node(), '<STR_LIT:a>')<EOL>

\
        cleans up and converts any nodes that
        should be considered text into text


for elm in self.parser.getElementsByTag(self.top_node, tag='<STR_LIT>'):<EOL><INDENT>elm.text = r'<STR_LIT>'.format(elm.text)<EOL><DEDENT>


gravity_items = self.parser.css_select(self.top_node, "<STR_LIT>")<EOL>for item in gravity_items:<EOL><INDENT>score = self.parser.getAttribute(item, '<STR_LIT>')<EOL>score = int(score, <NUM_LIT:0>)<EOL>if score < <NUM_LIT:1>:<EOL><INDENT>item.getparent().remove(item)<EOL><DEDENT><DEDENT>

\
        if there are elements inside our top node
        that have a negative gravity score,
        let's give em the boot


self.parser.stripTags(self.get_top_node(), '<STR_LIT:b>', '<STR_LIT>', '<STR_LIT:i>', '<STR_LIT>', '<STR_LIT>')<EOL>

\
        replace common tags with just
        text so we don't have any crazy formatting issues
        so replace <br>, <i>, <strong>, etc....
        with whatever text is inside them
        code : http://lxml.de/api/lxml.etree-module.html#strip_tags


all_nodes = self.parser.getElementsByTags(self.get_top_node(), ['<STR_LIT:*>'])<EOL>all_nodes.reverse()<EOL>for elm in all_nodes:<EOL><INDENT>tag = self.parser.getTag(elm)<EOL>text = self.parser.getText(elm)<EOL>stop_words = self.stopwords_class(language=self.get_language()).get_stopword_count(text)<EOL>if ((tag != '<STR_LIT>' or text != '<STR_LIT>') and stop_words.get_stopword_count() < <NUM_LIT:3> and<EOL>len(self.parser.getElementsByTag(elm, tag='<STR_LIT:object>')) == <NUM_LIT:0> and<EOL>len(self.parser.getElementsByTag(elm, tag='<STR_LIT>')) == <NUM_LIT:0>):<EOL><INDENT>self.parser.remove(elm)<EOL><DEDENT>else:<EOL><INDENT>trimmed = self.parser.getText(elm)<EOL>if trimmed.startswith("<STR_LIT:(>") and trimmed.endswith("<STR_LIT:)>"):<EOL><INDENT>self.parser.remove(elm)<EOL><DEDENT><DEDENT><DEDENT>

\
        remove paragraphs that have less than x number of words,
        would indicate that it's some sort of link


if isinstance(content, bytes):<EOL><INDENT>find_charset = re.compile(<EOL>br'<STR_LIT>', flags=re.I<EOL>).findall<EOL>find_xml = re.compile(<EOL>br'<STR_LIT>'<EOL>).findall<EOL>return [encoding.decode('<STR_LIT:utf-8>') for encoding in<EOL>find_charset(content) + find_xml(content)]<EOL><DEDENT>else:<EOL><INDENT>find_charset = re.compile(<EOL>r'<STR_LIT>', flags=re.I<EOL>).findall<EOL>find_xml = re.compile(<EOL>r'<STR_LIT>'<EOL>).findall<EOL>return find_charset(content) + find_xml(content)<EOL><DEDENT>

Code from:
https://github.com/sigmavirus24/requests-toolbelt/blob/master/requests_toolbelt/utils/deprecated.py
Return encodings from given content string.
:param content: string to extract encodings from.


@classmethod<EOL><INDENT>def store_image(cls, http_client, link_hash, src, config):<DEDENT>

<EOL>image = cls.read_localfile(link_hash, src, config)<EOL>if image:<EOL><INDENT>return image<EOL><DEDENT>if src.startswith('<STR_LIT>'):<EOL><INDENT>image = cls.write_localfile_base64(link_hash, src, config)<EOL>return image<EOL><DEDENT>data = http_client.fetch(src)<EOL>if data:<EOL><INDENT>image = cls.write_localfile(data, link_hash, src, config)<EOL>if image:<EOL><INDENT>return image<EOL><DEDENT><DEDENT>return None<EOL>

\
        Writes an image src http string to disk as a temporary file
        and returns the LocallyStoredImage object
        that has the info you should need on the image


def smart_unicode(string, encoding='<STR_LIT:utf-8>', strings_only=False, errors='<STR_LIT:strict>'):

<EOL>return force_unicode(string, encoding, strings_only, errors)<EOL>

Returns a unicode object representing 's'. Treats bytestrings using the
'encoding' codec.

If strings_only is True, don't convert (some) non-string-like objects.


return isinstance(obj, (type(None), int,<EOL>datetime.datetime, datetime.date, datetime.time,<EOL>float, Decimal))<EOL>

Determine if the object instance is of a protected type.

    Objects of protected types are preserved as-is when passed to
    force_unicode(strings_only=True).


def force_unicode(string, encoding='<STR_LIT:utf-8>', strings_only=False, errors='<STR_LIT:strict>'):

<EOL>if isinstance(string, str):<EOL><INDENT>return string<EOL><DEDENT>if strings_only and is_protected_type(string):<EOL><INDENT>return string<EOL><DEDENT>try:<EOL><INDENT>if not isinstance(string, str):<EOL><INDENT>if hasattr(string, '<STR_LIT>'):<EOL><INDENT>string = string.__unicode__()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>string = str(string, encoding, errors)<EOL><DEDENT>except UnicodeEncodeError:<EOL><INDENT>if not isinstance(string, Exception):<EOL><INDENT>raise<EOL><DEDENT>string = '<STR_LIT:U+0020>'.join([force_unicode(arg, encoding,<EOL>strings_only,<EOL>errors) for arg in string])<EOL><DEDENT><DEDENT><DEDENT>elif not isinstance(string, str):<EOL><INDENT>string = string.decode(encoding, errors)<EOL><DEDENT><DEDENT>except UnicodeDecodeError as ex:<EOL><INDENT>if not isinstance(string, Exception):<EOL><INDENT>raise DjangoUnicodeDecodeError(string, *ex.args)<EOL><DEDENT>else:<EOL><INDENT>string = '<STR_LIT:U+0020>'.join([force_unicode(arg, encoding, strings_only,<EOL>errors) for arg in string])<EOL><DEDENT><DEDENT>return string<EOL>

Similar to smart_unicode, except that lazy instances are resolved to
strings, rather than kept as lazy objects.

If strings_only is True, don't convert (some) non-string-like objects.


def smart_str(string, encoding='<STR_LIT:utf-8>', strings_only=False, errors='<STR_LIT:strict>'):

if strings_only and isinstance(string, (type(None), int)):<EOL><INDENT>return string<EOL><DEDENT>if isinstance(string, str):<EOL><INDENT>try:<EOL><INDENT>return string.encode(encoding, errors)<EOL><DEDENT>except UnicodeEncodeError:<EOL><INDENT>return string.encode('<STR_LIT:utf-8>', errors)<EOL><DEDENT><DEDENT>elif not isinstance(string, bytes):<EOL><INDENT>try:<EOL><INDENT>return str(string).encode(encoding, errors)<EOL><DEDENT>except UnicodeEncodeError:<EOL><INDENT>if isinstance(string, Exception):<EOL><INDENT>return '<STR_LIT:U+0020>'.join([smart_str(arg, encoding, strings_only,<EOL>errors) for arg in string])<EOL><DEDENT>return str(string).encode(encoding, errors)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return string<EOL><DEDENT>

Returns a bytestring version of 's', encoded as specified in 'encoding'.

If strings_only is True, don't convert (some) non-string-like objects.


return self._title<EOL>

str: Title extracted from the HTML source

            Note:
                Read only


return self._cleaned_text<EOL>

str: Cleaned text of the article without HTML tags; most commonly desired property

            Note:
                Read only


return self._meta_description<EOL>

str: Contents of the meta-description field from the HTML source

            Note:
                Read only


return self._meta_lang<EOL>

str: Contents of the meta-lang field from the HTML source

            Note:
                Read only


return self._meta_favicon<EOL>

str: Contents of the meta-favicon field from the HTML source

            Note:
                Read only


return self._meta_keywords<EOL>

str: Contents of the meta-keywords field from the HTML source

            Note:
                Read only


return self._meta_encoding<EOL>

str: Contents of the encoding/charset field from the HTML source

            Note:
                Read only


return self._canonical_link<EOL>

str: The canonical link of the article if found in the meta data

            Note:
                Read only


return self._domain<EOL>

str: Domain of the article parsed

            Note:
                Read only


return self._top_node<EOL>

etree: The top Element that is a candidate for the main body of the article

            Note:
                Read only


return self._top_image<EOL>

Image: The top image object that likely represents the article

            Returns:
                Image: See more information on the goose3.Image class
            Note:
                Read only


return self._tags<EOL>

list(str): List of article tags (non-metadata tags)

            Note:
                Read only


return self._opengraph<EOL>

dict: All opengraph tag data

            Note:
                Read only


return self._tweets<EOL>

list(str): A listing of embeded tweets in the article

            Note:
                Read only


return self._movies<EOL>

list(Video): A listing of all videos within the article such as
            YouTube or Vimeo

            Returns:
                list(Video):  See more information on the goose3.Video class
            Note:
                Read only


return self._links<EOL>

list(str): A listing of URL links within the article

            Note:
                Read only


return self._authors<EOL>

list(str): A listing of authors as parsed from the meta tags

            Note:
                Read only


return self._final_url<EOL>

str: The URL that was used to pull and parsed; `None` if raw_html was used
            and no url element was found.

            Note:
                Read only


return self._link_hash<EOL>

str: The MD5 of the final url to be used for various identification tasks

            Note:
                Read only


return self._raw_html<EOL>

str: The HTML represented as a string

            Note:
                Read only


return self._doc<EOL>

etree: lxml document that is being processed

            Note:
                Read only


return self._raw_doc<EOL>

etree: Original, uncleaned, and untouched lxml document to be processed

            Note:
                Read only


return self._schema<EOL>

dict: All schema tag data

            Note:
                Read only


return self._publish_date<EOL>

str: The date the article was published based on meta tag extraction

            Note:
                Read only


return self._publish_datetime_utc<EOL>

datetime.datetime: The date time version of the published date based on meta tag extraction \
            in the UTC timezone, if timezone information is known

            Note:
                Read only


return self._additional_data<EOL>

dict: A property bucket for consumers of goose3 to store custom data extractions

            Note:
                Read only


data = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:description>": self.meta_description,<EOL>"<STR_LIT>": self.meta_lang,<EOL>"<STR_LIT>": self.meta_keywords,<EOL>"<STR_LIT>": self.meta_favicon,<EOL>"<STR_LIT>": self.canonical_link,<EOL>"<STR_LIT>": self.meta_encoding<EOL>},<EOL>"<STR_LIT:image>": None,<EOL>"<STR_LIT>": self.domain,<EOL>"<STR_LIT:title>": self.title,<EOL>"<STR_LIT>": self.cleaned_text,<EOL>"<STR_LIT>": self.opengraph,<EOL>"<STR_LIT>": self.tags,<EOL>"<STR_LIT>": self.tweets,<EOL>"<STR_LIT>": [],<EOL>"<STR_LIT>": self.links,<EOL>"<STR_LIT>": self.authors,<EOL>"<STR_LIT>": self.publish_date<EOL>}<EOL>if self.top_image is not None:<EOL><INDENT>data['<STR_LIT:image>'] = {<EOL>'<STR_LIT:url>': self.top_image.src,<EOL>'<STR_LIT:width>': self.top_image.width,<EOL>'<STR_LIT>': self.top_image.height,<EOL>'<STR_LIT:type>': '<STR_LIT:image>'<EOL>}<EOL><DEDENT>for movie in self.movies:<EOL><INDENT>data['<STR_LIT>'].append({<EOL>'<STR_LIT>': movie.embed_type,<EOL>'<STR_LIT>': movie.provider,<EOL>'<STR_LIT:width>': movie.width,<EOL>'<STR_LIT>': movie.height,<EOL>'<STR_LIT>': movie.embed_code,<EOL>'<STR_LIT:src>': movie.src,<EOL>})<EOL><DEDENT>return data<EOL>

dict: The summation of all data available about the extracted article

            Note:
                Read only


return self._known_context_patterns<EOL>

list: The context patterns to search to find the likely article content

            Note:
                Each entry must be a dictionary with the following keys: `attr` and `value` \
                or just `tag`


def create_pat_from_dict(val):<EOL><INDENT>'''<STR_LIT>'''<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat = ArticleContextPattern(tag=val["<STR_LIT>"])<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat.attr = val["<STR_LIT>"]<EOL>pat.value = val["<STR_LIT:value>"]<EOL><DEDENT><DEDENT>elif "<STR_LIT>" in val:<EOL><INDENT>pat = ArticleContextPattern(attr=val["<STR_LIT>"], value=val["<STR_LIT:value>"])<EOL><DEDENT>if "<STR_LIT>" in val:<EOL><INDENT>pat.domain = val["<STR_LIT>"]<EOL><DEDENT>return pat<EOL><DEDENT>if isinstance(val, list):<EOL><INDENT>self._known_context_patterns = [<EOL>x if isinstance(x, ArticleContextPattern) else create_pat_from_dict(x)<EOL>for x in val<EOL>] + self.known_context_patterns<EOL><DEDENT>elif isinstance(val, ArticleContextPattern):<EOL><INDENT>self._known_context_patterns.insert(<NUM_LIT:0>, val)<EOL><DEDENT>elif isinstance(val, dict):<EOL><INDENT>self._known_context_patterns.insert(<NUM_LIT:0>, create_pat_from_dict(val))<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>".format(type(val)))<EOL><DEDENT>

val must be an ArticleContextPattern, a dictionary, or list of \
            dictionaries
            e.g., {'attr': 'class', 'value': 'my-article-class'}
                or [{'attr': 'class', 'value': 'my-article-class'},
                    {'attr': 'id', 'value': 'my-article-id'}]


return self._known_publish_date_tags<EOL>

list: The tags to search to find the likely published date

            Note:
                Each entry must be a dictionary with the following keys: `attribute`, `value`, \
                and `content`.


def create_pat_from_dict(val):<EOL><INDENT>'''<STR_LIT>'''<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat = PublishDatePattern(tag=val["<STR_LIT>"])<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat.attr = val["<STR_LIT>"]<EOL>pat.value = val["<STR_LIT:value>"]<EOL><DEDENT><DEDENT>elif "<STR_LIT>" in val:<EOL><INDENT>pat = PublishDatePattern(attr=val["<STR_LIT>"], value=val["<STR_LIT:value>"],<EOL>content=val["<STR_LIT:content>"])<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat.subcontent = val["<STR_LIT>"]<EOL><DEDENT><DEDENT>if "<STR_LIT>" in val:<EOL><INDENT>pat.domain = val["<STR_LIT>"]<EOL><DEDENT>return pat<EOL><DEDENT>if isinstance(val, list):<EOL><INDENT>self._known_publish_date_tags = [<EOL>x if isinstance(x, PublishDatePattern) else create_pat_from_dict(x)<EOL>for x in val<EOL>] + self.known_publish_date_tags<EOL><DEDENT>elif isinstance(val, PublishDatePattern):<EOL><INDENT>self._known_publish_date_tags.insert(<NUM_LIT:0>, val)<EOL><DEDENT>elif isinstance(val, dict):<EOL><INDENT>self._known_publish_date_tags.insert(<NUM_LIT:0>, create_pat_from_dict(val))<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>".format(type(val)))<EOL><DEDENT>

val must be a dictionary or list of dictionaries
            e.g., {'attrribute': 'name', 'value': 'my-pubdate', 'content': 'datetime'}
                or [{'attrribute': 'name', 'value': 'my-pubdate', 'content': 'datetime'},
                    {'attrribute': 'property', 'value': 'pub_time', 'content': 'content'}]


return self._known_author_patterns<EOL>


def create_pat_from_dict(val):<EOL><INDENT>'''<STR_LIT>'''<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat = AuthorPattern(tag=val["<STR_LIT>"])<EOL>if "<STR_LIT>" in val:<EOL><INDENT>pat.attr = val["<STR_LIT>"]<EOL>pat.value = val["<STR_LIT:value>"]<EOL><DEDENT><DEDENT>elif "<STR_LIT>" in val:<EOL><INDENT>pat = AuthorPattern(attr=val["<STR_LIT>"], value=val["<STR_LIT:value>"],<EOL>content=val["<STR_LIT:content>"])<EOL><DEDENT>if "<STR_LIT>" in val:<EOL><INDENT>pat.subpattern = create_pat_from_dict(val["<STR_LIT>"])<EOL><DEDENT>return pat<EOL><DEDENT>if isinstance(val, list):<EOL><INDENT>self._known_author_patterns = [<EOL>x if isinstance(x, AuthorPattern) else create_pat_from_dict(x)<EOL>for x in val<EOL>] + self.known_author_patterns<EOL><DEDENT>elif isinstance(val, AuthorPattern):<EOL><INDENT>self._known_author_patterns.insert(<NUM_LIT:0>, val)<EOL><DEDENT>elif isinstance(val, dict):<EOL><INDENT>self._known_author_patterns.insert(<NUM_LIT:0>, create_pat_from_dict(val))<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>".format(type(val)))<EOL><DEDENT>


return self._strict<EOL>

bool: Enable `strict mode` and throw exceptions instead of
            swallowing them.

            Note:
                Defaults to `True`


self._strict = bool(val)<EOL>


return self._http_timeout<EOL>

float: The time delay to pass to `requests` to wait for the response
            in seconds

            Note:
                Defaults to 30.0


self._http_timeout = float(val)<EOL>


return self._local_storage_path<EOL>

str: The local path to store temporary files

            Note:
                Defaults to the value of `os.path.join(tempfile.gettempdir(), 'goose')`


self._local_storage_path = val<EOL>


return self._debug<EOL>

bool: Turn on or off debugging

            Note:
                Defaults to `False`
            Warning:
                Debugging is currently not implemented


self._debug = bool(val)<EOL>


return self._parser_class<EOL>

str: The key of the parser to use

            Note:
                Defaults to `lxml`


self._parser_class = val<EOL>


return self._available_parsers<EOL>

list(str): A list of all possible parser values for the parser_class

            Note:
                Not settable


return self._http_auth<EOL>

tuple: Authentication class and information to pass to the requests
            library

            See Also:
                `Requests Authentication <http://docs.python-requests.org/en/master/user/authentication/>`__


self._http_auth = val<EOL>


return self._http_proxies<EOL>

dict: Proxy information to pass directly to the supporting `requests` object

            See Also:
                `Requests Proxy Support <http://docs.python-requests.org/en/master/user/advanced/#proxies>`__


self._http_proxies = val<EOL>


return self._http_headers<EOL>

dict: Custom headers to pass directly to the supporting `requests` object

            See Also:
                `Requests Custom Headers <http://docs.python-requests.org/en/master/user/quickstart/#custom-headers>`__


self._http_headers = val<EOL>


return self._browser_user_agent<EOL>

Browser user agent string to use when making URL requests

            Note:
                Defaults to `Goose/{goose3.__version__}`

            Examples:
                Using the non-standard browser agent string is advised when pulling
                frequently

                >>> config.browser_user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2)'
                >>> config.browser_user_agent = 'AppleWebKit/534.52.7 (KHTML, like Gecko)'
                >>> config.browser_user_agent = 'Version/5.1.2 Safari/534.52.7'


self._browser_user_agent = val<EOL>


return self._imagemagick_identify_path<EOL>

str: Path to the identify program that is part of imagemagick

            Note:
                Defaults to `"/opt/local/bin/identify"`
            Warning:
                Currently not used / implemented


self._imagemagick_identify_path = val<EOL>

set the imagemagick identify program path


return self._imagemagick_convert_path<EOL>

str: Path to the convert program that is part of imagemagick

            Note:
                Defaults to `"/opt/local/bin/convert"`
            Warning:
                Currently not used / implemented


self._imagemagick_convert_path = val<EOL>

set the imagemagick convert program path


return self._stopwords_class<EOL>

StopWords: The StopWords class to use when analyzing article content

            Note:
                Defaults to the english stop words
            Note:
                Current stop words available in `goose3.text` include: \n
                `StopWords`, `StopWordsChinese`, `StopWordsArabic`, and `StopWordsKorean`


<EOL>self._stopwords_class = val<EOL>


return self._target_language<EOL>

str: The default target language if the language is not extractable
            or if use_meta_language is set to False

            Note:
                Default language is 'en'


self._target_language = val<EOL>


return self._use_meta_language<EOL>

bool: Determine if language should be extracted from the meta tags
            or not. If this is set to `False` then the target_language will be
            used. Also, if extraction fails then the target_language will be
            utilized.

            Note:
                Defaults to `True`


self._use_meta_language = bool(val)<EOL>


return self._enable_image_fetching<EOL>

bool: Turn on or off image extraction

            Note:
                Defaults to `False`


self._enable_image_fetching = bool(val)<EOL>


return self._images_min_bytes<EOL>

int: Minimum number of bytes for an image to be evaluated to be the
            main image of the site

            Note:
                Defaults to 4500 bytes


self._images_min_bytes = int(val)<EOL>


return self._pretty_lists<EOL>

bool: Specify if lists should be pretty printed in the cleaned_text
            output

            Note:
                Defaults to `True`


self._pretty_lists = bool(val)<EOL>


self._parse_lists = bool(val)<EOL>


return self._parse_headers<EOL>

bool: Specify if headers should be pulled or not in the cleaned_text
            output

            Note:
                Defaults to `True`


self._parse_headers = bool(val)<EOL>


return AVAILABLE_PARSERS[self.parser_class]<EOL>

Retrieve the current parser class to use for extraction

            Returns:
                Parser: The parser to use


return self<EOL>


def __exit__(self, exc_type, exc_val, exc_tb):

self.close()<EOL>

Define what to do when the context manager exits


if self.fetcher is not None:<EOL><INDENT>self.shutdown_network()<EOL><DEDENT>self.finalizer.atexit = False<EOL>

Close the network connection and perform any other required cleanup

            Note:
                Auto closed when using goose as a context manager or when garbage collected


def extract(self, url=None, raw_html=None):

crawl_candidate = CrawlCandidate(self.config, url, raw_html)<EOL>return self.__crawl(crawl_candidate)<EOL>

Extract the most likely article content from the html page

            Args:
                url (str): URL to pull and parse
                raw_html (str): String representation of the HTML page
            Returns:
                Article: Representation of the article contents \
                including other parsed and extracted metadata


self.fetcher.close()<EOL>self.fetcher = None<EOL>

Close the network connection

            Note:
                Auto closed when using goose as a context manager or when garbage collected


def crawler_wrapper(parser, parsers_lst, crawl_candidate):<EOL><INDENT>try:<EOL><INDENT>crawler = Crawler(self.config, self.fetcher)<EOL>article = crawler.crawl(crawl_candidate)<EOL><DEDENT>except (UnicodeDecodeError, ValueError) as ex:<EOL><INDENT>if parsers_lst:<EOL><INDENT>parser = parsers_lst.pop(<NUM_LIT:0>)  <EOL>return crawler_wrapper(parser, parsers_lst, crawl_candidate)<EOL><DEDENT>else:<EOL><INDENT>raise ex<EOL><DEDENT><DEDENT>return article<EOL><DEDENT>parsers = list(self.config.available_parsers)<EOL>parsers.remove(self.config.parser_class)<EOL>return crawler_wrapper(self.config.parser_class, parsers, crawl_candidate)<EOL>


return self._embed_type<EOL>

str: The type of embeding such as embed, object, or iframe

            Note:
                Read only


return self._provider<EOL>

str: The video provider

            Note:
                Read only


return self._width<EOL>

int: The video width in pixels

            Note:
                Read only


return self._height<EOL>

int: The video height in pixels

            Note:
                Read only


return self._embed_code<EOL>

str: The embed code of the video

            Note:
                Read only


return self._src<EOL>

str: The URL source of the video

            Note:
                Read only


with open(filepath, '<STR_LIT:r>') as filepointer:<EOL><INDENT>res = filepointer.read()<EOL><DEDENT>return res<EOL>


cur_dir = path.abspath(path.dirname(path.dirname(__file__)))<EOL>return cur_dir<EOL>


if type(request) == dict:<EOL><INDENT>return request<EOL><DEDENT>params = get_payload(request)<EOL>try:<EOL><INDENT>del params['<STR_LIT>']<EOL>params[params.pop('<STR_LIT:name>')] = params.pop('<STR_LIT:value>')<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>return {<EOL>k.rstrip('<STR_LIT>'): safe_eval(v) if not type(v) == list else [safe_eval(sv) for sv in v]<EOL>for k, v in params.items()<EOL>}<EOL>

Converts the request parameters to Python.

:param request: <pyramid.request.Request> || <dict>

:return: <dict>


<EOL>param_values = get_param_values(request, model=model)<EOL>context = param_values.pop('<STR_LIT>', {})<EOL>if isinstance(context, (unicode, str)):<EOL><INDENT>context = projex.rest.unjsonify(context)<EOL><DEDENT>has_limit = '<STR_LIT>' in context or '<STR_LIT>' in param_values<EOL>orb_context = orb.Context(**context)<EOL>used = set()<EOL>query_context = {}<EOL>for key in orb.Context.Defaults:<EOL><INDENT>if key in param_values:<EOL><INDENT>used.add(key)<EOL>query_context[key] = param_values.get(key)<EOL><DEDENT><DEDENT>schema_values = {}<EOL>if model:<EOL><INDENT>for key, value in request.matchdict.items():<EOL><INDENT>if model.schema().column(key, raise_=False):<EOL><INDENT>schema_values[key] = value<EOL><DEDENT><DEDENT>for key, value in param_values.items():<EOL><INDENT>root_key = key.split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>schema_object = model.schema().column(root_key, raise_=False) or model.schema().collector(root_key)<EOL>if schema_object:<EOL><INDENT>value = param_values.pop(key)<EOL>if isinstance(schema_object, orb.Collector) and type(value) not in (tuple, list):<EOL><INDENT>value = [value]<EOL><DEDENT>schema_values[key] = value<EOL><DEDENT><DEDENT><DEDENT>query_context['<STR_LIT>'] = {<EOL>'<STR_LIT>': request<EOL>}<EOL>try:<EOL><INDENT>default_context = request.orb_default_context<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>query_context['<STR_LIT>'].update(request.orb_scope)<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if '<STR_LIT>' in default_context:<EOL><INDENT>query_context['<STR_LIT>'].update(default_context.pop('<STR_LIT>'))<EOL><DEDENT>for k, v in default_context.items():<EOL><INDENT>query_context.setdefault(k, v)<EOL><DEDENT><DEDENT>orb_context.update(query_context)<EOL>return schema_values, orb_context<EOL>

Extracts ORB context information from the request.

:param request: <pyramid.request.Request>
:param model: <orb.Model> || None

:return: {<str> key: <variant> value} values, <orb.Context>


def register(self, service, name='<STR_LIT>'):

try:<EOL><INDENT>is_model = issubclass(service, orb.Model)<EOL><DEDENT>except StandardError:<EOL><INDENT>is_model = False<EOL><DEDENT>if is_model:<EOL><INDENT>self.services[service.schema().dbname()] = (ModelService, service)<EOL><DEDENT>else:<EOL><INDENT>super(OrbApiFactory, self).register(service, name=name)<EOL><DEDENT>


def make_postcard(self, npix=<NUM_LIT>, shape=(<NUM_LIT>, <NUM_LIT>), buffer_size=<NUM_LIT:15>):

source = self.kic<EOL>client = kplr.API()<EOL>targ = client.target(source)<EOL>channel = [targ.params['<STR_LIT>'], targ.params['<STR_LIT>'], <EOL>targ.params['<STR_LIT>'], targ.params['<STR_LIT>']]<EOL>col = [targ.params['<STR_LIT>'], targ.params['<STR_LIT>'], <EOL>targ.params['<STR_LIT>'], targ.params['<STR_LIT>']] <EOL>row = [targ.params['<STR_LIT>'], targ.params['<STR_LIT>'], <EOL>targ.params['<STR_LIT>'], targ.params['<STR_LIT>']] <EOL>if None in row:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if None in col:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>center = np.array([npix/<NUM_LIT:2>, npix/<NUM_LIT:2>])<EOL>if (np.min(col) < npix/<NUM_LIT:2>):<EOL><INDENT>jump = npix/<NUM_LIT:2> - np.min(col) + buffer_size<EOL>col += jump<EOL>center[<NUM_LIT:1>] -= jump<EOL><DEDENT>if (np.min(row) < npix/<NUM_LIT:2>):<EOL><INDENT>jump = npix/<NUM_LIT:2> - np.min(row) + buffer_size<EOL>row += jump<EOL>center[<NUM_LIT:0>] -= jump<EOL><DEDENT>if (np.max(row) > shape[<NUM_LIT:0>] - npix/<NUM_LIT:2>):<EOL><INDENT>jump = shape[<NUM_LIT:0>]-npix/<NUM_LIT:2> - np.max(row) - buffer_size<EOL>row += jump<EOL>center[<NUM_LIT:0>] -= jump<EOL><DEDENT>if (np.max(col) > shape[<NUM_LIT:1>] - npix/<NUM_LIT:2>):<EOL><INDENT>jump = shape[<NUM_LIT:1>]-npix/<NUM_LIT:2> - np.max(col) - buffer_size<EOL>col += jump<EOL>center[<NUM_LIT:1>] -= jump<EOL><DEDENT>fin_arr = np.zeros((len(self.times), npix, npix))<EOL>for icount, iname in enumerate(self.obs_filenames): <EOL><INDENT>a = fits.open(self.ffi_dir+iname)<EOL>quarter = a[<NUM_LIT:0>].header['<STR_LIT>']<EOL>if int(quarter) == <NUM_LIT:0>:<EOL><INDENT>season = <NUM_LIT:3><EOL><DEDENT>else:<EOL><INDENT>season = (int(quarter) - <NUM_LIT:2>) % <NUM_LIT:4><EOL><DEDENT>img = a[channel[season]].data<EOL>img -= np.median(img)<EOL>ymin = int(max([int(row[season])-npix/<NUM_LIT:2>,<NUM_LIT:0>]))<EOL>ymax = int(min([int(row[season])+npix/<NUM_LIT:2>,img.shape[<NUM_LIT:0>]]))<EOL>xmin = int(max([int(col[season])-npix/<NUM_LIT:2>,<NUM_LIT:0>]))<EOL>xmax = int(min([int(col[season])+npix/<NUM_LIT:2>,img.shape[<NUM_LIT:1>]]))<EOL>pimg = img[ymin:ymax,xmin:xmax]<EOL>fin_arr[icount,:,:] = pimg<EOL><DEDENT>self.postcard = fin_arr<EOL>self.integrated_postcard = np.sum(self.postcard, axis=<NUM_LIT:0>)<EOL>self.center = center<EOL>

Develop a "postcard" region around the target star.
Other stars in this postcard will be used as possible reference stars.

Args:
    npix: The size of the postcard region. The region will be a square with sides npix pixels
        (default: ``300``)
    shape: The size of each individual image. For Kepler/K2 FFIs this should never need to be
        changed from the default, but will be different for e.g. TESS FFIs (default: ``(1070, 1132)``)
    buffer_size: The number of pixels at the edge of the detector to avoid (default: ``15``)


def find_other_sources(self,  edge_lim = <NUM_LIT>, min_val = <NUM_LIT>, <EOL>ntargets = <NUM_LIT>, extend_region_size=<NUM_LIT:3>, remove_excess=<NUM_LIT:4>,<EOL>plot_flag = False, plot_window=<NUM_LIT:15>):

j,i = self.center<EOL>region = self.integrated_postcard + <NUM_LIT:0.0><EOL>if plot_flag == True:<EOL><INDENT>ff = plt.imshow(self.integrated_postcard, interpolation='<STR_LIT>', cmap='<STR_LIT>', <EOL>vmax = np.percentile(region, <NUM_LIT>))<EOL>plt.colorbar(ff)<EOL>plt.show()<EOL><DEDENT>targets = np.zeros_like(self.integrated_postcard)<EOL>sizeimg = np.shape(targets)[<NUM_LIT:0>]<EOL>jj = j + <NUM_LIT:0><EOL>ii = i + <NUM_LIT:0><EOL>edge = edge_lim<EOL>lim = max(min_val, self.integrated_postcard[int(j), int(i)]*edge)<EOL>maxpt = np.percentile(self.integrated_postcard, <NUM_LIT>)<EOL>bin_img = (region > lim)<EOL>lab_img, n_features = label(bin_img)<EOL>key_targ = (lab_img == (lab_img[int(j), int(i)]))<EOL>tot = np.sum(key_targ)<EOL>targets[key_targ] = <NUM_LIT:1><EOL>region[key_targ] = <NUM_LIT:0.0><EOL>lim = np.zeros(ntargets)<EOL>for peaks in range(<NUM_LIT:1>,ntargets):<EOL><INDENT>k = np.argmax(region)<EOL>j,i = np.unravel_index(k, region.shape)<EOL>lim[peaks] = max(maxpt, edge*region[j,i])<EOL>bin_img = (region >= lim[peaks])<EOL>lab_img, n_features = label(bin_img)<EOL>key_targ = (lab_img == (lab_img[j,i]))<EOL>targets[key_targ] = peaks + <NUM_LIT:1><EOL>region[key_targ] = <NUM_LIT:0.0><EOL><DEDENT>lab_img, n_features = label(targets)<EOL>for i in range(<NUM_LIT:1>, ntargets+<NUM_LIT:1>):<EOL><INDENT>for j in range(extend_region_size):<EOL><INDENT>border= mh.labeled.border(targets, <NUM_LIT:0>, i)<EOL>targets[border*(region < (<NUM_LIT:10>)*lim[peaks])] = i<EOL><DEDENT><DEDENT>for i in range(<NUM_LIT:2>, ntargets+<NUM_LIT:1>):<EOL><INDENT>for j in range(<NUM_LIT:2>, ntargets+<NUM_LIT:1>):<EOL><INDENT>if i != j:<EOL><INDENT>border = mh.labeled.border(targets, i, j)<EOL>if np.sum(border) != <NUM_LIT:0>:<EOL><INDENT>targets[targets == j] = i<EOL><DEDENT><DEDENT><DEDENT><DEDENT>targets = mh.labeled.remove_bordering(targets)<EOL>for k in range(remove_excess):<EOL><INDENT>for i in range(ntargets):<EOL><INDENT>if np.sum(self.integrated_postcard[targets == i]) < <NUM_LIT>:<EOL><INDENT>targets[targets > i] -= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>self.targets = targets<EOL>if plot_flag == True:<EOL><INDENT>plt.imshow(self.targets, interpolation='<STR_LIT>')<EOL>plt.show()<EOL>plt.imshow(((targets == <NUM_LIT:1>)*self.integrated_postcard + (targets == <NUM_LIT:1>)*<NUM_LIT>)<EOL>[jj-plot_window:jj+plot_window,ii-plot_window:ii+plot_window], <EOL>interpolation='<STR_LIT>', cmap='<STR_LIT>', vmax=np.percentile(self.integrated_postcard, <NUM_LIT>))<EOL>plt.show()<EOL>plt.imshow((np.ceil(targets/<NUM_LIT>)*self.integrated_postcard+np.ceil(targets/<NUM_LIT>)*<NUM_LIT>), <EOL>interpolation='<STR_LIT>', cmap='<STR_LIT>', vmax=np.percentile(self.integrated_postcard, <NUM_LIT>))<EOL>plt.show()<EOL><DEDENT>

Identify apertures for all sources on the postcard, both for the
target and potential reference stars

Args:
    edge_lim: The initial limit for the creation of apertures. The aperture will be a region of
        contiguous pixels with flux values larger than the product of ``edge_lim`` and the brightest
        pixel value for this star, as long as that product is larger than ``min_val`` (default: ``0.015``)
    min_val: Threshold for the minimum flux value in the ``integrated_postcard`` for a pixel to be included
        in the default apertures (default: ``5000``)
    ntargets: The maximum number of potential reference stars to be included in the analysis (default: ``250``)
    extend_region_size: After the initial apertures are generated, they will be optionally extended an
        additional number of pixels following this flag. Safe practice for reasonable apertures is to
        leave ``min_val`` at a value well above the noise and then extend apertures via this flag until
        they are of suitable size (default: ``3``)
    remove_excess: Stars with apertures that touch will be combined into a single aperture.
        This is done by iterating through the starlist; this flag represents the number of times the
        list will be iterated through to delete redundant apertures (default: ``4``)
    plot_flag: If true, a series of diagnostic plots will appear while this function runs to observe
        apertures for the target star and other stars.
        (default: ``False``)
    plot_window: If ``plot_flag`` is ``True``, the size of the region to be plotted around the target star
        to show the drawn aperture for visualization purposes only (default: ``15``)


std_f = np.zeros(<NUM_LIT:4>)<EOL>data_save = np.zeros_like(self.postcard)<EOL>self.obs_flux = np.zeros_like(self.reference_flux)<EOL>for i in range(<NUM_LIT:4>):<EOL><INDENT>g = np.where(self.qs == i)[<NUM_LIT:0>]<EOL>wh = np.where(self.times[g] > <NUM_LIT>)<EOL>data_save[g] = np.roll(self.postcard[g], int(self.roll_best[i,<NUM_LIT:0>]), axis=<NUM_LIT:1>)<EOL>data_save[g] = np.roll(data_save[g], int(self.roll_best[i,<NUM_LIT:1>]), axis=<NUM_LIT:2>)<EOL>self.target_flux_pixels = data_save[:,self.targets == <NUM_LIT:1>]<EOL>self.target_flux = np.sum(self.target_flux_pixels, axis=<NUM_LIT:1>)<EOL>self.obs_flux[g] = self.target_flux[g] / self.reference_flux[g]<EOL>self.obs_flux[g] /= np.median(self.obs_flux[g[wh]])<EOL>fitline = np.polyfit(self.times[g][wh], self.obs_flux[g][wh], <NUM_LIT:1>)<EOL>std_f[i] = np.max([np.std(self.obs_flux[g][wh]/(fitline[<NUM_LIT:0>]*self.times[g][wh]+fitline[<NUM_LIT:1>])), <NUM_LIT>])<EOL><DEDENT>self.flux_uncert = std_f<EOL>

Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data
in each orientation. This function is called by other functions and generally the user will not need
to interact with it directly.


plt.figure(figsize=(<NUM_LIT>,<NUM_LIT:6>))<EOL>ax = plt.gca()<EOL>fig = plt.gcf()<EOL>plt.subplot(<NUM_LIT>)<EOL>data_save = np.zeros_like(self.postcard)<EOL>self.roll_best = np.zeros((<NUM_LIT:4>,<NUM_LIT:2>))<EOL>for i in range(<NUM_LIT:4>):<EOL><INDENT>g = np.where(self.qs == i)[<NUM_LIT:0>]<EOL>wh = np.where(self.times[g] > <NUM_LIT>)<EOL>self.roll_best[i] = self.do_rolltest(g, wh)<EOL><DEDENT>self.do_photometry()<EOL>for i in range(<NUM_LIT:4>):<EOL><INDENT>g = np.where(self.qs == i)[<NUM_LIT:0>]<EOL>plt.errorbar(self.times[g], self.obs_flux[g], yerr=self.flux_uncert[i], fmt=fmt[i])<EOL><DEDENT>plt.xlabel('<STR_LIT>', fontsize=<NUM_LIT:20>)<EOL>plt.ylabel('<STR_LIT>', fontsize=<NUM_LIT:20>)<EOL>plt.subplot(<NUM_LIT>)<EOL>implot = plt.imshow(img, interpolation='<STR_LIT>', cmap='<STR_LIT>', vmin=<NUM_LIT>*<NUM_LIT>, vmax=<NUM_LIT>*<NUM_LIT>)<EOL>cid = fig.canvas.mpl_connect('<STR_LIT>', self.onclick)<EOL>plt.show(block=True)<EOL>

Creates the figure shown in ``adjust_aperture`` for visualization purposes. Called by other functions
and generally not called by the user directly.

Args:
    img: The data frame to be passed through to be plotted. A cutout of the ``integrated_postcard``


def adjust_aperture(self, image_region=<NUM_LIT:15>, ignore_bright=<NUM_LIT:0>):

self.ignore_bright = ignore_bright<EOL>self.calc_fluxes()<EOL>self.coordsx = []<EOL>self.coordsy = []<EOL>jj, ii = self.center<EOL>jj, ii = int(jj), int(ii)  <EOL>plt.ion()<EOL>img = np.sum(((self.targets == <NUM_LIT:1>)*self.postcard + (self.targets == <NUM_LIT:1>)*<NUM_LIT>)<EOL>[:,jj-image_region:jj+image_region,ii-image_region:ii+image_region], axis=<NUM_LIT:0>)<EOL>self.generate_panel(img)<EOL>while len(self.coordsx) != <NUM_LIT:0>:<EOL><INDENT>for i in range(len(self.coordsx)):<EOL><INDENT>if self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] != <NUM_LIT:1>:<EOL><INDENT>self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] = <NUM_LIT:1><EOL><DEDENT>elif self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] == <NUM_LIT:1>:<EOL><INDENT>self.targets[self.coordsy[i]+jj-image_region,self.coordsx[i]+ii-image_region] = <NUM_LIT:0><EOL><DEDENT>if self.coordsy[i] == <NUM_LIT:0>:<EOL><INDENT>thiscol = np.where(self.targets[:,self.coordsx[i]+ii-image_region] == <NUM_LIT:1>)<EOL>self.targets[thiscol,self.coordsx[i]+ii-image_region] = <NUM_LIT:0><EOL><DEDENT>if self.coordsx[i] == <NUM_LIT:0>:<EOL><INDENT>thiscol = np.where(self.targets[self.coordsy[i]+jj-image_region,:] == <NUM_LIT:1>)<EOL>self.targets[self.coordsy[i]+jj-image_region, thiscol] = <NUM_LIT:0><EOL><DEDENT><DEDENT>self.coordsx = []<EOL>self.coordsy = []<EOL>img = np.sum(((self.targets == <NUM_LIT:1>)*self.postcard + <EOL>(self.targets == <NUM_LIT:1>)*<NUM_LIT>)[:,jj-image_region:jj+image_region,ii-image_region:ii+image_region],<EOL>axis=<NUM_LIT:0>)<EOL>self.generate_panel(img)<EOL><DEDENT>

Develop a panel showing the current aperture and the light curve as judged from that aperture.
Clicking on individual pixels on the aperture will toggle those pixels on or off into the
aperture (which will be updated after closing the plot).
Clicking on the 0th row or column will turn off all pixels in that column or row, respectively.
Will iterate continuously until the figure is closed without updating any pixels.

Args:
    image_region: The size of the region around the target star to be plotted. Images will be a square
        with side length ``image_region`` (default: ``15``)
    ignore_bright: The number of brightest stars to be ignored in the determination of the flux from
        reference stars. If there is reason to believe (for example) that saturated stars may behave
        differently than the target star, they can be avoided with this flag (default: ``0``)


def data_for_target(self, do_roll=True, ignore_bright=<NUM_LIT:0>):

self.ignore_bright = ignore_bright<EOL>self.calc_fluxes()<EOL>self.roll_best = np.zeros((<NUM_LIT:4>,<NUM_LIT:2>))<EOL>if do_roll == True:<EOL><INDENT>for i in range(<NUM_LIT:4>):<EOL><INDENT>g = np.where(self.qs == i)[<NUM_LIT:0>]<EOL>wh = np.where(self.times[g] > <NUM_LIT>)<EOL>self.roll_best[i] = self.do_rolltest(g, wh)<EOL><DEDENT><DEDENT>self.do_photometry()<EOL>

Determine the normalized photometry, accounting for effects shared by reference stars. Does not provide
the opportunity to adjust the aperture

Args:
    image_region: If ``True`` allow the aperture to be shifted up to one pixel in both the x and y
        directions to account for differential velocity aberration (default: ``True``)
    ignore_bright: The number of brightest stars to be ignored in the determination of the flux from
        reference stars. If there is reason to believe (for example) that saturated stars may behave
        differently than the target star, they can be avoided with this flag (default: ``0``)


def calc_fluxes(self, min_flux = <NUM_LIT>, outlier_iterations=<NUM_LIT:5>,<EOL>max_outlier_obs=<NUM_LIT:4>, outlier_limit=<NUM_LIT>):

jj, ii = self.center<EOL>numer = np.zeros(len(self.times))<EOL>denom = np.zeros(len(self.times))<EOL>factr = np.zeros(len(self.times))<EOL>numer_pix = self.postcard[:,self.targets == <NUM_LIT:1>]<EOL>numer = np.sum(numer_pix, axis=<NUM_LIT:1>)<EOL>tar_vals = np.zeros((len(self.times), int(np.max(self.targets)+<NUM_LIT:1>-<NUM_LIT:2>-self.ignore_bright)))<EOL>for i in range(<NUM_LIT:2>+self.ignore_bright,int(np.max(self.targets)+<NUM_LIT:1>)):<EOL><INDENT>tval = np.sum(self.postcard[:,self.targets == i], axis=<NUM_LIT:1>)<EOL>tar_vals[:,i-<NUM_LIT:2>-self.ignore_bright] = tval <EOL><DEDENT>for i in range(len(self.obs_filenames)):<EOL><INDENT>if np.max(tar_vals[i]) < min_flux:<EOL><INDENT>tar_vals[self.qs == self.qs[i]] = <NUM_LIT:0.0><EOL><DEDENT><DEDENT>all_tar = np.zeros((len(self.times), int(np.max(self.targets)-self.ignore_bright)))<EOL>all_tar[:,<NUM_LIT:0>] = numer<EOL>all_tar[:,<NUM_LIT:1>:] = tar_vals<EOL>self.photometry_array = all_tar<EOL>for i in range(len(tar_vals[<NUM_LIT:0>])):<EOL><INDENT>for j in range(<NUM_LIT:4>):<EOL><INDENT>g = np.where(self.qs == j)[<NUM_LIT:0>]  <EOL>tar_vals[g,i] /= (np.median(tar_vals[g,i])+<NUM_LIT>)<EOL><DEDENT><DEDENT>tar_vals_old = tar_vals + <NUM_LIT:0.0><EOL>for i in range(outlier_iterations):<EOL><INDENT>nonzeros = np.where(tar_vals[<NUM_LIT:0>,:] != <NUM_LIT:0>)[<NUM_LIT:0>]<EOL>med = np.median(tar_vals[:,nonzeros], axis=<NUM_LIT:1>)<EOL>std = np.std(tar_vals[:,nonzeros], axis=<NUM_LIT:1>)<EOL>if np.sum(tar_vals) != <NUM_LIT:0.0>:<EOL><INDENT>tar_vals_old = tar_vals + <NUM_LIT:0.0><EOL><DEDENT>for k in range(len(tar_vals[<NUM_LIT:0>])):<EOL><INDENT>h = np.where((np.abs(med-tar_vals[:,k])/std) > outlier_limit)[<NUM_LIT:0>]<EOL>if len(h) >= max_outlier_obs:<EOL><INDENT>tar_vals[:,k] = <NUM_LIT:0><EOL><DEDENT><DEDENT><DEDENT>if np.sum(tar_vals) == <NUM_LIT:0.0>:<EOL><INDENT>tar_vals = tar_vals_old + <NUM_LIT:0.0><EOL><DEDENT>denom = np.sum(tar_vals, axis=<NUM_LIT:1>)<EOL>self.target_flux_pixels = numer_pix<EOL>self.reference_flux = denom<EOL>

Determine the suitable reference stars, and then the total flux in those stars and
in the target star in each epoch

Args:
    min_flux: The size of the region around the target star to be plotted. Images will be a square
        with side length ``image_region`` (default: ``5000``)
    outlier_iterations: The number of iterations to remove outliers from the reference star sample
        (stars at epochs with more than ``max_outlier_obs`` observations more than ``outlier_limit`` standard
        deviations from the median value for all stars after normalization) (default: ``5``)
    max_outlier_obs: The maximum number of epochs at which a star is allowed to be more than ``outlier_limit``
        standard deviations from the median value for all stars before it is removed as a suitable
        reference star (default: ``4``)
    outlier_limit: The level of deviation (measured in standard deviations) which a target is allowed
        to be discrepant from the median. If it is this discrepant at more than ``max_outlier_obs``
        epochs, it is removed from consideration (default: ``1.7``)


self.cm = np.zeros((len(self.postcard), <NUM_LIT:2>))<EOL>for i in range(len(self.postcard)):<EOL><INDENT>target = self.postcard[i]<EOL>target[self.targets != <NUM_LIT:1>] = <NUM_LIT:0.0><EOL>self.cm[i] = center_of_mass(target)<EOL><DEDENT>

Identify the centroid positions for the target star at all epochs. Useful for verifying that there is
no correlation between flux and position, as might be expected for high proper motion stars.


client = kplr.API()<EOL>star = client.star(self.kic)<EOL>lcs = star.get_light_curves(short_cadence=False)<EOL>time, flux, ferr, qual = [], [], [], []<EOL>for lc in lcs:<EOL><INDENT>with lc.open() as f:<EOL><INDENT>hdu_data = f[<NUM_LIT:1>].data<EOL>time.append(hdu_data["<STR_LIT:time>"])<EOL>flux.append(hdu_data["<STR_LIT>"])<EOL>ferr.append(hdu_data["<STR_LIT>"])<EOL>qual.append(hdu_data["<STR_LIT>"])<EOL><DEDENT>tout = np.array([])<EOL>fout = np.array([])<EOL>eout = np.array([])<EOL>for i in range(len(flux)):<EOL><INDENT>t = time[i][qual[i] == <NUM_LIT:0>]<EOL>f = flux[i][qual[i] == <NUM_LIT:0>]<EOL>e = ferr[i][qual[i] == <NUM_LIT:0>]<EOL>t = t[np.isfinite(f)]<EOL>e = e[np.isfinite(f)]<EOL>f = f[np.isfinite(f)]<EOL>e /= np.median(f)<EOL>f /= np.median(f)<EOL>tout = np.append(tout, t[<NUM_LIT:50>:]+<NUM_LIT>)<EOL>fout = np.append(fout, f[<NUM_LIT:50>:])<EOL>eout = np.append(eout, e[<NUM_LIT:50>:])<EOL><DEDENT>self.spot_signal = np.zeros(<NUM_LIT>)<EOL>for i in range(len(self.times)):<EOL><INDENT>if self.times[i] < <NUM_LIT>:<EOL><INDENT>self.spot_signal[i] = <NUM_LIT:1.0><EOL><DEDENT>else:<EOL><INDENT>self.spot_signal[i] = fout[np.abs(self.times[i] - tout) == np.min(np.abs(self.times[i] - tout))]<EOL><DEDENT><DEDENT><DEDENT>

Identify the "expected" flux value at the time of each observation based on the
Kepler long-cadence data, to ensure variations observed are not the effects of a single
large starspot. Only works if the target star was targeted for long or short cadence
observations during the primary mission.


Y = self.photometry_array.T<EOL>Y /= np.median(Y, axis=<NUM_LIT:1>)[:, None]<EOL>C = np.median(Y, axis=<NUM_LIT:0>)<EOL>nstars, nobs = np.shape(Y)<EOL>Z = np.empty((nstars, <NUM_LIT:4>))<EOL>qs = self.qs.astype(int)<EOL>for s in range(<NUM_LIT:4>):<EOL><INDENT>Z[:, s] = np.median((Y / C)[:, qs == s], axis=<NUM_LIT:1>)<EOL><DEDENT>resid2 = (Y - Z[:, qs] * C)**<NUM_LIT:2><EOL>z = Z[:, qs]<EOL>trend = z * C[None, :]<EOL>lnS = np.log(np.nanmedian(resid2, axis=<NUM_LIT:0>))<EOL>jitter = np.log(<NUM_LIT:0.1>*np.nanmedian(np.abs(np.diff(Y, axis=<NUM_LIT:1>))))<EOL>cal_ferr = np.sqrt(np.exp(<NUM_LIT:2>*(jitter/trend))+z**<NUM_LIT:2>*np.exp(lnS)[None, :])<EOL>self.modeled_uncert = cal_ferr<EOL>self.target_uncert = cal_ferr[<NUM_LIT:0>]<EOL>

Estimate the photometric uncertainties on each data point following Equation A.2 of The Paper.
Based on the kepcal package of Dan Foreman-Mackey.


def multinest(parameter_names, transform, loglikelihood, output_basename, **problem):

import numpy<EOL>from numpy import log, exp<EOL>import pymultinest<EOL>parameters = parameter_names<EOL>n_params = len(parameters)<EOL>def myprior(cube, ndim, nparams):<EOL><INDENT>params = transform([cube[i] for i in range(ndim)])<EOL>for i in range(ndim):<EOL><INDENT>cube[i] = params[i]<EOL><DEDENT><DEDENT>def myloglike(cube, ndim, nparams):<EOL><INDENT>l = loglikelihood([cube[i] for i in range(ndim)])<EOL>return l<EOL><DEDENT>mn_args = dict(<EOL>importance_nested_sampling = False, <EOL>outputfiles_basename = output_basename,<EOL>resume = problem.get('<STR_LIT>', False), <EOL>verbose = True,<EOL>n_live_points = problem.get('<STR_LIT>', <NUM_LIT>),<EOL>const_efficiency_mode = False)<EOL>if '<STR_LIT>' in problem:<EOL><INDENT>mn_args['<STR_LIT>'] = problem['<STR_LIT>']<EOL><DEDENT>pymultinest.run(myloglike, myprior, n_params, **mn_args)<EOL>import json<EOL>with file('<STR_LIT>' % output_basename, '<STR_LIT:w>') as f:<EOL><INDENT>json.dump(parameters, f, indent=<NUM_LIT:2>)<EOL><DEDENT>a = pymultinest.Analyzer(n_params = n_params, <EOL>outputfiles_basename = output_basename)<EOL>s = a.get_stats()<EOL>with open('<STR_LIT>' % a.outputfiles_basename, mode='<STR_LIT:w>') as f:<EOL><INDENT>json.dump(s, f, indent=<NUM_LIT:2>)<EOL><DEDENT>chain = a.get_equal_weighted_posterior()[:,:-<NUM_LIT:1>]<EOL>lower = [m['<STR_LIT>'][<NUM_LIT:0>] for m in s['<STR_LIT>']]<EOL>upper = [m['<STR_LIT>'][<NUM_LIT:1>] for m in s['<STR_LIT>']]<EOL>stdev = (numpy.array(upper) - numpy.array(lower)) / <NUM_LIT:2><EOL>center = [m['<STR_LIT>'] for m in s['<STR_LIT>']]<EOL>data = numpy.loadtxt('<STR_LIT>' % output_basename)<EOL>i = data[:,-<NUM_LIT:1>].argmax()<EOL>final = data[i,:-<NUM_LIT:1>] <EOL>return dict(start=final, chain=chain,<EOL>stdev=stdev, upper=upper, lower=lower,<EOL>method='<STR_LIT>')<EOL>

**MultiNest Nested Sampling**

via `PyMultiNest <http://johannesbuchner.github.com/PyMultiNest/index.html>`_.

:param parameter_names: name of parameters; not directly used here,
        but for multinest_marginal.py plotting tool.


def optimize(function, x0, cons=[], ftol=<NUM_LIT>, disp=<NUM_LIT:0>, plot=False):

if disp > <NUM_LIT:0>:<EOL><INDENT>print()<EOL>print('<STR_LIT>')<EOL>print()<EOL>print('<STR_LIT>', function, '<STR_LIT::>', x0)<EOL><DEDENT>points = []<EOL>values = []<EOL>def recordfunction(x):<EOL><INDENT>v = function(x)<EOL>points.append(x)<EOL>values.append(v)<EOL>return v<EOL><DEDENT>(a, b, c), (va, vb, vc) = seek_minimum_bracket(recordfunction, x0, cons=cons, ftol=ftol, disp=disp, plot=plot)<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>' % len(points), (a, b, c), (va, vb, vc))<EOL><DEDENT>if disp > <NUM_LIT:2>:<EOL><INDENT>if plot:<EOL><INDENT>plot_values(values, points, lastpoint=-<NUM_LIT:1>, ftol=ftol)<EOL><DEDENT>pause()<EOL><DEDENT>result = brent(recordfunction, a, b, c, va, vb, vc, cons=cons, ftol=ftol, disp=disp, plot=plot)<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>' % len(points), result)<EOL><DEDENT>if disp > <NUM_LIT:1> or len(points) > <NUM_LIT:20>:<EOL><INDENT>if plot:<EOL><INDENT>plot_values(values, points, lastpoint=-<NUM_LIT:1>, ftol=ftol)<EOL><DEDENT>if disp > <NUM_LIT:2>:<EOL><INDENT>pause()<EOL><DEDENT><DEDENT>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>print()<EOL>print('<STR_LIT>')<EOL>print()<EOL><DEDENT>global neval<EOL>neval += len(points)<EOL>return result<EOL>

**Optimization method based on Brent's method**

First, a bracket (a b c) is sought that contains the minimum (b value is
smaller than both a or c).

The bracket is then recursively halfed. Here we apply some modifications
to ensure our suggested point is not too close to either a or c,
because that could be problematic with the local approximation.
Also, if the bracket does not seem to include the minimum,
it is expanded generously in the right direction until it covers it.

Thus, this function is fail safe, and will always find a local minimum.


def cache2errors(function, cache, disp=<NUM_LIT:0>, ftol=<NUM_LIT>):

vals = numpy.array(sorted(cache, key=lambda x: x[<NUM_LIT:0>]))<EOL>if disp > <NUM_LIT:0>: print('<STR_LIT>', vals)<EOL>vi = vals[:,<NUM_LIT:1>].min()<EOL>def renormedfunc(x):<EOL><INDENT>y = function(x)<EOL>cache.append([x, y])<EOL>if disp > <NUM_LIT:1>: print('<STR_LIT>', x, y, y - (vi + <NUM_LIT:1>))<EOL>return y - (vi + <NUM_LIT:1>)<EOL><DEDENT>vals[:,<NUM_LIT:1>] -= vi + <NUM_LIT:1><EOL>lowmask = vals[:,<NUM_LIT:1>] < <NUM_LIT:0><EOL>highmask = vals[:,<NUM_LIT:1>] > <NUM_LIT:0><EOL>indices = numpy.arange(len(vals))<EOL>b, vb = vals[indices[lowmask][ <NUM_LIT:0>],:]<EOL>c, vc = vals[indices[lowmask][-<NUM_LIT:1>],:]<EOL>if any(vals[:,<NUM_LIT:0>][highmask] < b):<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>a, va = vals[indices[highmask][vals[:,<NUM_LIT:0>][highmask] < b][-<NUM_LIT:1>],:]<EOL><DEDENT>else:<EOL><INDENT>a = b<EOL>va = vb<EOL>while b > -<NUM_LIT:50>:<EOL><INDENT>a = b - max(vals[-<NUM_LIT:1>,<NUM_LIT:0>] - vals[<NUM_LIT:0>,<NUM_LIT:0>], <NUM_LIT:1>)<EOL>va = renormedfunc(a)<EOL>if disp > <NUM_LIT:0>: print('<STR_LIT>' % (b, vb, a, va))<EOL>if va > <NUM_LIT:0>:<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>break<EOL><DEDENT>else:<EOL><INDENT>b = a<EOL>vb = va<EOL><DEDENT><DEDENT><DEDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>', a, b, va, vb)<EOL>if va > <NUM_LIT:0> and vb < <NUM_LIT:0>:<EOL><INDENT>leftroot = scipy.optimize.brentq(renormedfunc, a, b, rtol=ftol)<EOL><DEDENT>else:<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>leftroot = a<EOL><DEDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>', leftroot)<EOL>if any(vals[:,<NUM_LIT:0>][highmask] > c):<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>d, vd = vals[indices[highmask][vals[:,<NUM_LIT:0>][highmask] > c][ <NUM_LIT:0>],:]<EOL><DEDENT>else:<EOL><INDENT>d = c<EOL>vd = vc<EOL>while c < <NUM_LIT:50>:<EOL><INDENT>d = c + max(vals[-<NUM_LIT:1>,<NUM_LIT:0>] - vals[<NUM_LIT:0>,<NUM_LIT:0>], <NUM_LIT:1>)<EOL>vd = renormedfunc(d)<EOL>if disp > <NUM_LIT:0>: print('<STR_LIT>' % (c, vc, d, vd))<EOL>if vd > <NUM_LIT:0>:<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>break<EOL><DEDENT>else:<EOL><INDENT>c = d<EOL>vc = vd<EOL><DEDENT><DEDENT><DEDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>', c, d, vc, vd)<EOL>if vd > <NUM_LIT:0> and vc < <NUM_LIT:0>:<EOL><INDENT>rightroot = scipy.optimize.brentq(renormedfunc, c, d, rtol=ftol)<EOL><DEDENT>else:<EOL><INDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>')<EOL>rightroot = d<EOL><DEDENT>if disp > <NUM_LIT:0>: print('<STR_LIT>', rightroot)<EOL>assert leftroot < rightroot<EOL>if disp > <NUM_LIT:2>:<EOL><INDENT>fullvals = numpy.array(sorted(cache, key=lambda x: x[<NUM_LIT:0>]))<EOL>fullvals[:,<NUM_LIT:1>] -= vi + <NUM_LIT:1><EOL>plt.figure()<EOL>plt.plot(fullvals[:,<NUM_LIT:0>], fullvals[:,<NUM_LIT:1>], '<STR_LIT:s>')<EOL>plt.plot(vals[:,<NUM_LIT:0>], vals[:,<NUM_LIT:1>], '<STR_LIT:o>')<EOL>plt.xlim(a, d)<EOL>plt.ylim(min(va, vb, vc, vd), max(va, vb, vc, vd))<EOL>ymin, ymax = plt.ylim()<EOL>plt.vlines([leftroot, rightroot], ymin, ymax, linestyles='<STR_LIT>')<EOL>plt.savefig('<STR_LIT>')<EOL><DEDENT>return leftroot, rightroot<EOL>

This function will attempt to identify 1 sigma errors, assuming your
function is a chi^2. For this, the 1-sigma is bracketed.

If you were smart enough to build a cache list of [x,y] into your function,
you can pass it here. The values bracketing 1 sigma will be used as
starting values.
If no such values exist, e.g. because all values were very close to the
optimum (good starting values), the bracket is expanded.


def de(output_basename, parameter_names, transform, loglikelihood, prior, nsteps=<NUM_LIT>, vizfunc=None, printfunc=None, **problem):

import json<EOL>import inspyred<EOL>import random<EOL>prng = random.Random()<EOL>if '<STR_LIT>' in problem:<EOL><INDENT>prng.seed(problem['<STR_LIT>'])<EOL><DEDENT>n_params = len(parameter_names)<EOL>seeds = problem.get('<STR_LIT>', [])<EOL>if '<STR_LIT:start>' in problem:<EOL><INDENT>seeds.append(problem['<STR_LIT:start>'])<EOL><DEDENT>prefix = output_basename<EOL>def viz(candidate, args):<EOL><INDENT>if vizfunc is not None:<EOL><INDENT>vizfunc(candidate)<EOL><DEDENT><DEDENT>def print_candidate(candidate, l, args):<EOL><INDENT>if printfunc is not None:<EOL><INDENT>printfunc(cube=candidate, loglikelihood=l)<EOL><DEDENT>else:<EOL><INDENT>print(l, candidate)<EOL><DEDENT><DEDENT>def eval_candidate(candidate):<EOL><INDENT>params = transform(candidate)<EOL>l = loglikelihood(params)<EOL>p = prior(params)<EOL>if numpy.isinf(p) and p < <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>return -<NUM_LIT><EOL><DEDENT>if numpy.isnan(l):<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>return l, p<EOL><DEDENT>@inspyred.ec.utilities.memoize<EOL>@inspyred.ec.evaluators.evaluator<EOL>def fitness(candidate, args):<EOL><INDENT>l, p = eval_candidate(candidate)<EOL>return (l + p)<EOL><DEDENT>cutoff_store = <NUM_LIT:10><EOL>def solution_archiver(random, population, archive, args):<EOL><INDENT>psize = len(population)<EOL>population.sort(reverse=True)<EOL>best = population[<NUM_LIT:0>].fitness<EOL>all_candidates = sorted(population + archive, reverse=True)<EOL>all_fitness = numpy.array([c.fitness for c in all_candidates])<EOL>mask = best - all_fitness > cutoff_store / <NUM_LIT:3><EOL>if mask.sum() < <NUM_LIT:20>:<EOL><INDENT>mask = best - all_fitness > cutoff_store<EOL><DEDENT>newarchive = [c for i, c in enumerate(all_candidates) if i == <NUM_LIT:0> or all_fitness[i - <NUM_LIT:1>] != c.fitness]<EOL>print('<STR_LIT>', len(archive), len(newarchive))<EOL>json.dump([{'<STR_LIT>': [float(f) for f in c.candidate], '<STR_LIT>':c.fitness} for c in newarchive], <EOL>open(prefix + '<STR_LIT>', '<STR_LIT:w>'), indent=<NUM_LIT:4>)<EOL>return newarchive<EOL><DEDENT>def observer(population, num_generations, num_evaluations, args):<EOL><INDENT>population.sort(reverse=True)<EOL>candidate = population[<NUM_LIT:0>]<EOL>print(('<STR_LIT>'.format(num_evaluations)), '<STR_LIT>', end='<STR_LIT:U+0020>') <EOL>print_candidate(candidate.candidate, candidate.fitness, args)<EOL>if num_evaluations % len(population) == <NUM_LIT:0> or num_evaluations < len(population) or args.get('<STR_LIT>', False):<EOL><INDENT>viz(candidate.candidate, args)<EOL><DEDENT><DEDENT>def generator(random, args): <EOL><INDENT>u = [random.uniform(<NUM_LIT:0>, <NUM_LIT:1>) for _ in range(n_params)]<EOL>u = [random.gauss(<NUM_LIT:0.5>, <NUM_LIT:0.1>) for _ in range(n_params)]<EOL>return bounder(u, args)<EOL><DEDENT>ea = inspyred.ec.DEA(prng)<EOL>ea.terminator = inspyred.ec.terminators.evaluation_termination<EOL>ea.archiver = solution_archiver<EOL>bounder = inspyred.ec.Bounder(lower_bound=<NUM_LIT>, upper_bound=<NUM_LIT:1>-<NUM_LIT>)<EOL>import copy<EOL>from math import log<EOL>@inspyred.ec.variators.mutator<EOL>def double_exponential_mutation(random, candidate, args):<EOL><INDENT>mut_rate = args.setdefault('<STR_LIT>', <NUM_LIT:0.1>)<EOL>mean = args.setdefault('<STR_LIT>', <NUM_LIT:0.0>)<EOL>stdev = args.setdefault('<STR_LIT>', <NUM_LIT:1.0>)<EOL>scale = log(<NUM_LIT:0.5>) / - (stdev)<EOL>bounder = args['<STR_LIT>'].bounder<EOL>mutant = copy.copy(candidate)<EOL>for i, m in enumerate(mutant):<EOL><INDENT>dice = random.random()<EOL>if dice < mut_rate:<EOL><INDENT>sign = (dice < mut_rate / <NUM_LIT:2>) * <NUM_LIT:2> - <NUM_LIT:1><EOL>delta = -log(random.random()) / scale<EOL>mutant[i] += delta * sign<EOL><DEDENT>mutant = bounder(mutant, args)<EOL><DEDENT>return mutant<EOL><DEDENT>def minute_gaussian_mutation(random, candidates, args):<EOL><INDENT>args = dict(args)<EOL>args['<STR_LIT>'] = <NUM_LIT:1><EOL>args['<STR_LIT>'] = <NUM_LIT><EOL>return inspyred.ec.variators.gaussian_mutation(random, candidates, args)<EOL><DEDENT>ea.variator = [inspyred.ec.variators.n_point_crossover, inspyred.ec.variators.gaussian_mutation, minute_gaussian_mutation]<EOL>ea.replacer = inspyred.ec.replacers.steady_state_replacement<EOL>ea.observer = observer<EOL>pop_size = <NUM_LIT:20><EOL>final_pop = ea.evolve(pop_size=pop_size, <EOL>max_evaluations=nsteps, maximize=True, seeds=seeds, <EOL>gaussian_stdev=<NUM_LIT>, <EOL>bounder=bounder, generator=generator, evaluator=fitness,<EOL>)<EOL>best = max(final_pop)<EOL>seeds = [c.candidate for c in ea.archive]<EOL>print('<STR_LIT>', best)<EOL>return {'<STR_LIT:start>': best.candidate, '<STR_LIT:value>': best.fitness,<EOL>'<STR_LIT>': seeds, '<STR_LIT>': '<STR_LIT>'}<EOL>

**Differential evolution**

via `inspyred <http://inspyred.github.io/>`_

specially tuned. steady state replacement, n-point crossover,
        pop size 20, gaussian mutation noise 0.01 & 1e-6.
stores intermediate results (can be used for resume, see seeds)

:param start: start point
:param seeds: list of start points
:param vizfunc: callback to do visualization of current best solution
:param printfunc: callback to summarize current best solution
:param seed: RNG initialization (if set)


def mcmc_advance(start, stdevs, logp, nsteps = <NUM_LIT>, adapt=True, callback=None):

import scipy<EOL>from numpy import log<EOL>import progressbar<EOL>prob = logp(start)<EOL>chain = [start]<EOL>accepts = [True]<EOL>probs = [prob]<EOL>assert not numpy.isnan(start).any()<EOL>assert not numpy.isnan(stdevs).any()<EOL>i = <NUM_LIT:0><EOL>widgets=['<STR_LIT>', progressbar.Percentage(), progressbar.Counter('<STR_LIT>'),<EOL>progressbar.Bar(), progressbar.ETA()]<EOL>pbar = progressbar.ProgressBar(widgets=widgets,<EOL>maxval=nsteps).start()<EOL>prev = start<EOL>prev_prob = prob<EOL>print('<STR_LIT>', prob)<EOL>stepchange = <NUM_LIT:0.1><EOL>while len(chain) < nsteps:<EOL><INDENT>i = i + <NUM_LIT:1><EOL>next = scipy.random.normal(prev, stdevs)<EOL>next[next > <NUM_LIT:1>] = <NUM_LIT:1><EOL>next[next < <NUM_LIT:0>] = <NUM_LIT:0><EOL>next_prob = logp(next)<EOL>assert not numpy.isnan(next).any()<EOL>assert not numpy.isnan(next_prob).any()<EOL>delta = next_prob - prev_prob<EOL>dice = log(scipy.random.uniform(<NUM_LIT:0>, <NUM_LIT:1>))<EOL>accept = delta > dice<EOL>if accept:<EOL><INDENT>prev = next<EOL>prev_prob = next_prob<EOL>if adapt: stdevs *= (<NUM_LIT:1> + stepchange)<EOL><DEDENT>else:<EOL><INDENT>if adapt: stdevs *= (<NUM_LIT:1> + stepchange)**(-<NUM_LIT>) <EOL><DEDENT>if callback: callback(prev_prob, prev, accept)<EOL>chain.append(prev)<EOL>accepts.append(accept)<EOL>probs.append(prev_prob)<EOL>if adapt: stepchange = min(<NUM_LIT:0.1>, <NUM_LIT> / i)<EOL>widgets[<NUM_LIT:0>] = '<STR_LIT>' % numpy.mean(numpy.array(accepts[len(accepts)/<NUM_LIT:3>:])+<NUM_LIT:0>)<EOL>pbar.update(pbar.currval + <NUM_LIT:1>)<EOL><DEDENT>pbar.finish()<EOL>return chain, probs, accepts, stdevs<EOL>

Generic Metropolis MCMC. Advances the chain by nsteps.
Called by :func:`mcmc`

:param adapt: enables adaptive stepwidth alteration (converges).


def mcmc(transform, loglikelihood, parameter_names, nsteps=<NUM_LIT>, nburn=<NUM_LIT>, <EOL>stdevs=<NUM_LIT:0.1>, start = <NUM_LIT:0.5>, **problem):

if '<STR_LIT>' in problem:<EOL><INDENT>numpy.random.seed(problem['<STR_LIT>'])<EOL><DEDENT>n_params = len(parameter_names)<EOL>def like(cube):<EOL><INDENT>cube = numpy.array(cube)<EOL>if (cube <= <NUM_LIT>).any() or (cube >= <NUM_LIT:1>-<NUM_LIT>).any():<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>params = transform(cube)<EOL>return loglikelihood(params)<EOL><DEDENT>start = start + numpy.zeros(n_params)<EOL>stdevs = stdevs + numpy.zeros(n_params)<EOL>def compute_stepwidths(chain):<EOL><INDENT>return numpy.std(chain, axis=<NUM_LIT:0>) / <NUM_LIT:3><EOL><DEDENT>import matplotlib.pyplot as plt<EOL>plt.figure(figsize=(<NUM_LIT:7>, <NUM_LIT:7>))<EOL>steps = numpy.array([<NUM_LIT:0.1>]*(n_params))<EOL>print('<STR_LIT>')<EOL>chain, prob, _, steps_ = mcmc_advance(start, steps, like, nsteps=nburn / <NUM_LIT:2>, adapt=True)<EOL>steps = compute_stepwidths(chain)<EOL>print('<STR_LIT>')<EOL>chain, prob, _, steps_ = mcmc_advance(chain[-<NUM_LIT:1>], steps, like, nsteps=nburn / <NUM_LIT:2>, adapt=True)<EOL>steps = compute_stepwidths(chain)<EOL>print('<STR_LIT>')<EOL>chain, prob, _, steps_ = mcmc_advance(chain[-<NUM_LIT:1>], steps, like, nsteps=nsteps)<EOL>chain = numpy.array(chain)<EOL>i = numpy.argmax(prob)<EOL>final = chain[-<NUM_LIT:1>]<EOL>print('<STR_LIT>')<EOL>chain = numpy.array([transform(params) for params in chain])<EOL>return dict(start=chain[-<NUM_LIT:1>], maximum=chain[i], seeds=[final, chain[i]], chain=chain, method='<STR_LIT>')<EOL>

**Metropolis Hastings MCMC**

with automatic step width adaption.
Burnin period is also used to guess steps.

:param nburn: number of burnin steps
:param stdevs: step widths to start with


def ensemble(transform, loglikelihood, parameter_names, nsteps=<NUM_LIT>, nburn=<NUM_LIT>, <EOL>start=<NUM_LIT:0.5>, **problem):

import emcee<EOL>import progressbar<EOL>if '<STR_LIT>' in problem:<EOL><INDENT>numpy.random.seed(problem['<STR_LIT>'])<EOL><DEDENT>n_params = len(parameter_names)<EOL>nwalkers = <NUM_LIT:50> + n_params * <NUM_LIT:2><EOL>if nwalkers > <NUM_LIT:200>:<EOL><INDENT>nwalkers = <NUM_LIT:200><EOL><DEDENT>p0 = [numpy.random.rand(n_params) for i in range(nwalkers)]<EOL>start = start + numpy.zeros(n_params)<EOL>p0[<NUM_LIT:0>] = start<EOL>def like(cube):<EOL><INDENT>cube = numpy.array(cube)<EOL>if (cube <= <NUM_LIT>).any() or (cube >= <NUM_LIT:1>-<NUM_LIT>).any():<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>params = transform(cube)<EOL>return loglikelihood(params)<EOL><DEDENT>sampler = emcee.EnsembleSampler(nwalkers, n_params, like,<EOL>live_dangerously=True)<EOL>print('<STR_LIT>')<EOL>pos, prob, state = sampler.run_mcmc(p0, nburn / nwalkers)<EOL>sampler.reset()<EOL>print('<STR_LIT>')<EOL>pbar = progressbar.ProgressBar(<EOL>widgets=[progressbar.Percentage(), progressbar.Counter('<STR_LIT>'),<EOL>progressbar.Bar(), progressbar.ETA()],<EOL>maxval=nsteps).start()<EOL>for results in sampler.sample(pos, iterations=nsteps / nwalkers, rstate0=state):<EOL><INDENT>pbar.update(pbar.currval + <NUM_LIT:1>)<EOL><DEDENT>pbar.finish()<EOL>print("<STR_LIT>", numpy.mean(sampler.acceptance_fraction))<EOL>chain = sampler.flatchain<EOL>final = chain[-<NUM_LIT:1>]<EOL>print('<STR_LIT>')<EOL>chain_post = numpy.array([transform(params) for params in chain])<EOL>chain_prob = sampler.flatlnprobability<EOL>return dict(start=final, chain=chain_post, chain_prior=chain,<EOL>chain_prob=chain_prob,<EOL>method='<STR_LIT>')<EOL>

**Ensemble MCMC**

via `emcee <http://dan.iel.fm/emcee/>`_


def classical(transform, loglikelihood, parameter_names, prior, <EOL>start = <NUM_LIT:0.5>, ftol=<NUM_LIT:0.1>, disp=<NUM_LIT:0>, nsteps=<NUM_LIT>,<EOL>method='<STR_LIT>', **args):

import scipy.optimize<EOL>n_params = len(parameter_names)<EOL>def minfunc(params):<EOL><INDENT>l = loglikelihood(params)<EOL>p = prior(params)<EOL>if numpy.isinf(p) and p < <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>return -<NUM_LIT><EOL><DEDENT>if numpy.isnan(l):<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>return -l - p<EOL><DEDENT>def minfunc_cube(cube):<EOL><INDENT>cube = numpy.array(cube)<EOL>if (cube <= <NUM_LIT>).any() or (cube >= <NUM_LIT:1>-<NUM_LIT>).any():<EOL><INDENT>return <NUM_LIT><EOL><DEDENT>params = transform(cube)<EOL>l = loglikelihood(params)<EOL>p = prior(params)<EOL>if numpy.isinf(p) and p < <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>return -<NUM_LIT><EOL><DEDENT>if numpy.isnan(l):<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>return -l - p<EOL><DEDENT>start = start + numpy.zeros(n_params)<EOL>ret = {}<EOL>if method == '<STR_LIT>':<EOL><INDENT>final, value, _niter, neval, warnflag = scipy.optimize.fmin(minfunc_cube, start, ftol=ftol, disp=disp, maxfun=nsteps, full_output=True)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>cons =  [lambda params: params[i]     for i in range(n_params)]<EOL>cons += [lambda params: <NUM_LIT:1> - params[i] for i in range(n_params)]<EOL>final = scipy.optimize.fmin_cobyla(minfunc_cube, start, cons, rhoend=ftol / <NUM_LIT:10>, disp=disp, maxfun=nsteps)<EOL>neval = nsteps<EOL><DEDENT>elif method == '<STR_LIT>' or method == '<STR_LIT>':<EOL><INDENT>"""<STR_LIT>"""<EOL>s = '<STR_LIT:U+002CU+0020>'.join(parameter_names)<EOL>s = """<STR_LIT>""" % (s, s)<EOL>if method == '<STR_LIT>':<EOL><INDENT>f = eval(s, dict(minfunc=minfunc, numpy=numpy))<EOL>start = transform(start)<EOL><DEDENT>else:<EOL><INDENT>f = eval(s, dict(minfunc=minfunc_cube, numpy=numpy))<EOL><DEDENT>import minuit<EOL>m = minuit.Minuit(f)<EOL>for i, p in enumerate(parameter_names):<EOL><INDENT>m.values[p] = start[i]<EOL>if method == '<STR_LIT>':<EOL><INDENT>m.limits[p] = (<NUM_LIT>, <NUM_LIT:1> - <NUM_LIT>)<EOL><DEDENT><DEDENT>m.up = <NUM_LIT:0.5><EOL>m.tol = ftol * <NUM_LIT:100><EOL>m.printMode = disp<EOL>if method == '<STR_LIT>':<EOL><INDENT>m.migrad()<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>m.hesse()<EOL><DEDENT>final = [m.values[p] for p in parameter_names]<EOL>neval = m.ncalls<EOL>errors = [m.errors[p] for p in parameter_names]<EOL>if method == '<STR_LIT>':<EOL><INDENT>c0 = final<EOL>p0 = transform(c0)<EOL>stdev = numpy.zeros(n_params)<EOL>lower = numpy.zeros(n_params)<EOL>upper = numpy.zeros(n_params)<EOL>for i, w in enumerate(errors):<EOL><INDENT>c1 = numpy.copy(c0)<EOL>c1[i] -= w<EOL>c2 = numpy.copy(c0)<EOL>c2[i] += w<EOL>p1 = transform(c1)<EOL>p2 = transform(c2)<EOL>stdev[i] = numpy.abs(p2[i] - p1[i]) / <NUM_LIT:2><EOL>lower[i] = min(p2[i], p1[i])<EOL>upper[i] = max(p2[i], p1[i])<EOL><DEDENT>ret['<STR_LIT>'] = stdev<EOL>ret['<STR_LIT>'] = upper<EOL>ret['<STR_LIT>'] = lower<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>ret['<STR_LIT>'] = errors<EOL>ret['<STR_LIT>'] = numpy.matrix([[m.covariance[(a, b)] for b in parameter_names] for a in parameter_names])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>from openopt import NLP<EOL>lo = [<NUM_LIT>] * n_params<EOL>hi = [<NUM_LIT:1>-<NUM_LIT>] * n_params<EOL>iprint = <NUM_LIT:0> if disp == <NUM_LIT:0> else <NUM_LIT:10> if disp == <NUM_LIT:1> else <NUM_LIT:1><EOL>p = NLP(f=minfunc_cube, x0=start, lb=lo, ub=hi,<EOL>maxFunEvals=nsteps, ftol=ftol, iprint=iprint)<EOL>r = p.solve(method)<EOL>final = r.xf<EOL>neval = r.evals['<STR_LIT:f>']<EOL><DEDENT>ret.update(dict(start=final, maximum=transform(final), method=method, neval=neval))<EOL>return ret<EOL>

**Classic optimization methods**

:param start: start position vector (before transform)
:param ftol: accuracy required to stop at optimum
:param disp: verbosity
:param nsteps: number of steps
:param method: string
        neldermead, cobyla (via `scipy.optimize <http://docs.scipy.org/doc/scipy/reference/tutorial/optimize.html>`_)
        bobyqa, ralg, algencan, ipopt, mma, auglag and many others from the OpenOpt framework (via `openopt.NLP <http://openopt.org/NLP>`_)
        minuit (via `PyMinuit <https://code.google.com/p/pyminuit/>`_)


def onebyone(transform, loglikelihood, parameter_names, prior, start = <NUM_LIT:0.5>, ftol=<NUM_LIT:0.1>, disp=<NUM_LIT:0>, nsteps=<NUM_LIT>,<EOL>parallel=False, find_uncertainties=False, **args):

def minfunc(cube):<EOL><INDENT>cube = numpy.array(cube)<EOL>if (cube <= <NUM_LIT>).any() or (cube >= <NUM_LIT:1>-<NUM_LIT>).any():<EOL><INDENT>return <NUM_LIT><EOL><DEDENT>params = transform(cube)<EOL>l = loglikelihood(params)<EOL>p = prior(params)<EOL>if numpy.isinf(p) and p < <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>return -<NUM_LIT><EOL><DEDENT>if numpy.isnan(l):<EOL><INDENT>return -<NUM_LIT><EOL><DEDENT>return -l - p<EOL><DEDENT>if parallel:<EOL><INDENT>func = opt_grid_parallel<EOL><DEDENT>else:<EOL><INDENT>func = opt_grid<EOL><DEDENT>n_params = len(parameter_names)<EOL>start = start + numpy.zeros(n_params)<EOL>ret = func(start, minfunc, [(<NUM_LIT>, <NUM_LIT:1>-<NUM_LIT>)] * n_params, ftol=ftol, disp=disp, compute_errors=find_uncertainties)<EOL>if find_uncertainties:<EOL><INDENT>c0 = ret[<NUM_LIT:0>]<EOL>p0 = transform(c0)<EOL>stdev = numpy.zeros(n_params)<EOL>lower = numpy.zeros(n_params)<EOL>upper = numpy.zeros(n_params)<EOL>for i, (lo, hi) in enumerate(ret[<NUM_LIT:1>]):<EOL><INDENT>c1 = numpy.copy(c0)<EOL>c1[i] = lo<EOL>c2 = numpy.copy(c0)<EOL>c2[i] = hi<EOL>p1 = transform(c1)<EOL>p2 = transform(c2)<EOL>stdev[i] = numpy.abs(p2[i] - p1[i]) / <NUM_LIT:2><EOL>lower[i] = min(p2[i], p1[i])<EOL>upper[i] = max(p2[i], p1[i])<EOL><DEDENT>return dict(start=ret[<NUM_LIT:0>], maximum=p0,<EOL>stdev=stdev, upper=upper, lower=lower,<EOL>method='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>return dict(start=ret, maximum=transform(ret), method='<STR_LIT>')<EOL><DEDENT>

**Convex optimization based on Brent's method**

A strict assumption of one optimum between the parameter limits is used.
The bounds are narrowed until it is found, i.e. the likelihood function is flat
within the bounds.
* If optimum outside bracket, expands bracket until contained.
* Thus guaranteed to return local optimum.
* Supports parallelization (multiple parameters are treated independently)
* Supports finding ML uncertainties (Delta-Chi^2=1)

Very useful for 1-3d problems.
Otherwise useful, reproducible/deterministic algorithm for finding the minimum in
well-behaved likelihoods, where the parameters are weakly independent,
or to find a good starting point.
Optimizes each parameter in order, assuming they are largely independent.

For 1-dimensional algorithm used, see :func:`jbopt.opt_grid`

:param ftol: difference in values at which the function can be considered flat
:param compute_errors: compute standard deviation of gaussian around optimum


def opt_normalizations(params, func, limits, abandon_threshold=<NUM_LIT:100>, noimprovement_threshold=<NUM_LIT>,<EOL>disp=<NUM_LIT:0>):

newparams = numpy.copy(params)<EOL>lower = [lo for lo, hi in limits]<EOL>upper = [hi for lo, hi in limits]<EOL>for i, p in enumerate(params):<EOL><INDENT>startval = p<EOL>beststat = func(newparams)<EOL>bestval = startval<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (startval, beststat))<EOL><DEDENT>go_up = True<EOL>go_down = True<EOL>for n in list(<NUM_LIT>**numpy.arange(<NUM_LIT:1>, <NUM_LIT:20>)) + [None] + list(<NUM_LIT>**numpy.arange(<NUM_LIT:1>, <NUM_LIT>)):<EOL><INDENT>if n is None:<EOL><INDENT>startval = bestval<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (startval))<EOL><DEDENT>go_up = True<EOL>go_down = True<EOL>continue<EOL><DEDENT>if go_up and startval * n > upper[i]:<EOL><INDENT>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' %<EOL>(startval, n, upper[i]))<EOL><DEDENT>go_up = False<EOL><DEDENT>if go_down and startval / n < lower[i]:<EOL><INDENT>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' %<EOL>(startval, n, lower[i]))<EOL><DEDENT>go_down = False<EOL><DEDENT>if go_up:<EOL><INDENT>if disp > <NUM_LIT:1>:<EOL><INDENT>print('<STR_LIT>' % (startval * n))<EOL><DEDENT>newparams[i] = startval * n<EOL>newstat = func(newparams)<EOL>if disp > <NUM_LIT:1>:<EOL><INDENT>print('<STR_LIT>' % (newparams[i], newstat))<EOL><DEDENT>if newstat <= beststat:<EOL><INDENT>bestval = newparams[i]<EOL>beststat = newstat<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % newparams[i])<EOL><DEDENT><DEDENT>if newstat > beststat + abandon_threshold:<EOL><INDENT>go_up = False<EOL><DEDENT><DEDENT>if go_down:<EOL><INDENT>if disp > <NUM_LIT:1>:<EOL><INDENT>print('<STR_LIT>' % (startval / n))<EOL><DEDENT>newparams[i] = startval / n<EOL>newstat = func(newparams)<EOL>if disp > <NUM_LIT:1>:<EOL><INDENT>print('<STR_LIT>' % (newparams[i], newstat))<EOL><DEDENT>if newstat + noimprovement_threshold < beststat:  <EOL><INDENT>bestval = newparams[i]<EOL>beststat = newstat<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % newparams[i])<EOL><DEDENT><DEDENT>if newstat > beststat + abandon_threshold:<EOL><INDENT>go_down = False<EOL><DEDENT><DEDENT><DEDENT>newparams[i] = bestval<EOL>print('<STR_LIT>' % (i, newparams[i]))<EOL><DEDENT>print('<STR_LIT>' % (beststat))<EOL>return newparams<EOL>

**optimization algorithm for scale variables (positive value of unknown magnitude)**

Each parameter is a normalization of a feature, and its value is sought.
The parameters are handled in order (assumed to be independent),
but a second round can be run.
Various magnitudes of the normalization are tried. If the normalization converges
to zero, the largest value yielding a comparable value is used.

Optimizes each normalization parameter in rough steps
using multiples of 3 of start point
to find reasonable starting values for another algorithm.

parameters, minimization function, parameter space definition [(lo, hi) for i in params]

:param abandon_threshold:
        if in one direction the function increases by this much over the best value,
        abort search in this direction
:param noimprovement_threshold:
        when decreasing the normalization, if the function increases by less than
        this amount, abort search in this direction
:param disp:
        verbosity


def opt_grid(params, func, limits, ftol=<NUM_LIT>, disp=<NUM_LIT:0>, compute_errors=True):

caches = [[] for p in params]<EOL>newparams = numpy.copy(params)<EOL>errors = [[] for p in params]<EOL>for i, p in enumerate(params):<EOL><INDENT>cache = []<EOL>def func1(x0):<EOL><INDENT>newparams[i] = x0<EOL>v = func(newparams)<EOL>cache.append([x0, v])<EOL>return v<EOL><DEDENT>lo, hi = limits[i]<EOL>bestval = optimize(func1, x0=p,<EOL>cons=[lambda x: x - lo, lambda x: hi - x],<EOL>ftol=ftol, disp=disp - <NUM_LIT:1>)<EOL>beststat = func1(bestval)<EOL>if compute_errors:<EOL><INDENT>errors[i] = cache2errors(func1, cache, disp=disp - <NUM_LIT:1>)<EOL><DEDENT>newparams[i] = bestval<EOL>caches[i] = cache<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>if compute_errors:<EOL><INDENT>print('<STR_LIT>' % (<EOL>i, bestval, errors[i][<NUM_LIT:0>], errors[i][<NUM_LIT:1>], beststat))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' %<EOL>(i, bestval, beststat))<EOL><DEDENT><DEDENT><DEDENT>beststat = func(newparams)<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (beststat))<EOL><DEDENT>if compute_errors:<EOL><INDENT>return newparams, errors<EOL><DEDENT>else:<EOL><INDENT>return newparams<EOL><DEDENT>

see :func:`optimize1d.optimize`, considers each parameter in order

:param ftol:
        difference in values at which the function can be considered flat
:param compute_errors:
        compute standard deviation of gaussian around optimum


def opt_grid_parallel(params, func, limits, ftol=<NUM_LIT>, disp=<NUM_LIT:0>, compute_errors=True):

import multiprocessing<EOL>def spawn(f):<EOL><INDENT>def fun(q_in, q_out):<EOL><INDENT>while True:<EOL><INDENT>i, x = q_in.get()<EOL>if i == None:<EOL><INDENT>break<EOL><DEDENT>q_out.put((i, f(x)))<EOL><DEDENT><DEDENT>return fun<EOL><DEDENT>def parmap(f, X, nprocs=multiprocessing.cpu_count()):<EOL><INDENT>q_in = multiprocessing.Queue(<NUM_LIT:1>)<EOL>q_out = multiprocessing.Queue()<EOL>proc = [multiprocessing.Process(target=spawn(<EOL>f), args=(q_in, q_out)) for _ in range(nprocs)]<EOL>for p in proc:<EOL><INDENT>p.daemon = True<EOL>p.start()<EOL><DEDENT>sent = [q_in.put((i, x)) for i, x in enumerate(X)]<EOL>[q_in.put((None, None)) for _ in range(nprocs)]<EOL>res = [q_out.get() for _ in range(len(sent))]<EOL>[p.join() for p in proc]<EOL>return [x for i, x in sorted(res)]<EOL><DEDENT>nthreads = multiprocessing.cpu_count()<EOL>caches = [[] for p in params]<EOL>newparams = numpy.copy(params)<EOL>errors = [[] for p in params]<EOL>indices = list(range(<NUM_LIT:0>, len(params), nthreads))<EOL>k = <NUM_LIT:0><EOL>while k < len(params):<EOL><INDENT>j = min(len(params), k + nthreads * <NUM_LIT:2>)<EOL>def run1d(xxx_todo_changeme):<EOL><INDENT>(i, curparams, curlimits) = xxx_todo_changeme<EOL>cache = []<EOL>def func1(x0):<EOL><INDENT>curparams[i] = x0<EOL>v = func(curparams)<EOL>cache.append([x0, v])<EOL>return v<EOL><DEDENT>lo, hi = curlimits<EOL>bestval = optimize(func1, x0=p,<EOL>cons=[lambda x: x - lo, lambda x: hi - x],<EOL>ftol=ftol, disp=disp - <NUM_LIT:1>)<EOL>beststat = func1(bestval)<EOL>if compute_errors:<EOL><INDENT>errors = cache2errors(func1, cache, disp=disp - <NUM_LIT:1>)<EOL>return bestval, beststat, errors, cache<EOL><DEDENT>return bestval, beststat, cache<EOL><DEDENT>results = parmap(<EOL>run1d, [(i, numpy.copy(newparams), limits[i]) for i in range(k, j)])<EOL>for i, r in enumerate(results):<EOL><INDENT>if compute_errors:<EOL><INDENT>v, s, e, c = r<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (<EOL>i + k, v, e[<NUM_LIT:0>], e[<NUM_LIT:1>], s))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>v, s, c = r<EOL>e = []<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (i + k, v, s))<EOL><DEDENT><DEDENT>newparams[i + k] = v<EOL>caches[i + k] = c<EOL>errors[i + k] = e<EOL><DEDENT>k = j<EOL><DEDENT>beststat = func(newparams)<EOL>if disp > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>' % (beststat))<EOL><DEDENT>if compute_errors:<EOL><INDENT>return newparams, errors<EOL><DEDENT>else:<EOL><INDENT>return newparams<EOL><DEDENT>

parallelized version of :func:`opt_grid`


if chromosomes and chromosomes in valid_chromosomes:<EOL><INDENT>self.chromosomes = chromosomes<EOL><DEDENT>else:<EOL><INDENT>self.chromosomes = random.choice([XX, XY])<EOL><DEDENT>

This model uses the XY sex-determination system. Sex != gender.
        Assign either XX or XY randomly with a 50/50 chance of each, unless
        <chromosomes> are passed as an argument.


if gender and gender in genders:<EOL><INDENT>self.gender = gender<EOL><DEDENT>else:<EOL><INDENT>if not self.chromosomes: self.set_chromosomes()<EOL>self.gender = npchoice(genders, <NUM_LIT:1>, p=p_gender[self.chromosomes])[<NUM_LIT:0>]<EOL><DEDENT>

This model recognizes that sex chromosomes don't always line up with
        gender. Assign M, F, or NB according to the probabilities in p_gender.


def set_inherited_traits(self, egg_donor, sperm_donor):

if type(egg_donor) == str:<EOL><INDENT>self.reproduce_asexually(egg_donor, sperm_donor)<EOL><DEDENT>else:<EOL><INDENT>self.reproduce_sexually(egg_donor, sperm_donor)<EOL><DEDENT>

Accept either strings or Gods as inputs.


def reproduce_asexually(self, egg_word, sperm_word):

egg = self.generate_gamete(egg_word)<EOL>sperm = self.generate_gamete(sperm_word)<EOL>self.genome = list(set(egg + sperm)) <EOL>self.generation = <NUM_LIT:1><EOL>self.divinity = god<EOL>

Produce two gametes, an egg and a sperm, from the input strings.
        Combine them to produce a genome a la sexual reproduction.


def reproduce_sexually(self, egg_donor, sperm_donor):

egg_word = random.choice(egg_donor.genome)<EOL>egg = self.generate_gamete(egg_word)<EOL>sperm_word = random.choice(sperm_donor.genome)<EOL>sperm = self.generate_gamete(sperm_word)<EOL>self.genome = list(set(egg + sperm)) <EOL>self.parents = [egg_donor.name, sperm_donor.name]<EOL>self.generation = max(egg_donor.generation, sperm_donor.generation) + <NUM_LIT:1><EOL>sum_ = egg_donor.divinity + sperm_donor.divinity<EOL>self.divinity = int(npchoice(divinities, <NUM_LIT:1>, p=p_divinity[sum_])[<NUM_LIT:0>])<EOL>

Produce two gametes, an egg and a sperm, from input Gods. Combine
        them to produce a genome a la sexual reproduction. Assign divinity
        according to probabilities in p_divinity. The more divine the parents,
        the more divine their offspring.


if not self.gender: self.set_gender()<EOL>name = '<STR_LIT>'<EOL>if self.gender == female:<EOL><INDENT>name = names.female_names.pop()<EOL><DEDENT>elif self.gender == male:<EOL><INDENT>name = names.male_names.pop()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>name = names.nb_names.pop()<EOL><DEDENT>except:<EOL><INDENT>name = names.male_names.pop()<EOL><DEDENT><DEDENT>self.name = name<EOL>

Pick a random name from the lists loaded with the model. For Gods that
        identify as neither M nor F, the model attempts to retrieve an androgynous
        name. Note: not all of the scraped name lists contain androgynous names.


if self.divinity == human:<EOL><INDENT>obsession = random.choice(self.genome)<EOL>if self.gender == female:<EOL><INDENT>self.epithet = '<STR_LIT>'<EOL><DEDENT>elif self.gender == male:<EOL><INDENT>self.epithet = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>self.epithet = '<STR_LIT>'<EOL><DEDENT>self.epithet += '<STR_LIT>' + obsession<EOL>return <EOL><DEDENT>if self.gender == female:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>elif self.gender == male:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>if self.divinity == demi_god:<EOL><INDENT>title = '<STR_LIT>' + title if self.gender == non_binary else '<STR_LIT>' + title<EOL><DEDENT>num_domains = npchoice([<NUM_LIT:1>,<NUM_LIT:2>,<NUM_LIT:3>,<NUM_LIT:4>], <NUM_LIT:1>, p=[<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>])[<NUM_LIT:0>]<EOL>if num_domains == <NUM_LIT:1>:<EOL><INDENT>template = '<STR_LIT>'<EOL><DEDENT>if num_domains == <NUM_LIT:2>:<EOL><INDENT>template = '<STR_LIT>'<EOL><DEDENT>elif num_domains == <NUM_LIT:3>:<EOL><INDENT>template = '<STR_LIT>' <EOL><DEDENT>elif num_domains == <NUM_LIT:4>:<EOL><INDENT>template = '<STR_LIT>'<EOL><DEDENT>self.domains = [d.title() for d in random.sample(self.genome, num_domains)]<EOL>self.epithet = template % (title, *self.domains)<EOL>

Divine an appropriate epithet for this God. (See what I did there?)


p_rate_of_mutation = [<NUM_LIT>, <NUM_LIT:0.1>]<EOL>should_use_mutant_pool = (npchoice([<NUM_LIT:0>,<NUM_LIT:1>], <NUM_LIT:1>, p=p_rate_of_mutation)[<NUM_LIT:0>] == <NUM_LIT:1>)<EOL>if should_use_mutant_pool:<EOL><INDENT>pool = tokens.secondary_tokens<EOL><DEDENT>else:<EOL><INDENT>pool = tokens.primary_tokens<EOL><DEDENT>return get_matches(egg_or_sperm_word, pool, <NUM_LIT>)<EOL>

Extract 23 'chromosomes' aka words from 'gene pool' aka list of tokens
        by searching the list of tokens for words that are related to the given
        egg_or_sperm_word.


if self.gender == female:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>elif self.gender == male:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>p1 = self.parents[<NUM_LIT:0>]<EOL>p2 = self.parents[<NUM_LIT:1>]<EOL>template = '<STR_LIT>'<EOL>print(template % (title, p1.name, p1.epithet, p2.name, p2.epithet))<EOL>


def send_birth_announcement(parent_a, parent_b, child):

print("<STR_LIT>" % (child.name, child.epithet))<EOL>


self.gods[god.name] = god<EOL>

Add a god to this Pantheon's gods dictionary.


try:<EOL><INDENT>return self.gods[name_of_god]<EOL><DEDENT>except:<EOL><INDENT>print('<STR_LIT>' % name_of_god)<EOL><DEDENT>

Retrieve a god from this Pantheon's gods dictionary.


egg_donors = [god for god in self.gods.values() if god.chromosomes == '<STR_LIT>']<EOL>sperm_donors = [god for god in self.gods.values() if god.chromosomes == '<STR_LIT>']<EOL>for i in range(generations):<EOL><INDENT>print("<STR_LIT>" % (i+<NUM_LIT:1>))<EOL>gen_xx = []<EOL>gen_xy = []<EOL>for egg_donor in egg_donors:<EOL><INDENT>sperm_donor = random.choice(sperm_donors)<EOL>brood = self.breed(egg_donor, sperm_donor)<EOL>for child in brood:<EOL><INDENT>if child.divinity > human:<EOL><INDENT>self.add_god(child)<EOL><DEDENT>if child.chromosomes == '<STR_LIT>':<EOL><INDENT>gen_xx.append(child)<EOL><DEDENT>else:<EOL><INDENT>gen_xy.append(child)<EOL><DEDENT><DEDENT><DEDENT>egg_donors = [ed for ed in egg_donors if ed.generation > (i-<NUM_LIT:2>)]<EOL>sperm_donors = [sd for sd in sperm_donors if sd.generation > (i-<NUM_LIT:3>)]<EOL>egg_donors += gen_xx<EOL>sperm_donors += gen_xy<EOL><DEDENT>


def breed(self, egg_donor, sperm_donor):

offspring = []<EOL>try:<EOL><INDENT>num_children = npchoice([<NUM_LIT:1>,<NUM_LIT:2>], <NUM_LIT:1>, p=[<NUM_LIT>, <NUM_LIT>])[<NUM_LIT:0>] <EOL>for _ in range(num_children):<EOL><INDENT>child = God(egg_donor, sperm_donor)<EOL>offspring.append(child)<EOL>send_birth_announcement(egg_donor, sperm_donor, child)<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>return offspring<EOL>


save_tokens_to_dir(pool, filters)<EOL>


make_tokens_dir(pool, individuals)<EOL>


text_files = [fname for fname in os.listdir(corpora_dir)if fname.split('<STR_LIT:.>')[<NUM_LIT:1>] == '<STR_LIT>']<EOL>for text_fname in text_files:<EOL><INDENT>json_fname = text_fname.split('<STR_LIT:.>')[<NUM_LIT:0>] + '<STR_LIT>'<EOL>if os.path.isfile(corpora_dir + json_fname):<EOL><INDENT>continue <EOL><DEDENT>print("<STR_LIT>" + text_fname)<EOL>text = open(corpora_dir + text_fname).read()<EOL>words = nltk.word_tokenize(text)<EOL>with open(corpora_dir + json_fname, '<STR_LIT:w>') as outjson:<EOL><INDENT>json.dump(words, outjson)<EOL><DEDENT><DEDENT>

Generate a json file for each txt file in the /data/corpora directory.


return [f for f in os.listdir(corpora_dir) if f.split('<STR_LIT:.>')[<NUM_LIT:1>] == '<STR_LIT>']<EOL>

Retrieve the filenames of all tokenized text files in /data/corpora.
    Useful when you want to hand-pick <sources> for make_tokens_dir().


os.mkdir(tokens_dir + dir_)<EOL>for source in sources:<EOL><INDENT>if not os.path.isfile(corpora_dir + source):<EOL><INDENT>print('<STR_LIT>' + source)<EOL>return<EOL><DEDENT><DEDENT>with open(tokens_dir + dir_ + '<STR_LIT>', '<STR_LIT:w>') as outjson:<EOL><INDENT>json.dump(sources, outjson)<EOL><DEDENT>

Create a new directory named <dir_>. Create a new file within it called
    sources.json. The input <sources> is a list of names of tokenized texts.
    Write <sources> into sources.json.


with open(tokens_dir + dir_ + '<STR_LIT>', '<STR_LIT:r>') as injson:<EOL><INDENT>data = json.load(injson)<EOL>sources = [corpora_dir + fname for fname in data]<EOL><DEDENT>with open('<STR_LIT>', '<STR_LIT:r>') as f:<EOL><INDENT>skipwords = [line.rstrip() for line in f]<EOL><DEDENT>tokens_list = []<EOL>for fname in sources:<EOL><INDENT>print("<STR_LIT>" + fname)<EOL>with open(fname, '<STR_LIT:r>') as injson:<EOL><INDENT>data = json.load(injson)<EOL>words = [w.lower() for w in data if not w == '<STR_LIT>']<EOL>filtered = [w for w,p in nltk.pos_tag(words) if p in filters]<EOL>sanitized = [w for w in filtered if not w in skipwords]<EOL>tokens_list += sanitized<EOL><DEDENT><DEDENT>tokens_list = list(set(tokens_list)) <EOL>target = tokens_dir + dir_ + '<STR_LIT:/>' + '<STR_LIT:->'.join(filters) + '<STR_LIT>'<EOL>with open(target, '<STR_LIT:w>') as outjson:<EOL><INDENT>json.dump(tokens_list, outjson)<EOL><DEDENT>

Find sources.json in <dir_>. It contains a list of tokenized texts. For
    each tokenized text listed in sources.json, read its tokens, filter them,
    and add them to an aggregated list. Write the aggregated list to disk using
    a filename based on the <filters> given.


def get_matches(word, tokens, limit, offset=<NUM_LIT:0>):

return closest(tokens, word_vec(word), limit, offset)<EOL>

Return words from <tokens> that are most closely related to <word>.


return nlp.vocab[word].vector<EOL>


if norm(vec1) > <NUM_LIT:0> and norm(vec2) > <NUM_LIT:0>:<EOL><INDENT>return dot(vec1, vec2) / (norm(vec1) * norm(vec2))<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>

Compare vectors. Borrowed from A. Parish.


def closest(tokens, search_vec, limit, offset=<NUM_LIT:0>):

return sorted(tokens,<EOL>key=lambda x: cosine(search_vec, word_vec(x)),<EOL>reverse=True)[offset:offset+limit]<EOL>

Return the <limit> words from <tokens> whose vectors most closely
    resemble the search_vec. Skip the first <offset> results.


if not ethnicity: ethnicity = random.choice(get_ethnicities())<EOL>print("<STR_LIT>" + ethnicity)<EOL>filename = names_dir + ethnicity + '<STR_LIT>'<EOL>try:<EOL><INDENT>with open(filename, '<STR_LIT:r>') as injson:<EOL><INDENT>data = json.load(injson)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>return '<STR_LIT>' + filename<EOL><DEDENT>else:<EOL><INDENT>names = [ tuple(name.split('<STR_LIT:U+002C>')) for name in data ]<EOL>random.shuffle(names)<EOL>global female_names<EOL>female_names = [name for name,gender,*desc in names if gender == '<STR_LIT>']<EOL>global male_names<EOL>male_names = [name for name,gender,*desc in names if gender == '<STR_LIT>']<EOL>global nb_names<EOL>nb_names = [name for name,gender,*desc in names if gender == '<STR_LIT>']<EOL><DEDENT>

Set three globally available lists of names.


ethnicities = [ fname.split('<STR_LIT:.>')[<NUM_LIT:0>] for fname in os.listdir(names_dir) ]<EOL>return ethnicities<EOL>

Retrieve a list of the ethnicities for which name data was scraped.
    Exclude the file extension for human friendliness.


print('<STR_LIT>')<EOL>print('<STR_LIT:\n>'.join(get_ethnicities()))<EOL>print("<STR_LIT:\n>")<EOL>

Print a list of the ethnicities for which name data was scraped.


if hasattr(settings, '<STR_LIT>'):<EOL><INDENT>finders = settings.MEDIA_FIXTURES_FILES_FINDERS<EOL><DEDENT>else:<EOL><INDENT>finders = (<EOL>'<STR_LIT>',<EOL>)<EOL><DEDENT>for finder_path in finders:<EOL><INDENT>yield get_finder(finder_path)<EOL><DEDENT>

Set the media fixtures finders on settings.py.
Example:
    MEDIA_FIXTURES_FILES_FINDERS = (
        'django_media_fixtures.finders.FileSystemFinder',
        'django_media_fixtures.finders.AppDirectoriesFinder',  # default
    )


Finder = import_string(import_path)<EOL>if not issubclass(Finder, BaseFinder):<EOL><INDENT>raise ImproperlyConfigured('<STR_LIT>' %<EOL>(Finder, BaseFinder))<EOL><DEDENT>return Finder()<EOL>

Imports the media fixtures files finder class described by import_path, where
import_path is the full Python path to the class.


matches = []<EOL>for prefix, root in self.locations:<EOL><INDENT>if root not in searched_locations:<EOL><INDENT>searched_locations.append(root)<EOL><DEDENT>matched_path = self.find_location(root, path, prefix)<EOL>if matched_path:<EOL><INDENT>if not all:<EOL><INDENT>return matched_path<EOL><DEDENT>matches.append(matched_path)<EOL><DEDENT><DEDENT>return matches<EOL>

Looks for files in the extra locations
as defined in ``MEDIA_FIXTURES_FILES_DIRS``.


def find_location(self, root, path, prefix=None):

if prefix:<EOL><INDENT>prefix = '<STR_LIT>' % (prefix, os.sep)<EOL>if not path.startswith(prefix):<EOL><INDENT>return None<EOL><DEDENT>path = path[len(prefix):]<EOL><DEDENT>path = safe_join(root, path)<EOL>if os.path.exists(path):<EOL><INDENT>return path<EOL><DEDENT>

Finds a requested media file in a location, returning the found
absolute path (or ``None`` if no match).


for prefix, root in self.locations:<EOL><INDENT>storage = self.storages[root]<EOL>for path in utils.get_files(storage, ignore_patterns):<EOL><INDENT>yield path, storage<EOL><DEDENT><DEDENT>


for storage in six.itervalues(self.storages):<EOL><INDENT>if storage.exists('<STR_LIT>'):  <EOL><INDENT>for path in utils.get_files(storage, ignore_patterns):<EOL><INDENT>yield path, storage<EOL><DEDENT><DEDENT><DEDENT>


matches = []<EOL>for app in self.apps:<EOL><INDENT>app_location = self.storages[app].location<EOL>if app_location not in searched_locations:<EOL><INDENT>searched_locations.append(app_location)<EOL><DEDENT>match = self.find_in_app(app, path)<EOL>if match:<EOL><INDENT>if not all:<EOL><INDENT>return match<EOL><DEDENT>matches.append(match)<EOL><DEDENT><DEDENT>return matches<EOL>


storage = self.storages.get(app, None)<EOL>if storage:<EOL><INDENT>if storage.exists(path):<EOL><INDENT>matched_path = storage.path(path)<EOL>if matched_path:<EOL><INDENT>return matched_path<EOL><DEDENT><DEDENT><DEDENT>

Find a requested media file in an app's media fixtures locations.