original = self.current_window<EOL>if window != original:<EOL><INDENT>self.switch_to_window(window)<EOL><DEDENT>self._scopes.append(None)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>self._scopes.pop()<EOL>if original != window:<EOL><INDENT>self.switch_to_window(original)<EOL><DEDENT><DEDENT>

This method does the following:

1. Switches to the given window (it can be located by window instance/lambda/string).
2. Executes the given block (within window located at previous step).
3. Switches back (this step will be invoked even if exception happens at second step).

Args:
    window (Window | lambda): The desired :class:`Window`, or a lambda that will be run in
        the context of each open window and returns ``True`` for the desired window.


@contextmanager<EOL><INDENT>def scope(self, *args, **kwargs):<DEDENT>

new_scope = args[<NUM_LIT:0>] if isinstance(args[<NUM_LIT:0>], Base) else self.find(*args, **kwargs)<EOL>self._scopes.append(new_scope)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>self._scopes.pop()<EOL><DEDENT>

Executes the wrapped code within the context of a node. ``scope`` takes the same options
as :meth:`find`. For the duration of the context, any command to Capybara will be handled
as though it were scoped to the given element. ::

    with scope("xpath", "//div[@id='delivery-address']"):
        fill_in("Street", value="12 Main Street")

Just as with :meth:`find`, if multiple elements match the selector given to ``scope``, an
error will be raised, and just as with :meth:`find`, this behavior can be controlled
through the ``match`` and ``exact`` options.

It is possible to omit the first argument, in that case, the selector is assumed to be of
the type set in :data:`capybara.default_selector`. ::

    with scope("div#delivery-address"):
        fill_in("Street", value="12 Main Street")

Note that a lot of uses of ``scope`` can be replaced more succinctly with chaining::

    find("div#delivery-address").fill_in("Street", value="12 Main Street")

Args:
    *args: Variable length argument list for the call to :meth:`find`.
    **kwargs: Arbitrary keywords arguments for the call to :meth:`find`.


if isinstance(frame, Element):<EOL><INDENT>self.driver.switch_to_frame(frame)<EOL>self._scopes.append("<STR_LIT>")<EOL><DEDENT>elif frame == "<STR_LIT>":<EOL><INDENT>if self._scopes[-<NUM_LIT:1>] != "<STR_LIT>":<EOL><INDENT>raise ScopeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._scopes.pop()<EOL>self.driver.switch_to_frame("<STR_LIT>")<EOL><DEDENT>elif frame == "<STR_LIT>":<EOL><INDENT>if "<STR_LIT>" in self._scopes:<EOL><INDENT>idx = self._scopes.index("<STR_LIT>")<EOL>if any([scope not in ["<STR_LIT>", None] for scope in self._scopes[idx:]]):<EOL><INDENT>raise ScopeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>self._scopes = self._scopes[:idx]<EOL>self.driver.switch_to_frame("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Switch to the given frame.

If you use this method you are responsible for making sure you switch back to the parent
frame when done in the frame changed to. :meth:`frame` is preferred over this method and
should be used when possible. May not be supported by all drivers.

Args:
    frame (Element | str): The iframe/frame element to switch to.


if not self.current_url:<EOL><INDENT>return<EOL><DEDENT>path = urlparse(self.current_url).path<EOL>return path if path else None<EOL>

str: Path of the current page, without any domain information.


ret = []<EOL>for tok in tokens:<EOL><INDENT>if "<STR_LIT:U+0020>" in tok:<EOL><INDENT>tok = '<STR_LIT>' % tok<EOL><DEDENT>if "<STR_LIT:;>" in tok:<EOL><INDENT>tok = tok.replace("<STR_LIT:;>", "<STR_LIT>")<EOL><DEDENT>ret.append(tok)<EOL><DEDENT>return "<STR_LIT:U+0020>".join(ret)<EOL>

Serialize tokens:
* quote whitespace-containing tokens
* escape semicolons


json_zone_file = defaultdict(list)<EOL>record_lines = text.split("<STR_LIT:\n>")<EOL>parser = make_parser()<EOL>for record_line in record_lines:<EOL><INDENT>record_token = tokenize_line(record_line)<EOL>try:<EOL><INDENT>json_zone_file = parse_line(parser, record_token, json_zone_file)<EOL><DEDENT>except InvalidLineException:<EOL><INDENT>if ignore_invalid:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT><DEDENT>return json_zone_file<EOL>

Parse a zonefile into a dict.
@text must be flattened--each record must be on one line.
Also, all comments must be removed.


<EOL>lines = text.split("<STR_LIT:\n>")<EOL>ret = []<EOL>for line in lines:<EOL><INDENT>tokens = tokenize_line(line)<EOL>tokens_upper = [t.upper() for t in tokens]<EOL>if "<STR_LIT>" in tokens_upper:<EOL><INDENT>tokens.remove("<STR_LIT>")<EOL><DEDENT>elif "<STR_LIT>" in tokens_upper:<EOL><INDENT>tokens.remove("<STR_LIT>")<EOL><DEDENT>elif "<STR_LIT>" in tokens_upper:<EOL><INDENT>tokens.remove("<STR_LIT>")<EOL><DEDENT>elif "<STR_LIT>" in tokens_upper:<EOL><INDENT>tokens.remove("<STR_LIT>")<EOL><DEDENT>ret.append(serialize(tokens))<EOL><DEDENT>return "<STR_LIT:\n>".join(ret)<EOL>

Remove the CLASS from each DNS record, if present.
The only class that gets used today (for all intents
and purposes) is 'IN'.


return process_rr(data, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", template)<EOL>

Replace {cname} in template with the serialized CNAME records


if data is None:<EOL><INDENT>to_process = None<EOL><DEDENT>else:<EOL><INDENT>to_process = copy.deepcopy(data)<EOL>for datum in to_process:<EOL><INDENT>if isinstance(datum["<STR_LIT>"], list):<EOL><INDENT>datum["<STR_LIT>"] = "<STR_LIT:U+0020>".join(['<STR_LIT>' % entry.replace("<STR_LIT:;>", "<STR_LIT>")<EOL>for entry in datum["<STR_LIT>"]])<EOL><DEDENT>else:<EOL><INDENT>datum["<STR_LIT>"] = '<STR_LIT>' % datum["<STR_LIT>"].replace("<STR_LIT:;>", "<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>return process_rr(to_process, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", template)<EOL>

Replace {txt} in template with the serialized TXT records


return process_rr(data, "<STR_LIT>", "<STR_LIT:host>", "<STR_LIT>", template)<EOL>

Replace {ns} in template with the serialized NS records


Resource = apps.get_model("<STR_LIT>", "<STR_LIT>")<EOL>Resource.objects.all().delete()<EOL>TypeResource = apps.get_model("<STR_LIT>", "<STR_LIT>")<EOL>TypeResource.objects.all().delete()<EOL>

Brutally deleting all entries for this model...


bits = token.contents.split('<STR_LIT:U+0020>')<EOL>if len(bits) <= <NUM_LIT:1>:<EOL><INDENT>raise template.TemplateSyntaxError("<STR_LIT>" % bits[<NUM_LIT:0>])<EOL><DEDENT>if token.contents[<NUM_LIT>] == '<STR_LIT:">':<EOL><INDENT>end_quote = token.contents.index('<STR_LIT:">', <NUM_LIT>) + <NUM_LIT:1><EOL>args = [template.Variable(token.contents[<NUM_LIT>:end_quote])]<EOL>kwargs_start = end_quote<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>next_space = token.contents.index('<STR_LIT:U+0020>', <NUM_LIT>)<EOL>kwargs_start = next_space + <NUM_LIT:1><EOL><DEDENT>except ValueError:<EOL><INDENT>next_space = None<EOL>kwargs_start = None<EOL><DEDENT>args = [template.Variable(token.contents[<NUM_LIT>:next_space])]<EOL><DEDENT>kwargs = {}<EOL>kwargs_list = token.contents[kwargs_start:].split('<STR_LIT:U+002C>')<EOL>for kwargs_item in kwargs_list:<EOL><INDENT>if '<STR_LIT:=>' in kwargs_item:<EOL><INDENT>k, v = kwargs_item.split('<STR_LIT:=>', <NUM_LIT:1>)<EOL>k = k.strip()<EOL>kwargs[k] = template.Variable(v)<EOL><DEDENT><DEDENT>return args, kwargs<EOL>

Parse uniformly args and kwargs from a templatetag

Usage::

  For parsing a template like this:

  {% footag my_contents,height=10,zoom=20 as myvar %}

  You simply do this:

  @register.tag
  def footag(parser, token):
      args, kwargs = parse_args_kwargs(parser, token)


def prepare_context(self, args, kwargs, context):

return context<EOL>

Hook for overriding in subclasses.

Note that "args" and "kwargs" parameters are already resolved with context


return getattr(settings, '<STR_LIT>', get_static_url() + "<STR_LIT>")<EOL>

Return the ADMIN_MEDIA_PREFIX if it is in the settings.py else get
the static url from the previous function and add /admin/.


obj_dict = obj.__dict__<EOL>obj_dict_result = obj_dict.copy()<EOL>for key, value in obj_dict.items():<EOL><INDENT>if key.endswith('<STR_LIT>'):<EOL><INDENT>key2 = key.replace('<STR_LIT>', '<STR_LIT>')<EOL>try:<EOL><INDENT>field, model, direct, m2m = obj._meta.get_field_by_name(key2)<EOL>if isinstance(field, ForeignKey):<EOL><INDENT>obj_dict_result[key2] = obj_dict_result[key]<EOL>del obj_dict_result[key]<EOL><DEDENT><DEDENT>except FieldDoesNotExist:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>manytomany_list = obj._meta.many_to_many<EOL>for manytomany in manytomany_list:<EOL><INDENT>pks = [obj_rel.pk for obj_rel in manytomany.value_from_object(obj).select_related()]<EOL>if pks:<EOL><INDENT>obj_dict_result[manytomany.name] = pks<EOL><DEDENT><DEDENT>return obj_dict_result<EOL>

Edit to get the dict even when the object is a GenericRelatedObjectManager.
Added the try except.


return [[w.rstrip() for w in sent] for sent in sentences]<EOL>

Clear out spaces and newlines
from the list of list of strings.

Arguments:
----------
    sentences : list<list<str>>

Returns:
--------
    list<list<str>> : same strings as input,
        without spaces or newlines.


word_matches = list(re.finditer(word_with_period, text))<EOL>total_words = len(word_matches)<EOL>for i, match in enumerate(word_matches):<EOL><INDENT>match_start = match.start()<EOL>match_end = match.end()<EOL>for char_pos in range(match_start, match_end):<EOL><INDENT>if split_locations[char_pos] == SHOULD_SPLIT and match_end - char_pos > <NUM_LIT:1>:<EOL><INDENT>match_start = char_pos<EOL><DEDENT><DEDENT>word = text[match_start:match_end]<EOL>if not word.endswith('<STR_LIT:.>'):<EOL><INDENT>if (not word[<NUM_LIT:0>].isdigit() and<EOL>split_locations[match_start] == UNDECIDED):<EOL><INDENT>split_locations[match_start] = SHOULD_SPLIT<EOL><DEDENT>continue<EOL><DEDENT>period_pos = match_end - <NUM_LIT:1><EOL>word_is_in_abbr = word[:-<NUM_LIT:1>].lower() in ABBR<EOL>is_abbr_like = (<EOL>word_is_in_abbr or<EOL>one_letter_long_or_repeating.match(word[:-<NUM_LIT:1>]) is not None<EOL>)<EOL>is_digit = False if is_abbr_like else word[:-<NUM_LIT:1>].isdigit()<EOL>is_last_word = i == (total_words - <NUM_LIT:1>)<EOL>is_ending = is_last_word and (match_end == len(text) or text[match_end:].isspace())<EOL>is_not_ending = not is_ending<EOL>abbreviation_and_not_end = (<EOL>len(word) > <NUM_LIT:1> and<EOL>is_abbr_like and<EOL>is_not_ending<EOL>)<EOL>if abbreviation_and_not_end and (<EOL>(not is_last_word and word_matches[i+<NUM_LIT:1>].group(<NUM_LIT:0>)[<NUM_LIT:0>].islower()) or<EOL>(not is_last_word and word_matches[i+<NUM_LIT:1>].group(<NUM_LIT:0>) in PUNCT_SYMBOLS) or<EOL>word[<NUM_LIT:0>].isupper() or<EOL>word_is_in_abbr or<EOL>len(word) == <NUM_LIT:2>):<EOL><INDENT>if split_locations[period_pos] == SHOULD_SPLIT and period_pos + <NUM_LIT:1> < len(split_locations):<EOL><INDENT>split_locations[period_pos + <NUM_LIT:1>] = SHOULD_SPLIT<EOL><DEDENT>split_locations[period_pos] = SHOULD_NOT_SPLIT<EOL><DEDENT>elif (is_digit and<EOL>len(word[:-<NUM_LIT:1>]) <= <NUM_LIT:2> and<EOL>not is_last_word and<EOL>word_matches[i+<NUM_LIT:1>].group(<NUM_LIT:0>).lower() in MONTHS):<EOL><INDENT>if split_locations[period_pos] == SHOULD_SPLIT and period_pos + <NUM_LIT:1> < len(split_locations):<EOL><INDENT>split_locations[period_pos + <NUM_LIT:1>] = SHOULD_SPLIT<EOL><DEDENT>split_locations[period_pos] = SHOULD_NOT_SPLIT<EOL><DEDENT>elif split_locations[period_pos] == UNDECIDED:<EOL><INDENT>split_locations[period_pos] = SHOULD_SPLIT<EOL><DEDENT><DEDENT>

Annotate locations in a string that contain
periods as being true periods or periods
that are a part of shorthand (and thus should
not be treated as punctuation marks).

Arguments:
----------
    text : str
    split_locations : list<int>, same length as text.


def mark_begin_end_regex(regex, text, split_locations):

for match in regex.finditer(text):<EOL><INDENT>end_match = match.end()<EOL>begin_match = match.start()<EOL>for i in range(begin_match+<NUM_LIT:1>, end_match):<EOL><INDENT>split_locations[i] = SHOULD_NOT_SPLIT<EOL><DEDENT>if end_match < len(split_locations):<EOL><INDENT>if split_locations[end_match] == UNDECIDED:<EOL><INDENT>split_locations[end_match] = SHOULD_SPLIT<EOL><DEDENT><DEDENT>if split_locations[begin_match] == UNDECIDED:<EOL><INDENT>split_locations[begin_match] = SHOULD_SPLIT<EOL><DEDENT><DEDENT>

Regex that adds a 'SHOULD_SPLIT' marker at the end
location of each matching group of the given regex,
and adds a 'SHOULD_SPLIT' at the beginning of the
matching group. Each character within the matching
group will be marked as 'SHOULD_NOT_SPLIT'.

Arguments
---------
    regex : re.Expression
    text : str, same length as split_locations
    split_locations : list<int>, split decisions.


s3url = S3URL(source)<EOL>message('<STR_LIT>', source)<EOL>if not self.opt.dry_run:<EOL><INDENT>self.s3.delete_object(Bucket=s3url.bucket, Key=s3url.path)<EOL><DEDENT>


assert(type(sources) == list)<EOL>if len(sources) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>elif len(sources) == <NUM_LIT:1>:<EOL><INDENT>self.delete(sources[<NUM_LIT:0>])<EOL><DEDENT>elif len(sources) > self.opt.batch_delete_size:<EOL><INDENT>for i in range(<NUM_LIT:0>, len(sources), self.opt.batch_delete_size):<EOL><INDENT>self.pool.batch_delete(sources[i:i+self.opt.batch_delete_size])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>bucket = S3URL(sources[<NUM_LIT:0>]).bucket<EOL>deletes = []<EOL>for source in sources:<EOL><INDENT>s3url = S3URL(source)<EOL>if s3url.bucket != bucket:<EOL><INDENT>raise Failure('<STR_LIT>' % (s3url.bucket, bucket))<EOL><DEDENT>deletes.append({'<STR_LIT>': s3url.path})<EOL><DEDENT>response = self.s3.delete_objects(Bucket=bucket, Delete={'<STR_LIT>': deletes})<EOL>for res in response.get('<STR_LIT>') or []:<EOL><INDENT>message('<STR_LIT>', S3URL.combine('<STR_LIT>', bucket, res['<STR_LIT>']))<EOL><DEDENT>for err in response.get('<STR_LIT>') or []:<EOL><INDENT>message('<STR_LIT>', S3URL.combine('<STR_LIT>', bucket, res['<STR_LIT>']), err['<STR_LIT>'], err['<STR_LIT>'])<EOL><DEDENT>if response.get('<STR_LIT>') is not None:<EOL><INDENT>raise RetryFailure('<STR_LIT>' % len(response.get('<STR_LIT>')))<EOL><DEDENT><DEDENT>

Delete a list of files in batch of batch_delete_size (default=1000).


try:<EOL><INDENT>return self.s3.head_object(Bucket=s3url.bucket, Key=s3url.path)<EOL><DEDENT>except BotoClient.ClientError as e:<EOL><INDENT>if e.response['<STR_LIT>']['<STR_LIT>'] == <NUM_LIT>:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>

Get the s3 object with the S3 URL. Return None if not exist.


self.filename = filename<EOL>self.md5 = None<EOL>


@log_calls<EOL><INDENT>def get_files(self, source, target):<DEDENT>

pool = ThreadPool(ThreadUtil, self.opt)<EOL>source = self.source_expand(source)<EOL>if os.path.isdir(target):<EOL><INDENT>for src in source:<EOL><INDENT>self.get_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(source) > <NUM_LIT:1>:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>elif len(source) == <NUM_LIT:1>:<EOL><INDENT>self.get_single_file(pool, source[<NUM_LIT:0>], target)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>pool.join()<EOL>

Download files.
           This function can handle multiple files if source S3 URL has wildcard
           characters. It also handles recursive mode by download all files and
           keep the directory structure.


return S3URL.combine(self.proto, self.bucket, self.path)<EOL>


if path[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>path = path[<NUM_LIT:0>:-<NUM_LIT:1>]<EOL><DEDENT>return os.path.basename(path)<EOL>

Unix style basename.
           This fuction will return 'bar' for '/foo/bar/' instead of empty string.
           It is used to normalize the input trailing slash.


@log_calls<EOL><INDENT>def write_file_chunk(self, target, pos, chunk, body):<DEDENT>

fd = os.open(target, os.O_CREAT | os.O_WRONLY)<EOL>try:<EOL><INDENT>os.lseek(fd, pos, os.SEEK_SET)<EOL>data = body.read(chunk)<EOL>num_bytes_written = os.write(fd, data)<EOL>if(num_bytes_written != len(data)):<EOL><INDENT>raise RetryFailure('<STR_LIT>' % (num_bytes_written, sys.getsizeof(data)))<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>os.close(fd)<EOL><DEDENT>


result = []<EOL>if S3URL.is_valid(dir):<EOL><INDENT>basepath = S3URL(dir).path<EOL>for f in (f for f in self.s3walk(dir) if not f['<STR_LIT>']):<EOL><INDENT>result.append(os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for f in (f for f in self.local_walk(dir) if not os.path.isdir(f)):<EOL><INDENT>result.append(os.path.relpath(f, dir))<EOL><DEDENT><DEDENT>return result<EOL>

Generic version of directory walk. Return file list without base path
           for comparison.


if len(args) == <NUM_LIT:1>:<EOL><INDENT>self.pretty_print(self.s3handler().list_buckets())<EOL>return<EOL><DEDENT>self.validate('<STR_LIT>', args)<EOL>self.pretty_print(self.s3handler().s3walk(args[<NUM_LIT:1>]))<EOL>


def merge_opt_params(self, method, kargs):

for key in self.legal_params[method]:<EOL><INDENT>if not hasattr(self.opt, key) or getattr(self.opt, key) is None:<EOL><INDENT>continue<EOL><DEDENT>if key in kargs and type(kargs[key]) == dict:<EOL><INDENT>assert(type(getattr(self.opt, key)) == dict)<EOL>for k, v in getattr(self.opt, key).iteritems():<EOL><INDENT>kargs[key][k] = v<EOL><DEDENT><DEDENT>else:<EOL><INDENT>kargs[key] = getattr(self.opt, key)<EOL><DEDENT><DEDENT>return kargs<EOL>

Combine existing parameters with extra options supplied from command line
           options. Carefully merge special type of parameter if needed.


@log_calls<EOL><INDENT>def download(self, source, target, mpi=None, pos=<NUM_LIT:0>, chunk=<NUM_LIT:0>, part=<NUM_LIT:0>):<DEDENT>

s3url = S3URL(source)<EOL>obj = self.lookup(s3url)<EOL>if obj is None:<EOL><INDENT>raise Failure('<STR_LIT>' % (s3url.path,))<EOL><DEDENT>if not mpi:<EOL><INDENT>if self.opt.dry_run:<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif self.opt.sync_check and self.sync_check(LocalMD5Cache(target), obj):<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif not self.opt.force and os.path.exists(target):<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>fsize = int(obj['<STR_LIT>'])<EOL>if fsize < self.opt.max_singlepart_download_size:<EOL><INDENT>mpi = ThreadUtil.MultipartItem(tempfile_get(target))<EOL>mpi.total = <NUM_LIT:1><EOL>pos = <NUM_LIT:0><EOL>chunk = fsize<EOL><DEDENT>else:<EOL><INDENT>for args in self.get_file_splits(tempfile_get(target), source, target, fsize, self.opt.multipart_split_size):<EOL><INDENT>self.pool.download(*args)<EOL><DEDENT>return<EOL><DEDENT><DEDENT>tempfile = mpi.id<EOL>if self.opt.recursive:<EOL><INDENT>self.mkdirs(tempfile)<EOL><DEDENT>response = self.s3.get_object(Bucket=s3url.bucket, Key=s3url.path, Range='<STR_LIT>' % (pos, pos + chunk - <NUM_LIT:1>))<EOL>self.write_file_chunk(tempfile, pos, chunk, response['<STR_LIT>'])<EOL>if mpi.complete({'<STR_LIT>': part}):<EOL><INDENT>try:<EOL><INDENT>self.update_privilege(obj, tempfile)<EOL>self._verify_file_size(obj, tempfile)<EOL>tempfile_set(tempfile, target)<EOL>message('<STR_LIT>', source, target)<EOL><DEDENT>except Exception as e:<EOL><INDENT>tempfile_set(tempfile, None)<EOL>raise Failure('<STR_LIT>' % (e.message, source))<EOL><DEDENT><DEDENT>


s3url = S3URL(source)<EOL>message('<STR_LIT>', source)<EOL>if not self.opt.dry_run:<EOL><INDENT>resp = self.s3.create_bucket(Bucket=s3url.bucket)<EOL>if resp['<STR_LIT>']["<STR_LIT>"] == <NUM_LIT:200>:<EOL><INDENT>message('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise Failure('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>

Use the create_bucket API to create a new bucket


self.opt = opt<EOL>


@log_calls<EOL><INDENT>def _verify_file_size(self, obj, downloaded_file):<DEDENT>

file_size = os.path.getsize(downloaded_file)<EOL>if int(obj['<STR_LIT>']) != file_size:<EOL><INDENT>raise RetryFailure('<STR_LIT>' % (repr(obj)))<EOL><DEDENT>

Verify the file size of the downloaded file.


result = []<EOL>for root, dirs, files in os.walk(basedir):<EOL><INDENT>for f in files:<EOL><INDENT>result.append(os.path.join(root, f))<EOL><DEDENT><DEDENT>return result<EOL>

Walk through local directories from root basedir


func.__lock__ = threading.Lock()<EOL>def synced_func(*args, **kargs):<EOL><INDENT>with func.__lock__:<EOL><INDENT>return func(*args, **kargs)<EOL><DEDENT><DEDENT>return synced_func<EOL>


<EOL>if len(args) == <NUM_LIT:2>:<EOL><INDENT>args += ['<STR_LIT:.>']<EOL><DEDENT>self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().get_files(source, target)<EOL>


<EOL>if not (sys.stdout.isatty() and sys.stderr.isatty()):<EOL><INDENT>return<EOL><DEDENT>text = (msg % args)<EOL>if progress.prev_message:<EOL><INDENT>sys.stderr.write('<STR_LIT:U+0020>' * len(progress.prev_message) + '<STR_LIT:\r>')<EOL><DEDENT>sys.stderr.write(text + '<STR_LIT:\r>')<EOL>progress.prev_message = text<EOL>

Show current progress message to stderr.
       This function will remember the previous message so that next time,
       it will clear the previous message before showing next one.


@log_calls<EOL><INDENT>def sync_files(self, source, target):<DEDENT>

src_s3_url = S3URL.is_valid(source)<EOL>dst_s3_url = S3URL.is_valid(target)<EOL>if src_s3_url and not dst_s3_url:<EOL><INDENT>self.get_files(source, target)<EOL><DEDENT>elif not src_s3_url and dst_s3_url:<EOL><INDENT>self.put_files(source, target)<EOL>if self.opt.delete_removed:<EOL><INDENT>self.delete_removed_files(source, target)<EOL><DEDENT><DEDENT>elif src_s3_url and dst_s3_url:<EOL><INDENT>self.cp_files(source, target)<EOL><DEDENT>else:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>

Sync files to S3. Does implement deletions if syncing TO s3.
           Currently identical to get/put -r -f --sync-check with exception of deletions.


m = self.REGEX_TIME.search(value)<EOL>time = datetime.datetime.utcnow().time()<EOL>if m:<EOL><INDENT>time = datetime.time(int(m.group(<NUM_LIT:1>)), int(m.group(<NUM_LIT:2>)))<EOL>value = self.REGEX_TIME.sub('<STR_LIT>', value)<EOL><DEDENT>return (time, value)<EOL>

Search for time information in the string


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>self.s3handler().del_files(source)<EOL>


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>self.s3handler().print_files(source)<EOL>


self.opt = opt<EOL>self.tasks = TaskQueue()<EOL>self.processed_tasks = <NUM_LIT:0><EOL>self.thread_class = thread_class<EOL>self.workers = []<EOL>for i in range(opt.num_threads):<EOL><INDENT>self.workers.append(thread_class(self))<EOL><DEDENT>

Constructor of ThreadPool.
           Create workers and pool will automatically inherit all methods from
           thread_class by redirecting calls through __getattribute__().


@staticmethod<EOL><INDENT>def combine(proto, bucket, path):<DEDENT>

return '<STR_LIT>' % (proto, bucket, path)<EOL>

Combine each component and general a S3 url string, no path normalization
           here. The path should not start with slash.


self.s3 = None<EOL>self.opt = opt<EOL>self.connect()<EOL>


fileonly = (self.opt.last_modified_before is not None) or (self.opt.last_modified_after is not None)<EOL>if obj['<STR_LIT>']:<EOL><INDENT>if not fileonly:<EOL><INDENT>result.append(obj)<EOL><DEDENT>return<EOL><DEDENT>if (self.opt.last_modified_before is not None) and obj['<STR_LIT>'] >= self.opt.last_modified_before:<EOL><INDENT>return<EOL><DEDENT>if (self.opt.last_modified_after is not None) and obj['<STR_LIT>'] <= self.opt.last_modified_after:<EOL><INDENT>return<EOL><DEDENT>result.append(obj)<EOL>

Check all file item with given conditions.


def normalize_time(timestamp):<EOL><INDENT>'''<STR_LIT>'''<EOL>if timestamp is None:<EOL><INDENT>return '<STR_LIT:U+0020>' * <NUM_LIT:16><EOL><DEDENT>return TIMESTAMP_FORMAT % (timestamp.year, timestamp.month, timestamp.day, timestamp.hour, timestamp.minute)<EOL><DEDENT>cwidth = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>format = '<STR_LIT>'<EOL>result = []<EOL>for obj in objlist:<EOL><INDENT>last_modified = normalize_time(obj['<STR_LIT>'])<EOL>size = str(obj['<STR_LIT:size>']) if not obj['<STR_LIT>'] else '<STR_LIT>'<EOL>name = obj['<STR_LIT:name>']<EOL>item = (last_modified, size, name)<EOL>for i, value in enumerate(item):<EOL><INDENT>if cwidth[i] < len(value):<EOL><INDENT>cwidth[i] = len(value)<EOL><DEDENT><DEDENT>result.append(item)<EOL><DEDENT>for item in result:<EOL><INDENT>text = (format % tuple(cwidth)) % item<EOL>message('<STR_LIT:%s>', text.rstrip())<EOL><DEDENT>

Pretty print the result of s3walk. Here we calculate the maximum width
           of each column and align them.


@log_calls<EOL><INDENT>def dsync_files(self, source, target):<DEDENT>

src_s3_url = S3URL.is_valid(source)<EOL>dst_s3_url = S3URL.is_valid(target)<EOL>source_list = self.relative_dir_walk(source)<EOL>if len(source_list) == <NUM_LIT:0> or '<STR_LIT:.>' in source_list:<EOL><INDENT>raise Failure('<STR_LIT>')<EOL><DEDENT>sync_list = [(os.path.join(source, f), os.path.join(target, f)) for f in source_list]<EOL>pool = ThreadPool(ThreadUtil, self.opt)<EOL>if src_s3_url and not dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.download(src, dest)<EOL><DEDENT><DEDENT>elif not src_s3_url and dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.upload(src, dest)<EOL><DEDENT><DEDENT>elif src_s3_url and dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.copy(src, dest)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>pool.join()<EOL>if self.opt.delete_removed:<EOL><INDENT>target_list = self.relative_dir_walk(target)<EOL>remove_list = [os.path.join(target, f) for f in (set(target_list) - set(source_list))]<EOL>if S3URL.is_valid(target):<EOL><INDENT>pool = ThreadPool(ThreadUtil, self.opt)<EOL>pool.batch_delete(remove_list)<EOL>pool.join()<EOL><DEDENT>else:<EOL><INDENT>for f in remove_list:<EOL><INDENT>try:<EOL><INDENT>os.unlink(f)<EOL>message('<STR_LIT>', f)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


@log_calls<EOL><INDENT>def cp_single_file(self, pool, source, target, delete_source):<DEDENT>

if source[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>if self.opt.recursive:<EOL><INDENT>basepath = S3URL(source).path<EOL>for f in (f for f in self.s3walk(source) if not f['<STR_LIT>']):<EOL><INDENT>pool.copy(f['<STR_LIT:name>'], os.path.join(target, os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath)), delete_source=delete_source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>message('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pool.copy(source, target, delete_source=delete_source)<EOL><DEDENT>

Copy a single file or a directory by adding a task into queue


@log_calls<EOL><INDENT>def delete_removed_files(self, source, target):<DEDENT>

message("<STR_LIT>", source, target)<EOL>if os.path.isdir(source):<EOL><INDENT>unecessary = []<EOL>basepath = S3URL(target).path<EOL>for f in [f for f in self.s3walk(target) if not f['<STR_LIT>']]:<EOL><INDENT>local_name = os.path.join(source, os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath))<EOL>if not os.path.isfile(local_name):<EOL><INDENT>message("<STR_LIT>", local_name)<EOL>unecessary.append(f['<STR_LIT:name>'])<EOL><DEDENT><DEDENT>if len(unecessary) > <NUM_LIT:0>:<EOL><INDENT>pool = ThreadPool(ThreadUtil, self.opt)<EOL>for del_file in unecessary:<EOL><INDENT>pool.delete(del_file)<EOL><DEDENT>pool.join()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>

Remove remote files that are not present in the local source.
           (Obsolete) It is used for old sync command now.


@log_calls<EOL><INDENT>def read_file_chunk(self, source, pos, chunk):<DEDENT>

if chunk==<NUM_LIT:0>:<EOL><INDENT>return StringIO()<EOL><DEDENT>data = None<EOL>with open(source, '<STR_LIT:rb>') as f:<EOL><INDENT>f.seek(pos)<EOL>data = f.read(chunk)<EOL><DEDENT>if not data:<EOL><INDENT>raise Failure('<STR_LIT>' % source)<EOL><DEDENT>return StringIO(data)<EOL>


@log_calls<EOL><INDENT>def sync_check(self, md5cache, remoteKey):<DEDENT>

if not remoteKey:<EOL><INDENT>return False<EOL><DEDENT>if not os.path.exists(md5cache.filename):<EOL><INDENT>return False<EOL><DEDENT>localmd5 = md5cache.get_md5()<EOL>return ('<STR_LIT>' in remoteKey and remoteKey['<STR_LIT>'] == '<STR_LIT>' % localmd5) or('<STR_LIT>' in remoteKey and remoteKey['<STR_LIT>'] == localmd5) or('<STR_LIT>' in remoteKey['<STR_LIT>'] and remoteKey['<STR_LIT>']['<STR_LIT>'] == localmd5)<EOL>

Check MD5 for a local file and a remote file.
           Return True if they have the same md5 hash, otherwise False.


raise NotImplementedError('<STR_LIT>')<EOL>

It will search for "song_name website"  in the Search Engine results
and return the url of website


file_name=url.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>if os.path.exists(os.path.join(os.getcwd(),file_name)):<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>try:<EOL><INDENT>r=requests.get(url,stream=True,timeout=<NUM_LIT:200>)<EOL><DEDENT>except requests.exceptions.SSLError:<EOL><INDENT>try:<EOL><INDENT>response=requests.get(url,stream=True,verify=False,timeout=<NUM_LIT:200>)<EOL><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e)<EOL>quit()  <EOL><DEDENT><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e)<EOL>quit() <EOL><DEDENT>chunk_size = <NUM_LIT><EOL>total_size = int(r.headers['<STR_LIT>'])<EOL>total_chunks = total_size/chunk_size<EOL>file_iterable = r.iter_content(chunk_size = chunk_size)<EOL>tqdm_iter = tqdm(iterable = file_iterable,total = total_chunks,unit = '<STR_LIT>',<EOL>leave = False<EOL>)  <EOL>with open(file_name,'<STR_LIT:wb>') as f:<EOL><INDENT>for data in tqdm_iter:<EOL><INDENT>f.write(data)<EOL><DEDENT><DEDENT>'''<STR_LIT>'''<EOL>print('<STR_LIT>'%file_name)<EOL>

It will download file specified by url using requests module


def _spark_predict(self, cls, X, *args, **kwargs):

return X.map(lambda X: super(cls, self).predict(X, *args, **kwargs))<EOL>

Wraps a Scikit-learn Linear model's predict method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : ArrayRDD
            The distributed data to predict in a DictRDD.

        Returns
        -------
        self: the wrapped class


return self if other == <NUM_LIT:0> else self.__add__(other)<EOL>

Reverse add method for Linear models.

        Parameters
        ----------
        other : fitted sklearn linear model
            Model to add.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


self.coef_ /= other<EOL>self.intercept_ /= other<EOL>return self<EOL>

Division method for Linear models. Used for averaging.

        Parameters
        ----------
        other : integer
            Integer to divide with.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


model = copy.deepcopy(self)<EOL>model.coef_ += other.coef_<EOL>model.intercept_ += other.intercept_<EOL>return model<EOL>

Add method for Linear models with coef and intercept attributes.

        Parameters
        ----------
        other : fitted sklearn linear model
            Model to add.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


def _spark_fit(self, cls, Z, *args, **kwargs):

mapper = lambda X_y: super(cls, self).fit(<EOL>X_y[<NUM_LIT:0>], X_y[<NUM_LIT:1>], *args, **kwargs<EOL>)<EOL>models = Z.map(mapper)<EOL>avg = models.reduce(operator.add) / models.count()<EOL>self.__dict__.update(avg.__dict__)<EOL>return self<EOL>

Wraps a Scikit-learn Linear model's fit method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : TupleRDD or DictRDD
            The distributed train data in a DictRDD.

        Returns
        -------
        self: the wrapped class


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return self._spark_predict(SparkSGDClassifier, X)<EOL>

Distributed method to predict class labels for samples in X.

        Parameters
        ----------
        X : ArrayRDD containing {array-like, sparse matrix}
            Samples.

        Returns
        -------
        C : ArrayRDD
            Predicted class label per sample.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>self._classes_ = np.unique(classes)<EOL>return self._spark_fit(SparkLogisticRegression, Z)<EOL>

Fit the model according to the given training data.

        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector
            y - Target labels
        classes : iterable
            The set of available classes

        Returns
        -------
        self : object
            Returns self.


<EOL>c = blocked_rdd.map(lambda x: (x.T.dot(x), x.shape[<NUM_LIT:0>]))<EOL>prod, n = c.reduce(lambda x, y: (x[<NUM_LIT:0>] + y[<NUM_LIT:0>], x[<NUM_LIT:1>] + y[<NUM_LIT:1>]))<EOL>w, v = ln.eig(prod / n)<EOL>w = np.real(w)<EOL>v = np.real(v)<EOL>inds = np.argsort(w)[::-<NUM_LIT:1>]<EOL>s = np.sqrt(w[inds[<NUM_LIT:0>:k]]) * np.sqrt(n)<EOL>v = v[:, inds[<NUM_LIT:0>:k]].T<EOL>u = blocked_rdd.map(lambda x: np.inner(x, v) / s)<EOL>return u, s, v<EOL>

Calculate the SVD of a blocked RDD directly, returning only the leading k
singular vectors. Assumes n rows and d columns, efficient when n >> d
Must be able to fit d^2 within the memory of a single machine.
Parameters
----------
blocked_rdd : RDD
    RDD with data points in numpy array blocks
k : Int
    Number of singular vectors to return
Returns
----------
u : RDD of blocks
    Left eigenvectors
s : numpy array
    Singular values
v : numpy array
    Right eigenvectors


self.fit_transform(Z)<EOL>return self<EOL>

Fit LSI model on training data X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self : object
            Returns the transformer object.


check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>if isinstance(X, ArrayRDD):<EOL><INDENT>X = X.unblock()<EOL><DEDENT>return X.map(lambda x: self._mllib_model.predict(x))<EOL><DEDENT>else:<EOL><INDENT>rdd = X.map(lambda X: super(SparkKMeans, self).predict(X))<EOL>return ArrayRDD(rdd)<EOL><DEDENT>

Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : ArrayRDD containing array-like, sparse matrix
            New data to predict.

        Returns
        -------
        labels : ArrayRDD with predictions
            Index of the cluster each sample belongs to.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>if self.init == '<STR_LIT>':<EOL><INDENT>self._mllib_model = MLlibKMeans.train(<EOL>X.unblock(),<EOL>self.n_clusters,<EOL>maxIterations=self.max_iter,<EOL>initializationMode="<STR_LIT>")<EOL>self.cluster_centers_ = self._mllib_model.centers<EOL><DEDENT>else:<EOL><INDENT>models = X.map(lambda X: super(SparkKMeans, self).fit(X))<EOL>models = models.map(lambda model: model.cluster_centers_).collect()<EOL>return super(SparkKMeans, self).fit(np.concatenate(models))<EOL><DEDENT>

Compute k-means clustering.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD containing array-like or sparse matrix
            Train data.

        Returns
        -------
        self


return self.fit(Z).transform(Z)<EOL>

Learn a list of feature name -> indices mappings and transform Z.
        Like fit(Z) followed by transform(Z).

        Parameters
        ----------
        Z : Z : ArrayRDD or DictRDD with column 'X' containing Mapping or
            iterable over Mappings
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        Z : transformed, containing {array, sparse matrix}
            Feature vectors; always 2-d.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._validate_vocabulary()<EOL><DEDENT>self._check_vocabulary()<EOL>analyze = self.build_analyzer()<EOL>mapper = self.broadcast(self._count_vocab, Z.context)<EOL>Z = Z.transform(lambda X: list(map(analyze, X)), column='<STR_LIT:X>').transform(mapper, column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL>return Z<EOL>

Transform documents to document-term matrix.

        Extract token counts out of raw text documents using the vocabulary
        fitted with fit or the one provided to the constructor.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Document-term matrix.


fit_params_steps = dict((step, {})<EOL>for step, _ in self.transformer_list)<EOL>for pname, pval in six.iteritems(fit_params):<EOL><INDENT>step, param = pname.split('<STR_LIT>', <NUM_LIT:1>)<EOL>fit_params_steps[step][param] = pval<EOL><DEDENT>transformers = Parallel(n_jobs=self.n_jobs, backend="<STR_LIT>")(<EOL>delayed(_fit_one_transformer)(trans, Z, **fit_params_steps[name])<EOL>for name, trans in self.transformer_list)<EOL>self._update_transformer_list(transformers)<EOL>return self<EOL>

TODO: rewrite docstring
        Fit all transformers using X.
        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data, used to fit transformers.


def transform(self, fn, dtype=None, *args, **kwargs):

rdd = self._rdd.map(fn)<EOL>if dtype is None:<EOL><INDENT>return self.__class__(rdd, noblock=True, **self.get_params())<EOL><DEDENT>if dtype is np.ndarray:<EOL><INDENT>return ArrayRDD(rdd, bsize=self.bsize, noblock=True)<EOL><DEDENT>elif dtype is sp.spmatrix:<EOL><INDENT>return SparseRDD(rdd, bsize=self.bsize, noblock=True)<EOL><DEDENT>else:<EOL><INDENT>return BlockRDD(rdd, bsize=self.bsize, dtype=dtype, noblock=True)<EOL><DEDENT>

Equivalent to map, compatibility purpose only.
        Column parameter ignored.


return self._rdd.map(len).sum()<EOL>

Returns the number of elements in all blocks.


return self._rdd.flatMap(lambda cols: list(zip(*cols)))<EOL>


return self._rdd.flatMap(lambda x: list(x))<EOL>


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>self._classes_ = np.unique(classes)<EOL>return self._spark_fit(SparkLinearSVC, Z)<EOL>


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return X.map(<EOL>lambda X: super(SparkBaseNB, self).predict_proba(X))<EOL>

Return probability estimates for the RDD containing test vector X.

Parameters
----------
X : RDD containing array-like items, shape = [m_samples, n_features]

Returns
-------
C : RDD with array-like items , shape = [n_samples, n_classes]
    Returns the probability of the samples for each class in
    the models for each RDD block. The columns correspond to the classes
    in sorted order, as they appear in the attribute `classes_`.


<EOL>self._reset()<EOL>X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>def mapper(X):<EOL><INDENT>"""<STR_LIT>"""<EOL>X = check_array(X, ('<STR_LIT>', '<STR_LIT>'), dtype=np.float64)<EOL>if hasattr(X, "<STR_LIT>"):   <EOL><INDENT>mean, var = mean_variance_axis(X, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>mean, var = np.mean(X, axis=<NUM_LIT:0>), np.var(X, axis=<NUM_LIT:0>)<EOL><DEDENT>return X.shape[<NUM_LIT:0>], mean, var<EOL><DEDENT>def reducer(a, b):<EOL><INDENT>"""<STR_LIT>"""<EOL>n_a, mean_a, var_a = a<EOL>n_b, mean_b, var_b = b<EOL>n_ab = n_a + n_b<EOL>mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab<EOL>var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) +((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** <NUM_LIT:2>)<EOL>return (n_ab, mean_ab, var_ab)<EOL><DEDENT>if check_rdd_dtype(X, (sp.spmatrix)):<EOL><INDENT>if self.with_mean:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>self.n_samples_seen_, self.mean_, self.var_ = X.map(mapper).treeReduce(reducer)<EOL>if self.with_std:<EOL><INDENT>self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))<EOL><DEDENT>else:<EOL><INDENT>self.scale_ = None<EOL><DEDENT>return self<EOL>

Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.


def auth_decorator(func):<EOL><INDENT>def inner(self, *args, **kw):<EOL><INDENT>if self.get_authenticated_user(auth_func, realm):<EOL><INDENT>return func(self, *args, **kw)<EOL><DEDENT><DEDENT>return inner<EOL><DEDENT>return auth_decorator<EOL>

Decorator that protect methods with HTTP authentication.


pass<EOL>

Latex document generator depart handler.


now = timezone.now()<EOL>tasks = cls.objects.filter(enabled=True)<EOL>for task in tasks:<EOL><INDENT>if task.next_run == HAS_NOT_RUN:<EOL><INDENT>task.calc_next_run()<EOL><DEDENT>if task.next_run < now:<EOL><INDENT>if (task.start_running < now):<EOL><INDENT>if (task.end_running > now):<EOL><INDENT>task.run_asap()<EOL><DEDENT>else:<EOL><INDENT>task.enabled = False<EOL>task.save()<EOL>Channel(KILL_TASK_CHANNEL).send({'<STR_LIT:id>': task.pk})<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Internal task-runner class method, called by :py:func:`sisy.consumers.run_heartbeat`


task = Task.objects.get(label=label)<EOL>info = json.loads(task._func_info)<EOL>return info<EOL>

Return task info dictionary from task label.  Internal function,
    pretty much only used in migrations since the model methods aren't there.


@classmethod<EOL><INDENT>def run_iterations(cls, the_callable, iterations=<NUM_LIT:1>, label=None, schedule='<STR_LIT>', userdata = None, run_immediately=False, delay_until=None):<DEDENT>

task = task_with_callable(the_callable, label=label, schedule=schedule, userdata=userdata)<EOL>task.iterations = iterations<EOL>if delay_until is not None:<EOL><INDENT>if isinstance(delay_until, datetime):<EOL><INDENT>if delay_until > timezone.now():<EOL><INDENT>task.start_running = delay_until<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>if run_immediately:<EOL><INDENT>task.next_run = timezone.now()<EOL><DEDENT>else:<EOL><INDENT>task.calc_next_run()<EOL><DEDENT>task.save()<EOL>

Class method to run a callable with a specified number of iterations


mems = {x[<NUM_LIT:0>]: x[<NUM_LIT:1>] for x in inspect.getmembers(thing_obj)}<EOL>if member_string in mems:<EOL><INDENT>return mems[member_string]<EOL><DEDENT>

Get a member from an object by (string) name


then = arrow.get(message['<STR_LIT:time>'])<EOL>now = arrow.get()<EOL>if (now - then) > timezone.timedelta(seconds=(TICK_FREQ+<NUM_LIT:1>)):<EOL><INDENT>pass <EOL><DEDENT>else:<EOL><INDENT>Task.run_tasks()<EOL><DEDENT>

Internal ``CLOCK_CHANNEL`` consumer to process task runs


def imap(requests, stream=True, pool=None, size=<NUM_LIT:2>, exception_handler=None):

def send(r):<EOL><INDENT>return r.send(stream=stream)<EOL><DEDENT>pool = pool if pool else Pool(size)<EOL>for request in pool.imap(send, requests):<EOL><INDENT>if request.response is not None:<EOL><INDENT>yield request.response<EOL><DEDENT>elif exception_handler:<EOL><INDENT>exception_handler(request, request.exception)<EOL><DEDENT><DEDENT>if not pool:<EOL><INDENT>pool.close()<EOL><DEDENT>

Concurrently converts a generator object of Requests to
    a generator of Responses.

    :param requests: a generator of Request objects.
    :param stream: If False, the content will not be downloaded immediately.
    :param size: Specifies the number of requests to make at a time. default is 2
    :param exception_handler: Callback function, called when exception occured. Params: Request, Exception


merged_kwargs = {}<EOL>merged_kwargs.update(self.kwargs)<EOL>merged_kwargs.update(kwargs)<EOL>try:<EOL><INDENT>self.response = self.session.request(<EOL>self.method, self.url, **merged_kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.exception = e<EOL>self.traceback = traceback.format_exc()<EOL><DEDENT>return self<EOL>

Prepares request based on parameter passed to constructor and optional
``kwargs```. Then sends request and saves response to :attr:`response`.

:returns: ``Response``


try:<EOL><INDENT>width = int(os.environ['<STR_LIT>'])<EOL><DEDENT>except (KeyError, ValueError):<EOL><INDENT>width = get_terminal_size().columns<EOL><DEDENT>return width or <NUM_LIT><EOL>

Figure out a reasonable default with. Use os.environ['COLUMNS'] if possible,
    and failing that use 80.


saver = tf.train.import_meta_graph('<STR_LIT>'.format(input_checkpoint))<EOL>saver.restore(sess, input_checkpoint)<EOL>return saver<EOL>

Return a TensorFlow saver from a checkpoint containing the metagraph.


def freeze(sess, output_file_path, output_node_names):

with TemporaryDirectory() as temp_dir_name:<EOL><INDENT>checkpoint_path = os.path.join(temp_dir_name, '<STR_LIT>')<EOL>tf.train.Saver().save(sess, checkpoint_path)<EOL>freeze_from_checkpoint(checkpoint_path, output_file_path, output_node_names)<EOL><DEDENT>

Freeze and shrink the graph based on a session and the output node names.


def fit_transform(self, features, targets):

self.fit(features, targets)<EOL>return self.transform(features)<EOL>

Convenience function that fits the provided data then constructs a new feature from the provided features.

        Parameters
        ----------
        features: array-like {n_samples, n_features}
            Feature matrix
        targets: array-like {n_samples}
            List of true target values

        Returns
        ----------
        array-like: {n_samples}
            Constructed features from the provided feature matrix


var1_levels = list(set([variables[<NUM_LIT:0>] for variables in mdr_instance.feature_map]))<EOL>var2_levels = list(set([variables[<NUM_LIT:1>] for variables in mdr_instance.feature_map]))<EOL>max_count = np.array(list(mdr_instance.class_count_matrix.values())).flatten().max()<EOL>"""<STR_LIT>"""<EOL>fig, splots = plt.subplots(ncols=len(var1_levels), nrows=len(var2_levels), sharey=True, sharex=True)<EOL>fig.set_figwidth(<NUM_LIT:6>)<EOL>fig.set_figheight(<NUM_LIT:6>)<EOL>for (var1, var2) in itertools.product(var1_levels, var2_levels):<EOL><INDENT>class_counts = mdr_instance.class_count_matrix[(var1, var2)]<EOL>splot = splots[var2_levels.index(var2)][var1_levels.index(var1)]<EOL>splot.set_yticks([])<EOL>splot.set_xticks([])<EOL>splot.set_ylim(<NUM_LIT:0>, max_count * <NUM_LIT>)<EOL>splot.set_xlim(-<NUM_LIT:0.5>, <NUM_LIT>)<EOL>if var2_levels.index(var2) == <NUM_LIT:0>:<EOL><INDENT>splot.set_title('<STR_LIT>'.format(var1), fontsize=<NUM_LIT:12>)<EOL><DEDENT>if var1_levels.index(var1) == <NUM_LIT:0>:<EOL><INDENT>splot.set_ylabel('<STR_LIT>'.format(var2), fontsize=<NUM_LIT:12>)<EOL><DEDENT>bars = splot.bar(left=range(class_counts.shape[<NUM_LIT:0>]),<EOL>height=class_counts, width=<NUM_LIT:0.5>,<EOL>color='<STR_LIT>', align='<STR_LIT>')<EOL>bgcolor = '<STR_LIT>' if mdr_instance.feature_map[(var1, var2)] == <NUM_LIT:0> else '<STR_LIT>'<EOL>splot.set_axis_bgcolor(bgcolor)<EOL>for index, bar in enumerate(bars):<EOL><INDENT>splot.text(index, class_counts[index] + (max_count * <NUM_LIT:0.1>), class_counts[index], ha='<STR_LIT>')<EOL><DEDENT><DEDENT>fig.tight_layout()<EOL>return fig<EOL>

Visualizes the MDR grid of a given fitted MDR instance. Only works for 2-way MDR models.

    This function is currently incomplete.

    Parameters
    ----------
    mdr_instance: object
        A fitted instance of the MDR type to visualize.

    Returns
    ----------
    fig: matplotlib.figure
        Figure object for the visualized MDR grid.


return scipy.stats.entropy(list(Counter(X).values()), base=base)<EOL>

Calculates the entropy, H(X), in the given base

    Parameters
    ----------
    X: array-like (# samples)
        An array of values for which to compute the entropy
    base: integer (default: 2)
        The base in which to calculate entropy

    Returns
    ----------
    entropy: float
        The entropy calculated according to the equation H(X) = -sum(p_x * log p_x) for all states of X


return MDR().fit_predict(np.column_stack((X, Y)), labels)<EOL>

Fits a MDR model to variables X and Y with the given labels, then returns the resulting predictions

    This is a convenience method that should only be used internally.

    Parameters
    ----------
    X: array-like (# samples)
        An array of values corresponding to one feature in the MDR model
    Y: array-like (# samples)
        An array of values corresponding to one feature in the MDR model
    labels: array-like (# samples)
        The class labels corresponding to features X and Y

    Returns
    ----------
    predictions: array-like (# samples)
        The predictions from the fitted MDR model


return self.ensemble.predict(features)<EOL>

Uses the MDR ensemble to construct a new feature from the provided features

        Parameters
        ----------
        features: array-like {n_samples, n_features}
            Feature matrix to transform

        Returns
        ----------
        array-like: {n_samples}
            Constructed features from the provided feature matrix


def fit_predict(self, features, class_labels):

self.fit(features, class_labels)<EOL>return self.predict(features)<EOL>

Convenience function that fits the provided data then constructs predictions from the provided features.

        Parameters
        ----------
        features: array-like {n_samples, n_features}
            Feature matrix
        class_labels: array-like {n_samples}
            List of true class labels

        Returns
        ----------
        array-like: {n_samples}
            Constructed features from the provided feature matrix


def my_permission_factory(record, *args, **kwargs):

def can(self):<EOL><INDENT>rec = Record.get_record(record.id)<EOL>return rec.get('<STR_LIT>', '<STR_LIT>') == '<STR_LIT>'<EOL><DEDENT>return type('<STR_LIT>', (), {'<STR_LIT>': can})()<EOL>


def create_url_rule(endpoint, route=None, pid_type=None, template=None,<EOL>permission_factory_imp=None, view_imp=None,<EOL>record_class=None, methods=None):

assert route<EOL>assert pid_type<EOL>permission_factory = import_string(permission_factory_imp) ifpermission_factory_imp else None<EOL>view_method = import_string(view_imp) if view_imp else default_view_method<EOL>record_class = import_string(record_class) if record_class else Record<EOL>methods = methods or ['<STR_LIT:GET>']<EOL>view_func = partial(<EOL>record_view,<EOL>resolver=Resolver(pid_type=pid_type, object_type='<STR_LIT>',<EOL>getter=record_class.get_record),<EOL>template=template or '<STR_LIT>',<EOL>permission_factory=permission_factory,<EOL>view_method=view_method)<EOL>view_func.__module__ = record_view.__module__<EOL>view_func.__name__ = record_view.__name__<EOL>return dict(<EOL>endpoint=endpoint,<EOL>rule=route,<EOL>view_func=view_func,<EOL>methods=methods,<EOL>)<EOL>

Create Werkzeug URL rule for a specific endpoint.

    The method takes care of creating a persistent identifier resolver
    for the given persistent identifier type.

    :param endpoint: Name of endpoint.
    :param route: URL route (must include ``<pid_value>`` pattern). Required.
    :param pid_type: Persistent identifier type for endpoint. Required.
    :param template: Template to render.
        (Default: ``invenio_records_ui/detail.html``)
    :param permission_factory_imp: Import path to factory that creates a
        permission object for a given record.
    :param view_imp: Import path to view function. (Default: ``None``)
    :param record_class: Name of the record API class.
    :param methods: Method allowed for the endpoint.
    :returns: A dictionary that can be passed as keywords arguments to
        ``Blueprint.add_url_rule``.


if app:<EOL><INDENT>self.init_app(app)<EOL><DEDENT>

Extension initialization.

        :param app: The Flask application. (Default: ``None``)


self.init_config(app)<EOL>app.extensions['<STR_LIT>'] = _RecordUIState(app)<EOL>

Flask application initialization.

        :param app: The Flask application.


config_option_help = "<STR_LIT>"<EOL>parser = OptionParser()<EOL>parser.add_option("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>",<EOL>type="<STR_LIT:string>", dest="<STR_LIT>", help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT:-c>", "<STR_LIT>", action="<STR_LIT:store>",<EOL>type="<STR_LIT:string>", dest="<STR_LIT>", help=config_option_help)<EOL>parser.add_option("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>",<EOL>type="<STR_LIT:string>", dest="<STR_LIT>", help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>",<EOL>type="<STR_LIT:string>", dest="<STR_LIT>", help="<STR_LIT>")<EOL>parser.add_option("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>",<EOL>type="<STR_LIT:string>", dest="<STR_LIT>", help="<STR_LIT>")<EOL>options, args = parser.parse_args()<EOL>if options.config:<EOL><INDENT>if options.config == "<STR_LIT>":<EOL><INDENT>config_option_list = '<STR_LIT>'<EOL>config_sections = config.sections()<EOL>for section in config_sections:<EOL><INDENT>config_option_list = config_option_list+section+"<STR_LIT:\n>"<EOL>section_items = config.items(section)<EOL>for item in section_items:<EOL><INDENT>config_option_list = config_option_list +"<STR_LIT:U+0020>"+item[<NUM_LIT:0>]+"<STR_LIT:U+0020>"+item[<NUM_LIT:1>]+"<STR_LIT:\n>"<EOL><DEDENT><DEDENT>print(config_option_list)<EOL>quit()<EOL><DEDENT><DEDENT>def add_notes(note_name, existing_tags):<EOL><INDENT>call([editor, environ["<STR_LIT>"] + "<STR_LIT>"+note_name+"<STR_LIT>"])<EOL>definedtags = input("<STR_LIT>").split("<STR_LIT:U+0020>")<EOL>definedtags.append(note_name)<EOL>print(definedtags)<EOL>print(existing_tags)<EOL>definedtags = list(set(definedtags)-set(existing_tags))<EOL>print(definedtags)<EOL>if len(definedtags) > <NUM_LIT:0>:<EOL><INDENT>modify_tags_xml(note_name, definedtags, files,<EOL>rootfiles, tags, roottags, tree, TAGS_XML_DIR)<EOL><DEDENT><DEDENT>def get_tags_from_file(note_name):<EOL><INDENT>fil = get_file_from_files(note_name)<EOL>filetags = fil.iter('<STR_LIT>')<EOL>filetaglist = []<EOL>for tag in filetags:<EOL><INDENT>filetaglist.append(tag.text)<EOL><DEDENT>return filetaglist<EOL><DEDENT>if options.addfile:<EOL><INDENT>existing_tags = []<EOL>if isFile(options.addfile, files):<EOL><INDENT>existing_tags = get_tags_from_file(options.addfile)<EOL>input("<STR_LIT>"+"<STR_LIT:U+0020>".join(existing_tags) +<EOL>"<STR_LIT>")<EOL><DEDENT>add_notes(options.addfile, existing_tags)<EOL>quit()<EOL><DEDENT>if options.editfile:<EOL><INDENT>if isFile(options.editfile, files):<EOL><INDENT>add_notes(note_name, [])<EOL><DEDENT>else:<EOL><INDENT>input(<EOL>"<STR_LIT>")<EOL>add_notes(note_name)<EOL><DEDENT><DEDENT>if options.remove:<EOL><INDENT>pass<EOL><DEDENT>if len(args) != <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>")<EOL>quit()<EOL><DEDENT>_key_File = "<STR_LIT>"<EOL>_key_Results = "<STR_LIT>"<EOL>table = {_key_Results: []}<EOL>for tag in tags:<EOL><INDENT>if tag.attrib["<STR_LIT:value>"] == args[<NUM_LIT:0>]:<EOL><INDENT>fileelements = tag.iter("<STR_LIT:file>")<EOL>for fileelement in fileelements:<EOL><INDENT>f = open(<EOL>environ["<STR_LIT>"] + "<STR_LIT>" + fileelement.text+"<STR_LIT>", "<STR_LIT:r>")<EOL>table[_key_Results].append(<EOL>f.read()+"<STR_LIT>"+fileelement.text+"<STR_LIT>")<EOL>f.close()<EOL><DEDENT><DEDENT><DEDENT>print(tabulate(table, headers=[], tablefmt="<STR_LIT>"))<EOL>


env = args[-<NUM_LIT:1>]<EOL>execve(file, args[:-<NUM_LIT:1>], env)<EOL>

execle(file, *args, env)

    Execute the executable file with argument list args and
    environment env, replacing the current process.


head, tail = path.split(name)<EOL>if not tail:<EOL><INDENT>head, tail = path.split(head)<EOL><DEDENT>if head and tail and not path.exists(head):<EOL><INDENT>try:<EOL><INDENT>makedirs(head, mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if tail == curdir:           <EOL><INDENT>return<EOL><DEDENT><DEDENT>mkdir(name, mode)<EOL>

makedirs(path [, mode=0777])

    Super-mkdir; create a leaf directory and all intermediate ones.
    Works like mkdir, except that any intermediate path segment (not
    just the rightmost) will be created if it does not exist.  This is
    recursive.


rmdir(name)<EOL>head, tail = path.split(name)<EOL>if not tail:<EOL><INDENT>head, tail = path.split(head)<EOL><DEDENT>while head and tail:<EOL><INDENT>try:<EOL><INDENT>rmdir(head)<EOL><DEDENT>except error:<EOL><INDENT>break<EOL><DEDENT>head, tail = path.split(head)<EOL><DEDENT>

removedirs(path)

    Super-rmdir; remove a leaf directory and all empty intermediate
    ones.  Works like rmdir except that, if the leaf directory is
    successfully removed, directories corresponding to rightmost path
    segments will be pruned away until either the whole path is
    consumed or an error occurs.  Errors during this latter phase are
    ignored -- they generally mean that a directory was not empty.


env = args[-<NUM_LIT:1>]<EOL>execvpe(file, args[:-<NUM_LIT:1>], env)<EOL>

execlpe(file, *args, env)

    Execute the executable file (which is searched for along $PATH)
    with argument list args and environment env, replacing the current
    process.


return environ.get(key, default)<EOL>

Get an environment variable, return None if it doesn't exist.
    The optional second argument can specify an alternate default.


_execvpe(file, args, env)<EOL>

execvpe(file, args, env)

    Execute the executable file (which is searched for along $PATH)
    with argument list args and environment env , replacing the
    current process.
    args may be a list or tuple of strings.


return atof(str, int)<EOL>

Converts a string to an integer according to the locale settings.


categories = {}<EOL>def _init_categories(categories=categories):<EOL><INDENT>for k,v in list(globals().items()):<EOL><INDENT>if k[:<NUM_LIT:3>] == '<STR_LIT>':<EOL><INDENT>categories[k] = v<EOL><DEDENT><DEDENT><DEDENT>_init_categories()<EOL>del categories['<STR_LIT>']<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT:->'*<NUM_LIT>)<EOL>lang, enc = getdefaultlocale()<EOL>print('<STR_LIT>', lang or '<STR_LIT>')<EOL>print('<STR_LIT>', enc or '<STR_LIT>')<EOL>print()<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT:->'*<NUM_LIT>)<EOL>for name,category in list(categories.items()):<EOL><INDENT>print(name, '<STR_LIT>')<EOL>lang, enc = getlocale(category)<EOL>print('<STR_LIT>', lang or '<STR_LIT>')<EOL>print('<STR_LIT>', enc or '<STR_LIT>')<EOL>print()<EOL><DEDENT>print()<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT:->'*<NUM_LIT>)<EOL>resetlocale()<EOL>for name,category in list(categories.items()):<EOL><INDENT>print(name, '<STR_LIT>')<EOL>lang, enc = getlocale(category)<EOL>print('<STR_LIT>', lang or '<STR_LIT>')<EOL>print('<STR_LIT>', enc or '<STR_LIT>')<EOL>print()<EOL><DEDENT>try:<EOL><INDENT>setlocale(LC_ALL, "<STR_LIT>")<EOL><DEDENT>except:<EOL><INDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print()<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT:->'*<NUM_LIT>)<EOL>for name,category in list(categories.items()):<EOL><INDENT>print(name, '<STR_LIT>')<EOL>lang, enc = getlocale(category)<EOL>print('<STR_LIT>', lang or '<STR_LIT>')<EOL>print('<STR_LIT>', enc or '<STR_LIT>')<EOL>print()<EOL><DEDENT><DEDENT>


def format_string(f, val, grouping=False):

percents = list(_percent_re.finditer(f))<EOL>new_f = _percent_re.sub('<STR_LIT:%s>', f)<EOL>if isinstance(val, collections.abc.Mapping):<EOL><INDENT>new_val = []<EOL>for perc in percents:<EOL><INDENT>if perc.group()[-<NUM_LIT:1>]=='<STR_LIT:%>':<EOL><INDENT>new_val.append('<STR_LIT:%>')<EOL><DEDENT>else:<EOL><INDENT>new_val.append(format(perc.group(), val, grouping))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if not isinstance(val, tuple):<EOL><INDENT>val = (val,)<EOL><DEDENT>new_val = []<EOL>i = <NUM_LIT:0><EOL>for perc in percents:<EOL><INDENT>if perc.group()[-<NUM_LIT:1>]=='<STR_LIT:%>':<EOL><INDENT>new_val.append('<STR_LIT:%>')<EOL><DEDENT>else:<EOL><INDENT>starcount = perc.group('<STR_LIT>').count('<STR_LIT:*>')<EOL>new_val.append(_format(perc.group(),<EOL>val[i],<EOL>grouping,<EOL>False,<EOL>*val[i+<NUM_LIT:1>:i+<NUM_LIT:1>+starcount]))<EOL>i += (<NUM_LIT:1> + starcount)<EOL><DEDENT><DEDENT><DEDENT>val = tuple(new_val)<EOL>return new_f % val<EOL>

Formats a string in the same way that the % formatting would use,
    but takes the current locale into account.
    Grouping is applied if the third parameter is true.


<EOL>ts = localeconv()['<STR_LIT>']<EOL>if ts:<EOL><INDENT>string = string.replace(ts, '<STR_LIT>')<EOL><DEDENT>dd = localeconv()['<STR_LIT>']<EOL>if dd:<EOL><INDENT>string = string.replace(dd, '<STR_LIT:.>')<EOL><DEDENT>return func(string)<EOL>

Parses a string as a float according to the locale settings.


code = normalize(localename)<EOL>if '<STR_LIT:@>' in code:<EOL><INDENT>code, modifier = code.split('<STR_LIT:@>', <NUM_LIT:1>)<EOL>if modifier == '<STR_LIT>' and '<STR_LIT:.>' not in code:<EOL><INDENT>return code, '<STR_LIT>'<EOL><DEDENT><DEDENT>if '<STR_LIT:.>' in code:<EOL><INDENT>return tuple(code.split('<STR_LIT:.>')[:<NUM_LIT:2>])<EOL><DEDENT>elif code == '<STR_LIT:C>':<EOL><INDENT>return None, None<EOL><DEDENT>raise ValueError('<STR_LIT>' % localename)<EOL>

Parses the locale code for localename and returns the
        result as tuple (language code, encoding).

        The localename is normalized and passed through the locale
        alias engine. A ValueError is raised in case the locale name
        cannot be parsed.

        The language code corresponds to RFC 1766.  code and encoding
        can be None in case the values cannot be determined or are
        unknown to this implementation.


if locale and not isinstance(locale, (_str, _unicode)):<EOL><INDENT>locale = normalize(_build_localename(locale))<EOL><DEDENT>return _setlocale(category, locale)<EOL>

Set the locale for the given category.  The locale can be
        a string, an iterable of two strings (language code and encoding),
        or None.

        Iterables are converted to strings using the locale aliasing
        engine.  Locale strings are passed directly to the C lib.

        category may be given as one of the LC_* values.


try:<EOL><INDENT>import usercustomize<EOL><DEDENT>except ImportError:<EOL><INDENT>pass<EOL><DEDENT>

Run custom user specific code, if available.


if hasattr(sys, '<STR_LIT>') and getattr(sys.flags, '<STR_LIT>', False):<EOL><INDENT>return False<EOL><DEDENT>if hasattr(os, "<STR_LIT>") and hasattr(os, "<STR_LIT>"):<EOL><INDENT>if os.geteuid() != os.getuid():<EOL><INDENT>return None<EOL><DEDENT><DEDENT>if hasattr(os, "<STR_LIT>") and hasattr(os, "<STR_LIT>"):<EOL><INDENT>if os.getegid() != os.getgid():<EOL><INDENT>return None<EOL><DEDENT><DEDENT>return True<EOL>

Check if user site directory is safe for inclusion

    The function tests for the command line flag (including environment var),
    process uid/gid equal to effective uid/gid.

    None: Disabled for security reasons
    False: Disabled by user (command line option)
    True: Safe and enabled


for m in sys.modules.values():<EOL><INDENT>if ((_is_jython and not isinstance(m, ModuleType)) or<EOL>hasattr(m, '<STR_LIT>')):<EOL><INDENT>continue<EOL><DEDENT>f = getattr(m, '<STR_LIT>', None)<EOL>if f is None:<EOL><INDENT>continue<EOL><DEDENT>m.__file__ = os.path.abspath(f)<EOL><DEDENT>

Set all module' __file__ attribute to an absolute path


if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>import locale, codecs<EOL>enc = locale.getdefaultlocale()[<NUM_LIT:1>]<EOL>if enc.startswith('<STR_LIT>'):            <EOL><INDENT>try:<EOL><INDENT>codecs.lookup(enc)<EOL><DEDENT>except LookupError:<EOL><INDENT>import encodings<EOL>encodings._cache[enc] = encodings._unknown<EOL>encodings.aliases.aliases[enc] = '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT>

On Windows, some default encodings are not provided by Python,
    while they are always available as "mbcs" in each locale. Make
    them usable by aliasing to "mbcs" in such a case.


<EOL>names = cls.__dict__.get("<STR_LIT>")<EOL>if names is not None:<EOL><INDENT>return names<EOL><DEDENT>names = []<EOL>if not hasattr(cls, "<STR_LIT>"):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>for c in cls.__mro__:<EOL><INDENT>if "<STR_LIT>" in c.__dict__:<EOL><INDENT>slots = c.__dict__['<STR_LIT>']<EOL>if isinstance(slots, str):<EOL><INDENT>slots = (slots,)<EOL><DEDENT>for name in slots:<EOL><INDENT>if name in ("<STR_LIT>", "<STR_LIT>"):<EOL><INDENT>continue<EOL><DEDENT>elif name.startswith('<STR_LIT>') and not name.endswith('<STR_LIT>'):<EOL><INDENT>stripped = c.__name__.lstrip('<STR_LIT:_>')<EOL>if stripped:<EOL><INDENT>names.append('<STR_LIT>' % (stripped, name))<EOL><DEDENT>else:<EOL><INDENT>names.append(name)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>names.append(name)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>try:<EOL><INDENT>cls.__slotnames__ = names<EOL><DEDENT>except:<EOL><INDENT>pass <EOL><DEDENT>return names<EOL>

Return a list of slot names for a given class.

    This needs to find slots defined by the class and its bases, so we
    can't simply return the __slots__ attribute.  We must walk down
    the Method Resolution Order and concatenate the __slots__ of each
    class found there.  (This assumes classes don't modify their
    __slots__ attribute to misrepresent their slots after the class is
    defined.)


warnings.warnpy3k("<STR_LIT>",<EOL>stacklevel=<NUM_LIT:2>)<EOL>try:<EOL><INDENT>names = os.listdir(top)<EOL><DEDENT>except os.error:<EOL><INDENT>return<EOL><DEDENT>func(arg, top, names)<EOL>for name in names:<EOL><INDENT>name = join(top, name)<EOL>try:<EOL><INDENT>st = os.lstat(name)<EOL><DEDENT>except os.error:<EOL><INDENT>continue<EOL><DEDENT>if stat.S_ISDIR(st.st_mode):<EOL><INDENT>walk(name, func, arg)<EOL><DEDENT><DEDENT>

Directory tree walk with callback function.

    For each directory in the directory tree rooted at top (including top
    itself, but excluding '.' and '..'), call func(arg, dirname, fnames).
    dirname is the name of the directory, and fnames a list of the names of
    the files and subdirectories in dirname (excluding '.' and '..').  func
    may modify the fnames list in-place (e.g. via del or slice assignment),
    and walk will only recurse into the subdirectories whose names remain in
    fnames; this can be used to implement a filter, or to impose a specific
    order of visiting.  No semantics are defined for, or required of, arg,
    beyond that arg is always passed to func.  It can be used, e.g., to pass
    a filename pattern, or a mutable object designed to accumulate
    statistics.  Passing None for arg is common.


if not isabs(path):<EOL><INDENT>if isinstance(path, _unicode):<EOL><INDENT>cwd = os.getcwd()<EOL><DEDENT>else:<EOL><INDENT>cwd = os.getcwd()<EOL><DEDENT>path = join(cwd, path)<EOL><DEDENT>return normpath(path)<EOL>


return '<STR_LIT>', p<EOL>

Split a pathname into drive and path. On Posix, drive is always
    empty.


if not path:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>start_list = [x for x in abspath(start).split(sep) if x]<EOL>path_list = [x for x in abspath(path).split(sep) if x]<EOL>i = len(commonprefix([start_list, path_list]))<EOL>rel_list = [pardir] * (len(start_list)-i) + path_list[i:]<EOL>if not rel_list:<EOL><INDENT>return curdir<EOL><DEDENT>return join(*rel_list)<EOL>


try:<EOL><INDENT>st = os.lstat(path)<EOL><DEDENT>except (os.error, AttributeError):<EOL><INDENT>return False<EOL><DEDENT>return stat.S_ISLNK(st.st_mode)<EOL>


i = p.rfind('<STR_LIT:/>') + <NUM_LIT:1><EOL>head, tail = p[:i], p[i:]<EOL>if head and head != '<STR_LIT:/>'*len(head):<EOL><INDENT>head = head.rstrip('<STR_LIT:/>')<EOL><DEDENT>return head, tail<EOL>

Split a pathname.  Returns tuple "(head, tail)" where "tail" is
    everything after the final slash.  Either part may be empty.


if filename in cache:<EOL><INDENT>del cache[filename]<EOL><DEDENT>if not filename or (filename.startswith('<STR_LIT:<>') and filename.endswith('<STR_LIT:>>')):<EOL><INDENT>return []<EOL><DEDENT>fullname = filename<EOL>try:<EOL><INDENT>stat = os.stat(fullname)<EOL><DEDENT>except OSError:<EOL><INDENT>basename = filename<EOL>if module_globals and '<STR_LIT>' in module_globals:<EOL><INDENT>name = module_globals.get('<STR_LIT>')<EOL>loader = module_globals['<STR_LIT>']<EOL>get_source = getattr(loader, '<STR_LIT>', None)<EOL>if name and get_source:<EOL><INDENT>try:<EOL><INDENT>data = get_source(name)<EOL><DEDENT>except (ImportError, IOError):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>if data is None:<EOL><INDENT>return []<EOL><DEDENT>cache[filename] = (<EOL>len(data), None,<EOL>[line+'<STR_LIT:\n>' for line in data.splitlines()], fullname<EOL>)<EOL>return cache[filename][<NUM_LIT:2>]<EOL><DEDENT><DEDENT><DEDENT>if os.path.isabs(filename):<EOL><INDENT>return []<EOL><DEDENT>for dirname in sys.path:<EOL><INDENT>try:<EOL><INDENT>fullname = os.path.join(dirname, basename)<EOL><DEDENT>except (TypeError, AttributeError):<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>stat = os.stat(fullname)<EOL>break<EOL><DEDENT>except os.error:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return []<EOL><DEDENT><DEDENT>try:<EOL><INDENT>with open(fullname, '<STR_LIT>') as fp:<EOL><INDENT>lines = fp.readlines()<EOL><DEDENT><DEDENT>except IOError:<EOL><INDENT>return []<EOL><DEDENT>if lines and not lines[-<NUM_LIT:1>].endswith('<STR_LIT:\n>'):<EOL><INDENT>lines[-<NUM_LIT:1>] += '<STR_LIT:\n>'<EOL><DEDENT>size, mtime = stat.st_size, stat.st_mtime<EOL>cache[filename] = size, mtime, lines, fullname<EOL>return lines<EOL>

Update a cache entry and return its list of lines.
    If something's wrong, print a message, discard the cache entry,
    and return an empty list.


if filename is None:<EOL><INDENT>filenames = cache.keys()<EOL><DEDENT>else:<EOL><INDENT>if filename in cache:<EOL><INDENT>filenames = [filename]<EOL><DEDENT>else:<EOL><INDENT>return<EOL><DEDENT><DEDENT>for filename in filenames:<EOL><INDENT>size, mtime, lines, fullname = cache[filename]<EOL>if mtime is None:<EOL><INDENT>continue   <EOL><DEDENT>try:<EOL><INDENT>stat = os.stat(fullname)<EOL><DEDENT>except os.error:<EOL><INDENT>del cache[filename]<EOL>continue<EOL><DEDENT>if size != stat.st_size or mtime != stat.st_mtime:<EOL><INDENT>del cache[filename]<EOL><DEDENT><DEDENT>

Discard cache entries that are out of date.
    (This is not checked upon each call!)


global cache<EOL>cache = {}<EOL>


res = {}<EOL>for i in rng:<EOL><INDENT>res[i]=i<EOL><DEDENT>return res<EOL>

make_identity_dict(rng) -> dict

        Return a dictionary where elements of the rng sequence are
        mapped to themselves.


return next(self.reader)<EOL>

Return the next decoded line from the input stream.


m = {}<EOL>for k,v in list(decoding_map.items()):<EOL><INDENT>if not v in m:<EOL><INDENT>m[v] = k<EOL><DEDENT>else:<EOL><INDENT>m[v] = None<EOL><DEDENT><DEDENT>return m<EOL>

Creates an encoding map from a decoding map.

        If a target mapping in the decoding map occurs multiple
        times, then that target is mapped to None (undefined mapping),
        causing an exception when encountered by the charmap codec
        during translation.

        One example where this happens is cp875.py which decodes
        multiple character to \\u001a.


Writes the concatenated list of strings to the stream
            using .write().


return lookup(encoding).encode<EOL>

Lookup up the codec for the given encoding and return
        its encoder function.

        Raises a LookupError in case the encoding cannot be found.


def readlines(self, sizehint=None, keepends=True):

data = self.read()<EOL>return data.splitlines(keepends)<EOL>

Read all lines available on the input stream
            and return them as list of lines.

            Line breaks are implemented using the codec's decoder
            method and are included in the list entries.

            sizehint, if given, is ignored since there is no efficient
            way to finding the true end-of-line.


def decode(self, input, errors='<STR_LIT:strict>'):

raise NotImplementedError<EOL>

Decodes the object input and returns a tuple (output
            object, length consumed).

            input must be an object which provides the bf_getreadbuf
            buffer slot. Python strings, buffer objects and memory
            mapped files are examples of objects providing this slot.

            errors defines the error handling to apply. It defaults to
            'strict' handling.

            The method may not store state in the Codec instance. Use
            StreamReader for codecs which have to keep state in order to
            make decoding efficient.

            The decoder must be able to handle zero length input and
            return an empty object of the output object type in this
            situation.


def __init__(self, stream, errors='<STR_LIT:strict>'):

self.stream = stream<EOL>self.errors = errors<EOL>self.bytebuffer = "<STR_LIT>"<EOL>self.charbuffer = "<STR_LIT>"<EOL>self.linebuffer = None<EOL>

Creates a StreamReader instance.

            stream must be a file-like object open for reading
            (binary) data.

            The StreamReader may use different error handling
            schemes by providing the errors keyword argument. These
            parameters are predefined:

             'strict' - raise a ValueError (or a subclass)
             'ignore' - ignore the character and continue with the next
             'replace'- replace with a suitable replacement character;

            The set of allowed parameter values can be extended via
            register_error.


self.stream = stream<EOL>self.errors = errors<EOL>

Creates a StreamWriter instance.

            stream must be a file-like object open for writing
            (binary) data.

            The StreamWriter may use different error handling
            schemes by providing the errors keyword argument. These
            parameters are predefined:

             'strict' - raise a ValueError (or a subclass)
             'ignore' - ignore the character and continue with the next
             'replace'- replace with a suitable replacement character
             'xmlcharrefreplace' - Replace with the appropriate XML
                                   character reference.
             'backslashreplace'  - Replace with backslashed escape
                                   sequences (only for encoding).

            The set of allowed parameter values can be extended via
            register_error.


def __init__(self, stream, Reader, Writer, errors='<STR_LIT:strict>'):

self.stream = stream<EOL>self.reader = Reader(stream, errors)<EOL>self.writer = Writer(stream, errors)<EOL>self.errors = errors<EOL>

Creates a StreamReaderWriter instance.

            stream must be a Stream-like object.

            Reader, Writer must be factory functions or classes
            providing the StreamReader, StreamWriter interface resp.

            Error handling is done in the same way as defined for the
            StreamWriter/Readers.


def iterdecode(iterator, encoding, errors='<STR_LIT:strict>', **kwargs):

decoder = getincrementaldecoder(encoding)(errors, **kwargs)<EOL>for input in iterator:<EOL><INDENT>output = decoder.decode(input)<EOL>if output:<EOL><INDENT>yield output<EOL><DEDENT><DEDENT>output = decoder.decode("<STR_LIT>", True)<EOL>if output:<EOL><INDENT>yield output<EOL><DEDENT>

Decoding iterator.

Decodes the input strings from the iterator using an IncrementalDecoder.

errors and kwargs are passed through to the IncrementalDecoder
constructor.


Set the current state of the encoder. state must have been
returned by getstate().


data, consumed = self.encode(object, self.errors)<EOL>self.stream.write(data)<EOL>

Writes the object's contents encoded to self.stream.


def open(filename, mode='<STR_LIT:rb>', encoding=None, errors='<STR_LIT:strict>', buffering=<NUM_LIT:1>):

if encoding is not None:<EOL><INDENT>if '<STR_LIT>' in mode:<EOL><INDENT>mode = mode.strip().replace('<STR_LIT>', '<STR_LIT>')<EOL>if mode[:<NUM_LIT:1>] not in set('<STR_LIT>'):<EOL><INDENT>mode = '<STR_LIT:r>' + mode<EOL><DEDENT><DEDENT>if '<STR_LIT:b>' not in mode:<EOL><INDENT>mode = mode + '<STR_LIT:b>'<EOL><DEDENT><DEDENT>file = builtins.open(filename, mode, buffering)<EOL>if encoding is None:<EOL><INDENT>return file<EOL><DEDENT>info = lookup(encoding)<EOL>srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)<EOL>srw.encoding = encoding<EOL>return srw<EOL>

Open an encoded file using the given mode and return
        a wrapped version providing transparent encoding/decoding.

        Note: The wrapped version will only accept the object format
        defined by the codecs, i.e. Unicode objects for most builtin
        codecs. Output is also codec dependent and will usually be
        Unicode as well.

        Files are always opened in binary mode, even if no binary mode
        was specified. This is done to avoid data loss due to encodings
        using 8-bit values. The default file mode is 'rb' meaning to
        open the file in binary read mode.

        encoding specifies the encoding which is to be used for the
        file.

        errors may be given to define the error handling. It defaults
        to 'strict' which causes ValueErrors to be raised in case an
        encoding error occurs.

        buffering has the same meaning as for the builtin open() API.
        It defaults to line buffered.

        The returned wrapped file object provides an extra attribute
        .encoding which allows querying the used encoding. This
        attribute is only available if an encoding was specified as
        parameter.


return (b"<STR_LIT>", <NUM_LIT:0>)<EOL>

Return the current state of the decoder.

This must be a (buffered_input, additional_state_info) tuple.
buffered_input must be a bytes object containing bytes that
were passed to decode() that have not yet been converted.
additional_state_info must be a non-negative integer
representing the state of the decoder WITHOUT yet having
processed the contents of buffered_input.  In the initial state
and after reset(), getstate() must return (b"", 0).


self.errors = errors<EOL>

Creates an IncrementalDecoder instance.

The IncrementalDecoder may use different error handling schemes by
providing the errors keyword argument. See the module docstring
for a list of possible values.


def seek(self, offset, whence=<NUM_LIT:0>):

self.stream.seek(offset, whence)<EOL>self.reset()<EOL>

Set the input stream's current position.

            Resets the codec buffers used for keeping state.


def __init__(self, stream, encode, decode, Reader, Writer,<EOL>errors='<STR_LIT:strict>'):

self.stream = stream<EOL>self.encode = encode<EOL>self.decode = decode<EOL>self.reader = Reader(stream, errors)<EOL>self.writer = Writer(stream, errors)<EOL>self.errors = errors<EOL>

Creates a StreamRecoder instance which implements a two-way
            conversion: encode and decode work on the frontend (the
            input to .read() and output of .write()) while
            Reader and Writer work on the backend (reading and
            writing to the stream).

            You can use these objects to do transparent direct
            recodings from e.g. latin-1 to utf-8 and back.

            stream must be a file-like object.

            encode, decode must adhere to the Codec interface, Reader,
            Writer must be factory functions or classes providing the
            StreamReader, StreamWriter interface resp.

            encode and decode are needed for the frontend translation,
            Reader and Writer for the backend translation. Unicode is
            used as intermediate encoding.

            Error handling is done in the same way as defined for the
            StreamWriter/Readers.


try:<EOL><INDENT>result = repr(obj)<EOL><DEDENT>except Exception:<EOL><INDENT>result = object.__repr__(obj)<EOL><DEDENT>if not short or len(result) < pkg_resources._MAX_LENGTH:<EOL><INDENT>return result<EOL><DEDENT>return result[:pkg_resources._MAX_LENGTH] + '<STR_LIT>'<EOL>


egg = tempfile.NamedTemporaryFile(suffix='<STR_LIT>', delete=False)<EOL>zip_egg = zipfile.ZipFile(egg, '<STR_LIT:w>')<EOL>zip_info = zipfile.ZipInfo()<EOL>zip_info.filename = '<STR_LIT>'<EOL>zip_info.date_time = cls.ref_time.timetuple()<EOL>zip_egg.writestr(zip_info, '<STR_LIT>')<EOL>zip_info = zipfile.ZipInfo()<EOL>zip_info.filename = '<STR_LIT>'<EOL>zip_info.date_time = cls.ref_time.timetuple()<EOL>zip_egg.writestr(zip_info, '<STR_LIT>')<EOL>zip_egg.close()<EOL>egg.close()<EOL>sys.path.append(egg.name)<EOL>cls.finalizers.append(EggRemover(egg.name))<EOL>


Returns whether or not pre-releases as a whole are allowed by this
specifier.


if hasattr(zipfile.ZipFile, '<STR_LIT>'):<EOL><INDENT>return zipfile.ZipFile(*args, **kwargs)<EOL><DEDENT>return super(ContextualZipFile, cls).__new__(cls)<EOL>

Construct a ZipFile or ContextualZipFile as appropriate


def find_eggs_in_zip(importer, path_item, only=False):

if importer.archive.endswith('<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>metadata = EggMetadata(importer)<EOL>if metadata.has_metadata('<STR_LIT>'):<EOL><INDENT>yield Distribution.from_filename(path_item, metadata=metadata)<EOL><DEDENT>if only:<EOL><INDENT>return<EOL><DEDENT>for subitem in metadata.resource_listdir('<STR_LIT:/>'):<EOL><INDENT>if subitem.endswith('<STR_LIT>'):<EOL><INDENT>subpath = os.path.join(path_item, subitem)<EOL>for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath):<EOL><INDENT>yield dist<EOL><DEDENT><DEDENT><DEDENT>

Find eggs in zip files; possibly multiple nested eggs.


return get_distribution(dist).get_entry_info(group, name)<EOL>

Return the EntryPoint object for `group`+`name`, or ``None``


return re.sub('<STR_LIT>', '<STR_LIT:_>', extra).lower()<EOL>

Convert an arbitrary string to a standard 'extra' name

    Any runs of non-alphanumeric characters are replaced with a single '_',
    and the result is always lowercased.


_distribution_finders[importer_type] = distribution_finder<EOL>

Register `distribution_finder` to find distributions in sys.path items

    `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
    handler), and `distribution_finder` is a callable that, passed a path
    item and the importer instance, yields ``Distribution`` instances found on
    that path item.  See ``pkg_resources.find_on_path`` for an example.


if isinstance(moduleOrReq, Requirement):<EOL><INDENT>return working_set.find(moduleOrReq) or require(str(moduleOrReq))[<NUM_LIT:0>]<EOL><DEDENT>try:<EOL><INDENT>module = sys.modules[moduleOrReq]<EOL><DEDENT>except KeyError:<EOL><INDENT>__import__(moduleOrReq)<EOL>module = sys.modules[moduleOrReq]<EOL><DEDENT>loader = getattr(module, '<STR_LIT>', None)<EOL>return _find_adapter(_provider_factories, loader)(module)<EOL>

Return an IResourceProvider for the named module or requirement


@classmethod<EOL><INDENT>def parse_map(cls, data, dist=None):<DEDENT>

if isinstance(data, dict):<EOL><INDENT>data = data.items()<EOL><DEDENT>else:<EOL><INDENT>data = split_sections(data)<EOL><DEDENT>maps = {}<EOL>for group, lines in data:<EOL><INDENT>if group is None:<EOL><INDENT>if not lines:<EOL><INDENT>continue<EOL><DEDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>group = group.strip()<EOL>if group in maps:<EOL><INDENT>raise ValueError("<STR_LIT>", group)<EOL><DEDENT>maps[group] = cls.parse_group(group, lines, dist)<EOL><DEDENT>return maps<EOL>


importer = get_importer(path_item)<EOL>if importer is None:<EOL><INDENT>return None<EOL><DEDENT>loader = importer.find_module(packageName)<EOL>if loader is None:<EOL><INDENT>return None<EOL><DEDENT>module = sys.modules.get(packageName)<EOL>if module is None:<EOL><INDENT>module = sys.modules[packageName] = imp.new_module(packageName)<EOL>module.__path__ = []<EOL>_set_parent_ns(packageName)<EOL><DEDENT>elif not hasattr(module,'<STR_LIT>'):<EOL><INDENT>raise TypeError("<STR_LIT>", packageName)<EOL><DEDENT>handler = _find_adapter(_namespace_handlers, importer)<EOL>subpath = handler(importer, path_item, packageName, module)<EOL>if subpath is not None:<EOL><INDENT>path = module.__path__<EOL>path.append(subpath)<EOL>loader.load_module(packageName)<EOL>for path_item in path:<EOL><INDENT>if path_item not in module.__path__:<EOL><INDENT>module.__path__.append(path_item)<EOL><DEDENT><DEDENT><DEDENT>return subpath<EOL>

Ensure that named package includes a subpath of path_item (if needed)


self.zip_pre = importer.archive+os.sep<EOL>self.loader = importer<EOL>if importer.prefix:<EOL><INDENT>self.module_path = os.path.join(importer.archive, importer.prefix)<EOL><DEDENT>else:<EOL><INDENT>self.module_path = importer.archive<EOL><DEDENT>self._setup_prefix()<EOL>

Create a metadata provider from a zipimporter


def find_plugins(self, plugin_env, full_env=None, installer=None,<EOL>fallback=True):

plugin_projects = list(plugin_env)<EOL>plugin_projects.sort()<EOL>error_info = {}<EOL>distributions = {}<EOL>if full_env is None:<EOL><INDENT>env = Environment(self.entries)<EOL>env += plugin_env<EOL><DEDENT>else:<EOL><INDENT>env = full_env + plugin_env<EOL><DEDENT>shadow_set = self.__class__([])<EOL>list(map(shadow_set.add, self))<EOL>for project_name in plugin_projects:<EOL><INDENT>for dist in plugin_env[project_name]:<EOL><INDENT>req = [dist.as_requirement()]<EOL>try:<EOL><INDENT>resolvees = shadow_set.resolve(req, env, installer)<EOL><DEDENT>except ResolutionError as v:<EOL><INDENT>error_info[dist] = v<EOL>if fallback:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>list(map(shadow_set.add, resolvees))<EOL>distributions.update(dict.fromkeys(resolvees))<EOL>break<EOL><DEDENT><DEDENT><DEDENT>distributions = list(distributions)<EOL>distributions.sort()<EOL>return distributions, error_info<EOL>

Find all activatable distributions in `plugin_env`

        Example usage::

            distributions, errors = working_set.find_plugins(
                Environment(plugin_dirlist)
            )
            # add plugins+libs to sys.path
            map(working_set.add, distributions)
            # display errors
            print('Could not load', errors)

        The `plugin_env` should be an ``Environment`` instance that contains
        only distributions that are in the project's "plugin directory" or
        directories. The `full_env`, if supplied, should be an ``Environment``
        contains all currently-available distributions.  If `full_env` is not
        supplied, one is created automatically from the ``WorkingSet`` this
        method is called on, which will typically mean that every directory on
        ``sys.path`` will be scanned for distributions.

        `installer` is a standard installer callback as used by the
        ``resolve()`` method. The `fallback` flag indicates whether we should
        attempt to resolve older versions of a plugin if the newest version
        cannot be resolved.

        This method returns a 2-tuple: (`distributions`, `error_info`), where
        `distributions` is a list of the distributions found in `plugin_env`
        that were loadable, along with any other distributions that are needed
        to resolve their dependencies.  `error_info` is a dictionary mapping
        unloadable plugin distributions to an exception instance describing the
        error that occurred. Usually this will be a ``DistributionNotFound`` or
        ``VersionConflict`` instance.


if isinstance(strs, string_types):<EOL><INDENT>for s in strs.splitlines():<EOL><INDENT>s = s.strip()<EOL>if s and not s.startswith('<STR_LIT:#>'):<EOL><INDENT>yield s<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for ss in strs:<EOL><INDENT>for s in yield_lines(ss):<EOL><INDENT>yield s<EOL><DEDENT><DEDENT><DEDENT>

Yield non-empty/non-comment lines of a string or sequence


distribution_key = project_name.lower()<EOL>return self._distmap.get(distribution_key, [])<EOL>

Return a newest-to-oldest list of distributions for `project_name`

        Uses case-insensitive `project_name` comparison, assuming all the
        project's distributions use their project's name converted to all
        lowercase as their key.


def resource_isdir(self, package_or_requirement, resource_name):

return get_provider(package_or_requirement).resource_isdir(<EOL>resource_name<EOL>)<EOL>

Is the named resource an existing directory?


_namespace_handlers[importer_type] = namespace_handler<EOL>

Register `namespace_handler` to declare namespace packages

    `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
    handler), and `namespace_handler` is a callable like this::

        def namespace_handler(importer, path_entry, moduleName, module):
            # return a path_entry to use for child packages

    Namespace handlers are only called if the importer object has already
    agreed that it can handle the relevant path item, and they should only
    return a subpath if the module __path__ does not already contain an
    equivalent subpath.  For an example namespace handler, see
    ``pkg_resources.file_ns_handler``.


ns = sys._getframe(<NUM_LIT:1>).f_globals<EOL>name = ns['<STR_LIT>']<EOL>ns.clear()<EOL>ns['<STR_LIT>'] = name<EOL>require(dist_spec)[<NUM_LIT:0>].run_script(script_name, ns)<EOL>

Locate distribution `dist_spec` and run its `script_name` script


def add(self, dist, entry=None, insert=True, replace=False):

if insert:<EOL><INDENT>dist.insert_on(self.entries, entry)<EOL><DEDENT>if entry is None:<EOL><INDENT>entry = dist.location<EOL><DEDENT>keys = self.entry_keys.setdefault(entry,[])<EOL>keys2 = self.entry_keys.setdefault(dist.location,[])<EOL>if not replace and dist.key in self.by_key:<EOL><INDENT>return<EOL><DEDENT>self.by_key[dist.key] = dist<EOL>if dist.key not in keys:<EOL><INDENT>keys.append(dist.key)<EOL><DEDENT>if dist.key not in keys2:<EOL><INDENT>keys2.append(dist.key)<EOL><DEDENT>self._added_new(dist)<EOL>

Add `dist` to working set, associated with `entry`

        If `entry` is unspecified, it defaults to the ``.location`` of `dist`.
        On exit from this routine, `entry` is added to the end of the working
        set's ``.entries`` (if it wasn't already present).

        `dist` is only added to the working set if it's for a project that
        doesn't already have a distribution in the set, unless `replace=True`.
        If it's added, any callbacks registered with the ``subscribe()`` method
        will be called.


ws = cls()<EOL>try:<EOL><INDENT>from __main__ import __requires__<EOL><DEDENT>except ImportError:<EOL><INDENT>return ws<EOL><DEDENT>try:<EOL><INDENT>ws.require(__requires__)<EOL><DEDENT>except VersionConflict:<EOL><INDENT>return cls._build_from_requirements(__requires__)<EOL><DEDENT>return ws<EOL>


try:<EOL><INDENT>return os.environ['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>if os.name!='<STR_LIT>':<EOL><INDENT>return os.path.expanduser('<STR_LIT>')<EOL><DEDENT>app_data = '<STR_LIT>'<EOL>app_homes = [<EOL>(('<STR_LIT>',), None),<EOL>(('<STR_LIT>',), app_data),<EOL>(('<STR_LIT>','<STR_LIT>'), app_data),<EOL>(('<STR_LIT>',), app_data),<EOL>(('<STR_LIT>',), None),<EOL>(('<STR_LIT>',), app_data),<EOL>]<EOL>for keys, subdir in app_homes:<EOL><INDENT>dirname = '<STR_LIT>'<EOL>for key in keys:<EOL><INDENT>if key in os.environ:<EOL><INDENT>dirname = os.path.join(dirname, os.environ[key])<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if subdir:<EOL><INDENT>dirname = os.path.join(dirname, subdir)<EOL><DEDENT>return os.path.join(dirname, '<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>

Determine the default cache location

    This returns the ``PYTHON_EGG_CACHE`` environment variable, if set.
    Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the
    "Application Data" directory.  On all other systems, it's "~/.python-eggs".


List of resource names in the directory (like ``os.listdir()``)


Return a readable file-like object for `resource_name`

        `manager` must be an ``IResourceManager``


if not WRITE_SUPPORT:<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT>dirname, filename = split(path)<EOL>if dirname and filename and not isdir(dirname):<EOL><INDENT>_bypass_ensure_directory(dirname)<EOL>mkdir(dirname, mode)<EOL><DEDENT>

Sandbox-bypassing version of ensure_directory()


try:<EOL><INDENT>from sysconfig import get_platform<EOL><DEDENT>except ImportError:<EOL><INDENT>from distutils.util import get_platform<EOL><DEDENT>plat = get_platform()<EOL>if sys.platform == "<STR_LIT>" and not plat.startswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>version = _macosx_vers()<EOL>machine = os.uname()[<NUM_LIT:4>].replace("<STR_LIT:U+0020>", "<STR_LIT:_>")<EOL>return "<STR_LIT>" % (int(version[<NUM_LIT:0>]), int(version[<NUM_LIT:1>]),<EOL>_macosx_arch(machine))<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return plat<EOL>

Return this platform's string for platform-specific distributions

    XXX Currently this is the same as ``distutils.util.get_platform()``, but it
    needs some hacks for Linux and Mac OS X.


if isinstance(self.parsed_version, packaging.version.Version):<EOL><INDENT>spec = "<STR_LIT>" % (self.project_name, self.parsed_version)<EOL><DEDENT>else:<EOL><INDENT>spec = "<STR_LIT>" % (self.project_name, self.parsed_version)<EOL><DEDENT>return Requirement.parse(spec)<EOL>

Return a ``Requirement`` that matches this distribution exactly


for key in self._distmap.keys():<EOL><INDENT>if self[key]:<EOL><INDENT>yield key<EOL><DEDENT><DEDENT>

Yield the unique project names of the available distributions


def file_ns_handler(importer, path_item, packageName, module):

subpath = os.path.join(path_item, packageName.split('<STR_LIT:.>')[-<NUM_LIT:1>])<EOL>normalized = _normalize_cached(subpath)<EOL>for item in module.__path__:<EOL><INDENT>if _normalize_cached(item)==normalized:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return subpath<EOL><DEDENT>

Compute an ns-package subpath for a filesystem or zipfile importer


needed = self.resolve(parse_requirements(requirements))<EOL>for dist in needed:<EOL><INDENT>self.add(dist)<EOL><DEDENT>return needed<EOL>

Ensure that distributions matching `requirements` are activated

        `requirements` must be a string or a (possibly-nested) sequence
        thereof, specifying the distributions and versions required.  The
        return value is a sequence of the distributions that needed to be
        activated to fulfill the requirements; all relevant distributions are
        included, even if they were already activated in this working set.


def load(self, require=True, *args, **kwargs):

if not require or args or kwargs:<EOL><INDENT>warnings.warn(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>DeprecationWarning,<EOL>stacklevel=<NUM_LIT:2>,<EOL>)<EOL><DEDENT>if require:<EOL><INDENT>self.require(*args, **kwargs)<EOL><DEDENT>return self.resolve()<EOL>

Require packages for this EntryPoint, then resolve it.


def resource_listdir(self, package_or_requirement, resource_name):

return get_provider(package_or_requirement).resource_listdir(<EOL>resource_name<EOL>)<EOL>

List the contents of the named resource directory


return (self.python is None or dist.py_version is None<EOL>or dist.py_version==self.python)and compatible_platforms(dist.platform, self.platform)<EOL>

Is distribution `dist` acceptable for this environment?

        The distribution must match the platform and python version
        requirements specified when this environment was created, or False
        is returned.


module = __import__(self.module_name, fromlist=['<STR_LIT>'], level=<NUM_LIT:0>)<EOL>try:<EOL><INDENT>return functools.reduce(getattr, self.attrs, module)<EOL><DEDENT>except AttributeError as exc:<EOL><INDENT>raise ImportError(str(exc))<EOL><DEDENT>

Resolve the entry point from its module and attrs.


_provider_factories[loader_type] = provider_factory<EOL>

Register `provider_factory` to make providers for `loader_type`

    `loader_type` is the type or class of a PEP 302 ``module.__loader__``,
    and `provider_factory` is a function that, passed a *module* object,
    returns an ``IResourceProvider`` for that module.


Does the package's distribution contain the named metadata?


return list(self.itervalues())<EOL>

Dict-like values() that returns a list of values of cookies from the
        jar. See keys() and items().


if not isinstance(cookiejar, cookielib.CookieJar):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if isinstance(cookies, dict):<EOL><INDENT>cookiejar = cookiejar_from_dict(<EOL>cookies, cookiejar=cookiejar, overwrite=False)<EOL><DEDENT>elif isinstance(cookies, cookielib.CookieJar):<EOL><INDENT>try:<EOL><INDENT>cookiejar.update(cookies)<EOL><DEDENT>except AttributeError:<EOL><INDENT>for cookie_in_jar in cookies:<EOL><INDENT>cookiejar.set_cookie(cookie_in_jar)<EOL><DEDENT><DEDENT><DEDENT>return cookiejar<EOL>

Add cookies to cookiejar and returns a merged CookieJar.

    :param cookiejar: CookieJar object to add the cookies to.
    :param cookies: Dictionary or CookieJar object to be added.


def get_dict(self, domain=None, path=None):

dictionary = {}<EOL>for cookie in iter(self):<EOL><INDENT>if (domain is None or cookie.domain == domain) and (path is None<EOL>or cookie.path == path):<EOL><INDENT>dictionary[cookie.name] = cookie.value<EOL><DEDENT><DEDENT>return dictionary<EOL>

Takes as an argument an optional domain and path and returns a plain
        old Python dict of name-value pairs of cookies that meet the
        requirements.


remove_cookie_by_name(self, name)<EOL>

Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s
        ``remove_cookie_by_name()``.


new_cj = RequestsCookieJar()<EOL>new_cj.update(self)<EOL>return new_cj<EOL>

Return a copy of this RequestsCookieJar.