@log_calls<EOL><INDENT>def put_single_file(self, pool, source, target):<DEDENT>

if os.path.isdir(source):<EOL><INDENT>if self.opt.recursive:<EOL><INDENT>for f in (f for f in self.local_walk(source) if not os.path.isdir(f)):<EOL><INDENT>target_url = S3URL(target)<EOL>joined_path = os.path.normpath(os.path.join(target_url.path, os.path.relpath(f, source)))<EOL>pool.upload(f, S3URL.combine('<STR_LIT>', target_url.bucket, joined_path))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>message('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pool.upload(source, target)<EOL><DEDENT>

Upload a single file or a directory by adding a task into queue


@log_calls<EOL><INDENT>def put_files(self, source, target):<DEDENT>

pool = ThreadPool(ThreadUtil, self.opt)<EOL>if not isinstance(source, list):<EOL><INDENT>source = [source]<EOL><DEDENT>if target[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>for src in source:<EOL><INDENT>self.put_single_file(pool, src, os.path.join(target, self.get_basename(src)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(source) == <NUM_LIT:1>:<EOL><INDENT>self.put_single_file(pool, source[<NUM_LIT:0>], target)<EOL><DEDENT>else:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT><DEDENT>pool.join()<EOL>

Upload files to S3.
           This function can handle multiple file upload if source is a list.
           Also, it works for recursive mode which copy all files and keep the
           directory structure under the given source directory.


s3url = S3URL(source)<EOL>message('<STR_LIT>', source)<EOL>if not self.opt.dry_run:<EOL><INDENT>resp = self.s3.create_bucket(Bucket=s3url.bucket)<EOL>if resp['<STR_LIT>']["<STR_LIT>"] == <NUM_LIT:200>:<EOL><INDENT>message('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise Failure('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>

Use the create_bucket API to create a new bucket


@log_calls<EOL><INDENT>def update_privilege(self, obj, target):<DEDENT>

if '<STR_LIT>' in obj['<STR_LIT>']:<EOL><INDENT>os.chmod(target, int(obj['<STR_LIT>']['<STR_LIT>'], <NUM_LIT:8>))<EOL><DEDENT>

Get privileges from metadata of the source in s3, and apply them to target


sources = self.source_expand(source)<EOL>for source in sources:<EOL><INDENT>s3url = S3URL(source)<EOL>response = self.s3.get_object(Bucket=s3url.bucket, Key=s3url.path)<EOL>message('<STR_LIT:%s>', response['<STR_LIT>'].read())<EOL><DEDENT>


@log_calls<EOL><INDENT>def get_single_file(self, pool, source, target):<DEDENT>

if source[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>if self.opt.recursive:<EOL><INDENT>basepath = S3URL(source).path<EOL>for f in (f for f in self.s3walk(source) if not f['<STR_LIT>']):<EOL><INDENT>pool.download(f['<STR_LIT:name>'], os.path.join(target, os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>message('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pool.download(source, target)<EOL><DEDENT>

Download a single file or a directory by adding a task into queue


@log_calls<EOL><INDENT>def get_files(self, source, target):<DEDENT>

pool = ThreadPool(ThreadUtil, self.opt)<EOL>source = self.source_expand(source)<EOL>if os.path.isdir(target):<EOL><INDENT>for src in source:<EOL><INDENT>self.get_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(source) > <NUM_LIT:1>:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>elif len(source) == <NUM_LIT:1>:<EOL><INDENT>self.get_single_file(pool, source[<NUM_LIT:0>], target)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>pool.join()<EOL>

Download files.
           This function can handle multiple files if source S3 URL has wildcard
           characters. It also handles recursive mode by download all files and
           keep the directory structure.


@log_calls<EOL><INDENT>def delete_removed_files(self, source, target):<DEDENT>

message("<STR_LIT>", source, target)<EOL>if os.path.isdir(source):<EOL><INDENT>unecessary = []<EOL>basepath = S3URL(target).path<EOL>for f in [f for f in self.s3walk(target) if not f['<STR_LIT>']]:<EOL><INDENT>local_name = os.path.join(source, os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath))<EOL>if not os.path.isfile(local_name):<EOL><INDENT>message("<STR_LIT>", local_name)<EOL>unecessary.append(f['<STR_LIT:name>'])<EOL><DEDENT><DEDENT>if len(unecessary) > <NUM_LIT:0>:<EOL><INDENT>pool = ThreadPool(ThreadUtil, self.opt)<EOL>for del_file in unecessary:<EOL><INDENT>pool.delete(del_file)<EOL><DEDENT>pool.join()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>

Remove remote files that are not present in the local source.
           (Obsolete) It is used for old sync command now.


@log_calls<EOL><INDENT>def cp_single_file(self, pool, source, target, delete_source):<DEDENT>

if source[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>if self.opt.recursive:<EOL><INDENT>basepath = S3URL(source).path<EOL>for f in (f for f in self.s3walk(source) if not f['<STR_LIT>']):<EOL><INDENT>pool.copy(f['<STR_LIT:name>'], os.path.join(target, os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath)), delete_source=delete_source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>message('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pool.copy(source, target, delete_source=delete_source)<EOL><DEDENT>

Copy a single file or a directory by adding a task into queue


@log_calls<EOL><INDENT>def cp_files(self, source, target, delete_source=False):<DEDENT>

pool = ThreadPool(ThreadUtil, self.opt)<EOL>source = self.source_expand(source)<EOL>if target[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>for src in source:<EOL><INDENT>self.cp_single_file(pool, src, os.path.join(target, self.get_basename(S3URL(src).path)), delete_source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if len(source) > <NUM_LIT:1>:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>elif len(source) == <NUM_LIT:1>:<EOL><INDENT>self.cp_single_file(pool, source[<NUM_LIT:0>], target, delete_source)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>pool.join()<EOL>

Copy files
           This function can handle multiple files if source S3 URL has wildcard
           characters. It also handles recursive mode by copying all files and
           keep the directory structure.


src_files = []<EOL>for obj in self.s3walk(source):<EOL><INDENT>if not obj['<STR_LIT>']: <EOL><INDENT>src_files.append(obj['<STR_LIT:name>'])<EOL><DEDENT><DEDENT>pool = ThreadPool(ThreadUtil, self.opt)<EOL>pool.batch_delete(src_files)<EOL>pool.join()<EOL>


result = []<EOL>if S3URL.is_valid(dir):<EOL><INDENT>basepath = S3URL(dir).path<EOL>for f in (f for f in self.s3walk(dir) if not f['<STR_LIT>']):<EOL><INDENT>result.append(os.path.relpath(S3URL(f['<STR_LIT:name>']).path, basepath))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for f in (f for f in self.local_walk(dir) if not os.path.isdir(f)):<EOL><INDENT>result.append(os.path.relpath(f, dir))<EOL><DEDENT><DEDENT>return result<EOL>

Generic version of directory walk. Return file list without base path
           for comparison.


@log_calls<EOL><INDENT>def dsync_files(self, source, target):<DEDENT>

src_s3_url = S3URL.is_valid(source)<EOL>dst_s3_url = S3URL.is_valid(target)<EOL>source_list = self.relative_dir_walk(source)<EOL>if len(source_list) == <NUM_LIT:0> or '<STR_LIT:.>' in source_list:<EOL><INDENT>raise Failure('<STR_LIT>')<EOL><DEDENT>sync_list = [(os.path.join(source, f), os.path.join(target, f)) for f in source_list]<EOL>pool = ThreadPool(ThreadUtil, self.opt)<EOL>if src_s3_url and not dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.download(src, dest)<EOL><DEDENT><DEDENT>elif not src_s3_url and dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.upload(src, dest)<EOL><DEDENT><DEDENT>elif src_s3_url and dst_s3_url:<EOL><INDENT>for src, dest in sync_list:<EOL><INDENT>pool.copy(src, dest)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>pool.join()<EOL>if self.opt.delete_removed:<EOL><INDENT>target_list = self.relative_dir_walk(target)<EOL>remove_list = [os.path.join(target, f) for f in (set(target_list) - set(source_list))]<EOL>if S3URL.is_valid(target):<EOL><INDENT>pool = ThreadPool(ThreadUtil, self.opt)<EOL>pool.batch_delete(remove_list)<EOL>pool.join()<EOL><DEDENT>else:<EOL><INDENT>for f in remove_list:<EOL><INDENT>try:<EOL><INDENT>os.unlink(f)<EOL>message('<STR_LIT>', f)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


@log_calls<EOL><INDENT>def sync_files(self, source, target):<DEDENT>

src_s3_url = S3URL.is_valid(source)<EOL>dst_s3_url = S3URL.is_valid(target)<EOL>if src_s3_url and not dst_s3_url:<EOL><INDENT>self.get_files(source, target)<EOL><DEDENT>elif not src_s3_url and dst_s3_url:<EOL><INDENT>self.put_files(source, target)<EOL>if self.opt.delete_removed:<EOL><INDENT>self.delete_removed_files(source, target)<EOL><DEDENT><DEDENT>elif src_s3_url and dst_s3_url:<EOL><INDENT>self.cp_files(source, target)<EOL><DEDENT>else:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>

Sync files to S3. Does implement deletions if syncing TO s3.
           Currently identical to get/put -r -f --sync-check with exception of deletions.


result = []<EOL>for src in self.source_expand(source):<EOL><INDENT>size = <NUM_LIT:0><EOL>for f in self.s3walk(src):<EOL><INDENT>size += f['<STR_LIT:size>']<EOL><DEDENT>result.append((src, size))<EOL><DEDENT>return result<EOL>

Get the size component of the given s3url. If it is a
           directory, combine the sizes of all the files under
           that directory. Subdirectories will not be counted unless
           --recursive option is set.


self.filename = filename<EOL>self.md5 = None<EOL>


def file_hash(self, filename, block_size=<NUM_LIT:2>**<NUM_LIT:20>):

m = hashlib.md5()<EOL>with open(filename, '<STR_LIT:rb>') as f:<EOL><INDENT>while True:<EOL><INDENT>data = f.read(block_size)<EOL>if not data:<EOL><INDENT>break<EOL><DEDENT>m.update(data)<EOL><DEDENT><DEDENT>return m.hexdigest()<EOL>

Calculate MD5 hash code for a local file


if self.md5 is None:<EOL><INDENT>self.md5 = self.file_hash(self.filename)<EOL><DEDENT>return self.md5<EOL>

Get or calculate MD5 value of the local file.


S3Handler.__init__(self, pool.opt)<EOL>ThreadPool.Worker.__init__(self, pool)<EOL>


path = os.path.dirname(target)<EOL>if path and path != PATH_SEP and not os.path.isdir(path):<EOL><INDENT>try:<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>except OSError as ose:<EOL><INDENT>if ose.errno != errno.EEXIST:<EOL><INDENT>raise Failure('<STR_LIT>' % (path,))<EOL><DEDENT><DEDENT><DEDENT>

Ensure all directories are created for a given target file.


@log_calls<EOL><INDENT>def sync_check(self, md5cache, remoteKey):<DEDENT>

if not remoteKey:<EOL><INDENT>return False<EOL><DEDENT>if not os.path.exists(md5cache.filename):<EOL><INDENT>return False<EOL><DEDENT>localmd5 = md5cache.get_md5()<EOL>return ('<STR_LIT>' in remoteKey and remoteKey['<STR_LIT>'] == '<STR_LIT>' % localmd5) or('<STR_LIT>' in remoteKey and remoteKey['<STR_LIT>'] == localmd5) or('<STR_LIT>' in remoteKey['<STR_LIT>'] and remoteKey['<STR_LIT>']['<STR_LIT>'] == localmd5)<EOL>

Check MD5 for a local file and a remote file.
           Return True if they have the same md5 hash, otherwise False.


@log_calls<EOL><INDENT>def partial_match(self, path, filter_path):<DEDENT>

if not path or not filter_path:<EOL><INDENT>return True<EOL><DEDENT>if path[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>path = path[<NUM_LIT:0>:-<NUM_LIT:1>]<EOL><DEDENT>if filter_path[-<NUM_LIT:1>] == PATH_SEP:<EOL><INDENT>filter_path += '<STR_LIT:*>'<EOL><DEDENT>pi = path.split(PATH_SEP)<EOL>fi = filter_path.split(PATH_SEP)<EOL>min_len = min(len(pi), len(fi))<EOL>matched = fnmatch.fnmatch(PATH_SEP.join(pi[<NUM_LIT:0>:min_len]), PATH_SEP.join(fi[<NUM_LIT:0>:min_len]))<EOL>return matched and (self.opt.recursive or len(pi) <= len(fi))<EOL>

Partially match a path and a filter_path with wildcards.
           This function will return True if this path partially match a filter path.
           This is used for walking through directories with multiple level wildcard.


@log_calls<EOL><INDENT>def s3walk(self, s3url, s3dir, filter_path, result):<DEDENT>

paginator = self.s3.get_paginator('<STR_LIT>')<EOL>filter_path_level = filter_path.count(PATH_SEP)<EOL>for page in paginator.paginate(Bucket=s3url.bucket, Prefix=s3dir, Delimiter=PATH_SEP, PaginationConfig={'<STR_LIT>': <NUM_LIT:1000>}):<EOL><INDENT>for obj in page.get('<STR_LIT>') or []:<EOL><INDENT>obj_name = obj['<STR_LIT>']<EOL>if not self.partial_match(obj_name, filter_path):<EOL><INDENT>continue<EOL><DEDENT>if self.opt.recursive or (obj_name.count(PATH_SEP) != filter_path_level + <NUM_LIT:1>):<EOL><INDENT>self.pool.s3walk(s3url, obj_name, filter_path, result)<EOL><DEDENT>else:<EOL><INDENT>self.conditional(result, {<EOL>'<STR_LIT:name>': S3URL.combine(s3url.proto, s3url.bucket, obj_name),<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT:size>': <NUM_LIT:0>,<EOL>'<STR_LIT>': None<EOL>})<EOL><DEDENT><DEDENT>for obj in page.get('<STR_LIT>') or []:<EOL><INDENT>obj_name = obj['<STR_LIT>']<EOL>if not self.partial_match(obj_name, filter_path):<EOL><INDENT>continue<EOL><DEDENT>if self.opt.recursive or obj_name.count(PATH_SEP) == filter_path_level:<EOL><INDENT>self.conditional(result, {<EOL>'<STR_LIT:name>': S3URL.combine(s3url.proto, s3url.bucket, obj_name),<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT:size>': obj['<STR_LIT>'],<EOL>'<STR_LIT>': obj['<STR_LIT>']<EOL>})<EOL><DEDENT><DEDENT><DEDENT>

Thread worker for s3walk.
           Recursively walk into all subdirectories if they still match the filter
           path partially.


fileonly = (self.opt.last_modified_before is not None) or (self.opt.last_modified_after is not None)<EOL>if obj['<STR_LIT>']:<EOL><INDENT>if not fileonly:<EOL><INDENT>result.append(obj)<EOL><DEDENT>return<EOL><DEDENT>if (self.opt.last_modified_before is not None) and obj['<STR_LIT>'] >= self.opt.last_modified_before:<EOL><INDENT>return<EOL><DEDENT>if (self.opt.last_modified_after is not None) and obj['<STR_LIT>'] <= self.opt.last_modified_after:<EOL><INDENT>return<EOL><DEDENT>result.append(obj)<EOL>

Check all file item with given conditions.


@log_calls<EOL><INDENT>def get_file_splits(self, id, source, target, fsize, splitsize):<DEDENT>

pos = <NUM_LIT:0><EOL>part = <NUM_LIT:1> <EOL>mpi = ThreadUtil.MultipartItem(id)<EOL>splits = []<EOL>while pos < fsize:<EOL><INDENT>chunk = min(splitsize, fsize - pos)<EOL>assert(chunk > <NUM_LIT:0>)<EOL>splits.append((source, target, mpi, pos, chunk, part))<EOL>part += <NUM_LIT:1><EOL>pos += chunk<EOL><DEDENT>mpi.total = len(splits)<EOL>return splits<EOL>

Get file splits for upload/download/copy operation.


try:<EOL><INDENT>return str(oct(os.stat(source).st_mode)[-<NUM_LIT:3>:])<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise Failure('<STR_LIT>', source, e)<EOL><DEDENT>


try:<EOL><INDENT>return self.s3.head_object(Bucket=s3url.bucket, Key=s3url.path)<EOL><DEDENT>except BotoClient.ClientError as e:<EOL><INDENT>if e.response['<STR_LIT>']['<STR_LIT>'] == <NUM_LIT>:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>

Get the s3 object with the S3 URL. Return None if not exist.


@log_calls<EOL><INDENT>def read_file_chunk(self, source, pos, chunk):<DEDENT>

if chunk==<NUM_LIT:0>:<EOL><INDENT>return StringIO()<EOL><DEDENT>data = None<EOL>with open(source, '<STR_LIT:rb>') as f:<EOL><INDENT>f.seek(pos)<EOL>data = f.read(chunk)<EOL><DEDENT>if not data:<EOL><INDENT>raise Failure('<STR_LIT>' % source)<EOL><DEDENT>return StringIO(data)<EOL>


@log_calls<EOL><INDENT>def upload(self, source, target, mpi=None, pos=<NUM_LIT:0>, chunk=<NUM_LIT:0>, part=<NUM_LIT:0>):<DEDENT>

s3url = S3URL(target)<EOL>obj = self.lookup(s3url)<EOL>if not mpi:<EOL><INDENT>fsize = os.path.getsize(source)<EOL>md5cache = LocalMD5Cache(source)<EOL>if self.opt.dry_run:<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif self.opt.sync_check and self.sync_check(md5cache, obj):<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif not self.opt.force and obj:<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>if fsize < self.opt.max_singlepart_upload_size:<EOL><INDENT>data = self.read_file_chunk(source, <NUM_LIT:0>, fsize)<EOL>self.s3.put_object(Bucket=s3url.bucket,<EOL>Key=s3url.path,<EOL>Body=data,<EOL>Metadata={'<STR_LIT>': md5cache.get_md5(),<EOL>'<STR_LIT>': self.get_file_privilege(source)})<EOL>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>response = self.s3.create_multipart_upload(Bucket=s3url.bucket,<EOL>Key=s3url.path,<EOL>Metadata={'<STR_LIT>': md5cache.get_md5(),<EOL>'<STR_LIT>': self.get_file_privilege(source)})<EOL>upload_id = response['<STR_LIT>']<EOL>for args in self.get_file_splits(upload_id, source, target, fsize, self.opt.multipart_split_size):<EOL><INDENT>self.pool.upload(*args)<EOL><DEDENT>return<EOL><DEDENT>data = self.read_file_chunk(source, pos, chunk)<EOL>response = self.s3.upload_part(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id, Body=data, PartNumber=part)<EOL>if mpi.complete({'<STR_LIT>': response['<STR_LIT>'], '<STR_LIT>': part}):<EOL><INDENT>try:<EOL><INDENT>self.s3.complete_multipart_upload(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id, MultipartUpload={'<STR_LIT>': mpi.sorted_parts()})<EOL>message('<STR_LIT>', source, target)<EOL><DEDENT>except Exception as e:<EOL><INDENT>message('<STR_LIT>', str(e))<EOL>self.s3.abort_multipart_upload(Bucket=s3url.bucket, Key=s3url.path, UploadId=mpi.id)<EOL>raise RetryFailure('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>


@log_calls<EOL><INDENT>def _verify_file_size(self, obj, downloaded_file):<DEDENT>

file_size = os.path.getsize(downloaded_file)<EOL>if int(obj['<STR_LIT>']) != file_size:<EOL><INDENT>raise RetryFailure('<STR_LIT>' % (repr(obj)))<EOL><DEDENT>

Verify the file size of the downloaded file.


@log_calls<EOL><INDENT>def write_file_chunk(self, target, pos, chunk, body):<DEDENT>

fd = os.open(target, os.O_CREAT | os.O_WRONLY)<EOL>try:<EOL><INDENT>os.lseek(fd, pos, os.SEEK_SET)<EOL>data = body.read(chunk)<EOL>num_bytes_written = os.write(fd, data)<EOL>if(num_bytes_written != len(data)):<EOL><INDENT>raise RetryFailure('<STR_LIT>' % (num_bytes_written, sys.getsizeof(data)))<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>os.close(fd)<EOL><DEDENT>


@log_calls<EOL><INDENT>def download(self, source, target, mpi=None, pos=<NUM_LIT:0>, chunk=<NUM_LIT:0>, part=<NUM_LIT:0>):<DEDENT>

s3url = S3URL(source)<EOL>obj = self.lookup(s3url)<EOL>if obj is None:<EOL><INDENT>raise Failure('<STR_LIT>' % (s3url.path,))<EOL><DEDENT>if not mpi:<EOL><INDENT>if self.opt.dry_run:<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif self.opt.sync_check and self.sync_check(LocalMD5Cache(target), obj):<EOL><INDENT>message('<STR_LIT>', source, target)<EOL>return<EOL><DEDENT>elif not self.opt.force and os.path.exists(target):<EOL><INDENT>raise Failure('<STR_LIT>' % target)<EOL><DEDENT>fsize = int(obj['<STR_LIT>'])<EOL>if fsize < self.opt.max_singlepart_download_size:<EOL><INDENT>mpi = ThreadUtil.MultipartItem(tempfile_get(target))<EOL>mpi.total = <NUM_LIT:1><EOL>pos = <NUM_LIT:0><EOL>chunk = fsize<EOL><DEDENT>else:<EOL><INDENT>for args in self.get_file_splits(tempfile_get(target), source, target, fsize, self.opt.multipart_split_size):<EOL><INDENT>self.pool.download(*args)<EOL><DEDENT>return<EOL><DEDENT><DEDENT>tempfile = mpi.id<EOL>if self.opt.recursive:<EOL><INDENT>self.mkdirs(tempfile)<EOL><DEDENT>response = self.s3.get_object(Bucket=s3url.bucket, Key=s3url.path, Range='<STR_LIT>' % (pos, pos + chunk - <NUM_LIT:1>))<EOL>self.write_file_chunk(tempfile, pos, chunk, response['<STR_LIT>'])<EOL>if mpi.complete({'<STR_LIT>': part}):<EOL><INDENT>try:<EOL><INDENT>self.update_privilege(obj, tempfile)<EOL>self._verify_file_size(obj, tempfile)<EOL>tempfile_set(tempfile, target)<EOL>message('<STR_LIT>', source, target)<EOL><DEDENT>except Exception as e:<EOL><INDENT>tempfile_set(tempfile, None)<EOL>raise Failure('<STR_LIT>' % (e.message, source))<EOL><DEDENT><DEDENT>


@log_calls<EOL><INDENT>def copy(self, source, target, mpi=None, pos=<NUM_LIT:0>, chunk=<NUM_LIT:0>, part=<NUM_LIT:0>, delete_source=False):<DEDENT>

if self.opt.dry_run:<EOL><INDENT>message('<STR_LIT>' % (source, target))<EOL>return<EOL><DEDENT>source_url = S3URL(source)<EOL>target_url = S3URL(target)<EOL>if not mpi:<EOL><INDENT>obj = self.lookup(source_url)<EOL>fsize = int(obj['<STR_LIT>'])<EOL>if fsize < self.opt.max_singlepart_copy_size:<EOL><INDENT>self.s3.copy_object(Bucket=target_url.bucket, Key=target_url.path,<EOL>CopySource={'<STR_LIT>': source_url.bucket, '<STR_LIT>': source_url.path})<EOL>message('<STR_LIT>' % (source, target))<EOL>if delete_source:<EOL><INDENT>self.delete(source)<EOL><DEDENT>return<EOL><DEDENT>response = self.s3.create_multipart_upload(Bucket=target_url.bucket,<EOL>Key=target_url.path,<EOL>Metadata=obj['<STR_LIT>'])<EOL>upload_id = response['<STR_LIT>']<EOL>for args in self.get_file_splits(upload_id, source, target, fsize, self.opt.multipart_split_size):<EOL><INDENT>self.pool.copy(*args, delete_source=delete_source)<EOL><DEDENT>return<EOL><DEDENT>response = self.s3.upload_part_copy(Bucket=target_url.bucket,<EOL>Key=target_url.path,<EOL>CopySource={'<STR_LIT>': source_url.bucket, '<STR_LIT>': source_url.path},<EOL>CopySourceRange='<STR_LIT>' % (pos, pos + chunk - <NUM_LIT:1>),<EOL>UploadId=mpi.id,<EOL>PartNumber=part)<EOL>if mpi.complete({'<STR_LIT>': response['<STR_LIT>']['<STR_LIT>'], '<STR_LIT>': part}):<EOL><INDENT>try:<EOL><INDENT>self.s3.complete_multipart_upload(Bucket=target_url.bucket, Key=target_url.path, UploadId=mpi.id, MultipartUpload={'<STR_LIT>': mpi.sorted_parts()})<EOL>if delete_source:<EOL><INDENT>self.delete(source)<EOL><DEDENT>message('<STR_LIT>' % (source, target))<EOL><DEDENT>except Exception as e:<EOL><INDENT>message('<STR_LIT>', str(e))<EOL>self.s3.abort_multipart_upload(Bucket=source_url.bucket, Key=source_url.path, UploadId=mpi.id)<EOL>raise RetryFailure('<STR_LIT>' % source)<EOL><DEDENT><DEDENT>

Copy a single file from source to target using boto S3 library.


s3url = S3URL(source)<EOL>message('<STR_LIT>', source)<EOL>if not self.opt.dry_run:<EOL><INDENT>self.s3.delete_object(Bucket=s3url.bucket, Key=s3url.path)<EOL><DEDENT>


assert(type(sources) == list)<EOL>if len(sources) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>elif len(sources) == <NUM_LIT:1>:<EOL><INDENT>self.delete(sources[<NUM_LIT:0>])<EOL><DEDENT>elif len(sources) > self.opt.batch_delete_size:<EOL><INDENT>for i in range(<NUM_LIT:0>, len(sources), self.opt.batch_delete_size):<EOL><INDENT>self.pool.batch_delete(sources[i:i+self.opt.batch_delete_size])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>bucket = S3URL(sources[<NUM_LIT:0>]).bucket<EOL>deletes = []<EOL>for source in sources:<EOL><INDENT>s3url = S3URL(source)<EOL>if s3url.bucket != bucket:<EOL><INDENT>raise Failure('<STR_LIT>' % (s3url.bucket, bucket))<EOL><DEDENT>deletes.append({'<STR_LIT>': s3url.path})<EOL><DEDENT>response = self.s3.delete_objects(Bucket=bucket, Delete={'<STR_LIT>': deletes})<EOL>for res in response.get('<STR_LIT>') or []:<EOL><INDENT>message('<STR_LIT>', S3URL.combine('<STR_LIT>', bucket, res['<STR_LIT>']))<EOL><DEDENT>for err in response.get('<STR_LIT>') or []:<EOL><INDENT>message('<STR_LIT>', S3URL.combine('<STR_LIT>', bucket, res['<STR_LIT>']), err['<STR_LIT>'], err['<STR_LIT>'])<EOL><DEDENT>if response.get('<STR_LIT>') is not None:<EOL><INDENT>raise RetryFailure('<STR_LIT>' % len(response.get('<STR_LIT>')))<EOL><DEDENT><DEDENT>

Delete a list of files in batch of batch_delete_size (default=1000).


self.opt = opt<EOL>


if len(args) == <NUM_LIT:0>:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>cmd = args[<NUM_LIT:0>]<EOL>if cmd + '<STR_LIT>' in CommandHandler.__dict__:<EOL><INDENT>CommandHandler.__dict__[cmd + '<STR_LIT>'](self, args)<EOL><DEDENT>else:<EOL><INDENT>raise InvalidArgument('<STR_LIT>' % cmd)<EOL><DEDENT>

Main entry to handle commands. Dispatch to individual command handler.


return S3Handler(self.opt)<EOL>

Create a S3Handler instances for multithread operations.


@log_calls<EOL><INDENT>def validate(self, format, args):<DEDENT>

fmtMap = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>fmts = format.split('<STR_LIT:|>')<EOL>if len(fmts) != len(args):<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>for i, fmt in enumerate(fmts):<EOL><INDENT>valid = False<EOL>for f in fmt.split('<STR_LIT:U+002C>'):<EOL><INDENT>if f == '<STR_LIT>' and args[i] + '<STR_LIT>' in CommandHandler.__dict__:<EOL><INDENT>valid = True<EOL><DEDENT>if f == '<STR_LIT>' and S3URL.is_valid(args[i]):<EOL><INDENT>valid = True<EOL><DEDENT>if f == '<STR_LIT>' and not S3URL.is_valid(args[i]):<EOL><INDENT>valid = True<EOL><DEDENT><DEDENT>if not valid:<EOL><INDENT>raise InvalidArgument('<STR_LIT>' % (args[i], fmtMap[fmt.split('<STR_LIT:U+002C>')[<NUM_LIT:0>]]))<EOL><DEDENT><DEDENT>

Validate input parameters with given format.
           This function also checks for wildcards for recursive mode.


def normalize_time(timestamp):<EOL><INDENT>'''<STR_LIT>'''<EOL>if timestamp is None:<EOL><INDENT>return '<STR_LIT:U+0020>' * <NUM_LIT:16><EOL><DEDENT>return TIMESTAMP_FORMAT % (timestamp.year, timestamp.month, timestamp.day, timestamp.hour, timestamp.minute)<EOL><DEDENT>cwidth = [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]<EOL>format = '<STR_LIT>'<EOL>result = []<EOL>for obj in objlist:<EOL><INDENT>last_modified = normalize_time(obj['<STR_LIT>'])<EOL>size = str(obj['<STR_LIT:size>']) if not obj['<STR_LIT>'] else '<STR_LIT>'<EOL>name = obj['<STR_LIT:name>']<EOL>item = (last_modified, size, name)<EOL>for i, value in enumerate(item):<EOL><INDENT>if cwidth[i] < len(value):<EOL><INDENT>cwidth[i] = len(value)<EOL><DEDENT><DEDENT>result.append(item)<EOL><DEDENT>for item in result:<EOL><INDENT>text = (format % tuple(cwidth)) % item<EOL>message('<STR_LIT:%s>', text.rstrip())<EOL><DEDENT>

Pretty print the result of s3walk. Here we calculate the maximum width
           of each column and align them.


if len(args) == <NUM_LIT:1>:<EOL><INDENT>self.pretty_print(self.s3handler().list_buckets())<EOL>return<EOL><DEDENT>self.validate('<STR_LIT>', args)<EOL>self.pretty_print(self.s3handler().s3walk(args[<NUM_LIT:1>]))<EOL>


if len(args) == <NUM_LIT:1>:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>self.validate('<STR_LIT>', args)<EOL>self.s3handler().create_bucket(args[<NUM_LIT:1>])<EOL>


<EOL>if len(args) < <NUM_LIT:3>:<EOL><INDENT>raise InvalidArgument('<STR_LIT>')<EOL><DEDENT>self.validate('<STR_LIT:|>'.join(['<STR_LIT>'] + ['<STR_LIT>'] * (len(args) - <NUM_LIT:2>) + ['<STR_LIT>']), args)<EOL>source = args[<NUM_LIT:1>:-<NUM_LIT:1>] <EOL>target = args[-<NUM_LIT:1>]<EOL>self.s3handler().put_files(source, target)<EOL>


<EOL>if len(args) == <NUM_LIT:2>:<EOL><INDENT>args += ['<STR_LIT:.>']<EOL><DEDENT>self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().get_files(source, target)<EOL>


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>self.s3handler().print_files(source)<EOL>


self.opt.recursive = True<EOL>self.opt.sync_check = True<EOL>self.opt.force = True<EOL>self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().dsync_files(source, target)<EOL>


self.opt.recursive = True<EOL>self.opt.sync_check = True<EOL>self.opt.force = True<EOL>self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().sync_files(source, target)<EOL>

Handler for sync command.
           XXX Here we emulate sync command with get/put -r -f --sync-check. So
               it doesn't provide delete operation.


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().cp_files(source, target)<EOL>


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>target = args[<NUM_LIT:2>]<EOL>self.s3handler().cp_files(source, target, delete_source=True)<EOL>


self.validate('<STR_LIT>', args)<EOL>source = args[<NUM_LIT:1>]<EOL>self.s3handler().del_files(source)<EOL>


for src, size in self.s3handler().size(args[<NUM_LIT:1>:]):<EOL><INDENT>message('<STR_LIT>' % (size, src))<EOL><DEDENT>


total_size = <NUM_LIT:0><EOL>for src, size in self.s3handler().size(args[<NUM_LIT:1>:]):<EOL><INDENT>total_size += size<EOL><DEDENT>message(str(total_size))<EOL>


m = self.REGEX_DATE.search(value)<EOL>date = datetime.datetime.utcnow().date()<EOL>if m:<EOL><INDENT>date = datetime.date(int(m.group(<NUM_LIT:1>)), int(m.group(<NUM_LIT:2>)), int(m.group(<NUM_LIT:3>)))<EOL>value = self.REGEX_DATE.sub('<STR_LIT>', value)<EOL><DEDENT>return (date, value)<EOL>

Search for date information in the string


m = self.REGEX_TIME.search(value)<EOL>time = datetime.datetime.utcnow().time()<EOL>if m:<EOL><INDENT>time = datetime.time(int(m.group(<NUM_LIT:1>)), int(m.group(<NUM_LIT:2>)))<EOL>value = self.REGEX_TIME.sub('<STR_LIT>', value)<EOL><DEDENT>return (time, value)<EOL>

Search for time information in the string


m = self.REGEX_DELTA.search(value)<EOL>delta = datetime.timedelta(days=<NUM_LIT:0>)<EOL>if m:<EOL><INDENT>d = int(m.group(<NUM_LIT:1>))<EOL>if m.group(<NUM_LIT:3>) == '<STR_LIT>' or m.group(<NUM_LIT:3>) == '<STR_LIT>':<EOL><INDENT>d = -d<EOL><DEDENT>if m.group(<NUM_LIT:2>) == '<STR_LIT>':<EOL><INDENT>delta = datetime.timedelta(minutes=d)<EOL><DEDENT>elif m.group(<NUM_LIT:2>) == '<STR_LIT>':<EOL><INDENT>delta = datetime.timedelta(hours=d)<EOL><DEDENT>elif m.group(<NUM_LIT:2>) == '<STR_LIT>':<EOL><INDENT>delta = datetime.timedelta(days=d)<EOL><DEDENT>elif m.group(<NUM_LIT:2>) == '<STR_LIT>':<EOL><INDENT>delta = datetime.timedelta(weeks=d)<EOL><DEDENT>value = self.REGEX_DELTA.sub('<STR_LIT>', value)<EOL><DEDENT>return (delta, value)<EOL>

Search for timedelta information in the string


try:<EOL><INDENT>return json.loads(value)<EOL><DEDENT>except:<EOL><INDENT>raise optparse.OptionValueError("<STR_LIT>" % (opt, value))<EOL><DEDENT>


class K(object):<EOL><INDENT>def __init__(self, obj, *args):<EOL><INDENT>self.obj = obj<EOL><DEDENT>def __lt__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) < <NUM_LIT:0><EOL><DEDENT>def __gt__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) > <NUM_LIT:0><EOL><DEDENT>def __eq__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) == <NUM_LIT:0><EOL><DEDENT>def __le__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) <= <NUM_LIT:0><EOL><DEDENT>def __ge__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) >= <NUM_LIT:0><EOL><DEDENT>def __ne__(self, other):<EOL><INDENT>return mycmp(self.obj, other.obj) != <NUM_LIT:0><EOL><DEDENT>def __hash__(self):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT><DEDENT>return K<EOL>

Converts a cmp= function into a key= function.


<EOL>from commands.download import Download<EOL>options = docopt(__doc__, version=VERSION)<EOL>print("<STR_LIT>")<EOL>p=Download(options)<EOL>p.run()<EOL>


It will print the list of songs that can be downloaded


soup=BeautifulSoup(html)<EOL>links=[]<EOL>a_list=soup.findAll('<STR_LIT:a>','<STR_LIT>')<EOL>for x in range(len(a_list)-<NUM_LIT:1>):<EOL><INDENT>link = a_list[x].get('<STR_LIT>')<EOL>name = a_list[x]<EOL>name = str(name)<EOL>name=re.sub(r'<STR_LIT>','<STR_LIT>',name)<EOL>name=re.sub(r'<STR_LIT>','<STR_LIT>',name)<EOL>links.append([link,name])<EOL><DEDENT>return links<EOL>

It will return all hyper links found in the mr-jatt page for download


soup=BeautifulSoup(html)<EOL>a_list=soup.findAll('<STR_LIT:a>','<STR_LIT>')<EOL>text=[str(x) for x in a_list]<EOL>text='<STR_LIT>'.join(text)<EOL>text=text.lower()<EOL>string1='<STR_LIT>'<EOL>string2='<STR_LIT>'<EOL>string3='<STR_LIT>'<EOL>href='<STR_LIT>'<EOL>if string3 in text:<EOL><INDENT>href=a_list[<NUM_LIT:2>].get('<STR_LIT>')<EOL><DEDENT>elif string2 in text:<EOL><INDENT>href=a_list[<NUM_LIT:1>].get('<STR_LIT>')<EOL><DEDENT>elif string1 in text:<EOL><INDENT>href=a_list[<NUM_LIT:0>].get('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>return (True,'<STR_LIT>')  <EOL><DEDENT>return (False,href)<EOL>

Returns true if user entered artist or movie name


file_download=FileDownload()<EOL>html=file_download.get_html_response(url)<EOL>if flag == False:<EOL><INDENT>soup=BeautifulSoup(html)<EOL>a_list=soup.findAll('<STR_LIT:a>','<STR_LIT>')<EOL>text=[str(x) for x in a_list]<EOL>text='<STR_LIT>'.join(text)<EOL>text=text.lower()<EOL>string1='<STR_LIT>'<EOL>string2='<STR_LIT>'<EOL>string3='<STR_LIT>'<EOL>href='<STR_LIT>'<EOL>if string3 in text:<EOL><INDENT>print('<STR_LIT>')<EOL>href=a_list[<NUM_LIT:2>].get('<STR_LIT>')<EOL><DEDENT>elif string2 in text:<EOL><INDENT>print('<STR_LIT>')<EOL>href=a_list[<NUM_LIT:1>].get('<STR_LIT>')<EOL><DEDENT>elif string1 in text:<EOL><INDENT>print('<STR_LIT>') <EOL>href=a_list[<NUM_LIT:0>].get('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>self.missing_schema(html,song_name)<EOL>quit()  <EOL><DEDENT>return href <EOL><DEDENT>else:<EOL><INDENT>x,href=self.check_if_song_name(html)<EOL>links = []<EOL>if x==True:<EOL><INDENT>links=self.list_of_all_href(html)<EOL><DEDENT>else:<EOL><INDENT>file_download=FileDownload()<EOL>file_download.file_download_cross_platform(href)<EOL>quit() <EOL><DEDENT>return links<EOL><DEDENT>

It will the resource URL if song is found,
Otherwise it will return the list of songs that can be downloaded


raise NotImplementedError('<STR_LIT>')<EOL>

It will parse the html page specified by url
        and return the resource url from where the music file needs to be downloaded


It will search for "song_name website"  in the Search Engine results
and return the url of website


name='<STR_LIT:+>'.join(song_name)<EOL>prefix='<STR_LIT>' <EOL>website=website.split("<STR_LIT:U+0020>")<EOL>suffix='<STR_LIT:+>'.join(website)<EOL>url=prefix+name+suffix<EOL>return url<EOL>

It will return the google url to be searched


It will parse google html response
                and return the first url


song_name is a list of strings
website is a string
It will return the url from where music file needs to be downloaded


print("<STR_LIT>"%url)<EOL>try:<EOL><INDENT>response=requests.get(url,timeout=<NUM_LIT:50>)<EOL><DEDENT>except requests.exceptions.SSLError:<EOL><INDENT>try:<EOL><INDENT>response=requests.get(url,verify=False,timeout=<NUM_LIT:50>)<EOL><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e)<EOL>quit()<EOL><DEDENT><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e) <EOL>quit()<EOL><DEDENT>return response.content<EOL>

It will download the html page specified by url and return the html response


file_name=url.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>if os.path.exists(os.path.join(os.getcwd(),file_name)):<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>try:<EOL><INDENT>r=requests.get(url,stream=True,timeout=<NUM_LIT:200>)<EOL><DEDENT>except requests.exceptions.SSLError:<EOL><INDENT>try:<EOL><INDENT>response=requests.get(url,stream=True,verify=False,timeout=<NUM_LIT:200>)<EOL><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e)<EOL>quit()  <EOL><DEDENT><DEDENT>except requests.exceptions.RequestException as e:<EOL><INDENT>print(e)<EOL>quit() <EOL><DEDENT>chunk_size = <NUM_LIT><EOL>total_size = int(r.headers['<STR_LIT>'])<EOL>total_chunks = total_size/chunk_size<EOL>file_iterable = r.iter_content(chunk_size = chunk_size)<EOL>tqdm_iter = tqdm(iterable = file_iterable,total = total_chunks,unit = '<STR_LIT>',<EOL>leave = False<EOL>)  <EOL>with open(file_name,'<STR_LIT:wb>') as f:<EOL><INDENT>for data in tqdm_iter:<EOL><INDENT>f.write(data)<EOL><DEDENT><DEDENT>'''<STR_LIT>'''<EOL>print('<STR_LIT>'%file_name)<EOL>

It will download file specified by url using requests module


file_name=url.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>print('<STR_LIT>'%file_name)<EOL>command='<STR_LIT>'<EOL>url='<STR_LIT:">'+url+'<STR_LIT:">'<EOL>command=command+url<EOL>os.system(command)<EOL>

It will download file specified by url using wget utility of linux


def main(properties=properties, options=options, **custom_options):

return init(**dict(options, **custom_options))(**properties)<EOL>

Imports and runs setup function with given properties.


if not minver == maxver == None:<EOL><INDENT>import sys<EOL>if not minver <= sys.version < (maxver or '<STR_LIT>'):<EOL><INDENT>sys.stderr.write(<EOL>'<STR_LIT>' % (<EOL>sys.argv[<NUM_LIT:0>], minver or '<STR_LIT>', maxver or '<STR_LIT>', sys.version.split()[<NUM_LIT:0>]))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if use_distribute:<EOL><INDENT>from distribute_setup import use_setuptools<EOL>use_setuptools(to_dir=dist)<EOL>from setuptools import setup<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>from setuptools import setup<EOL><DEDENT>except ImportError:<EOL><INDENT>from distutils.core import setup<EOL><DEDENT><DEDENT>if use_markdown_readme:<EOL><INDENT>try:<EOL><INDENT>import setuptools.command.sdist<EOL>setuptools.command.sdist.READMES = tuple(list(getattr(setuptools.command.sdist, '<STR_LIT>', ()))<EOL>+ ['<STR_LIT>'])<EOL><DEDENT>except ImportError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if use_stdeb:<EOL><INDENT>import platform<EOL>if '<STR_LIT>' in platform.dist():<EOL><INDENT>try:<EOL><INDENT>import stdeb<EOL><DEDENT>except ImportError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>return setup<EOL>

Imports and returns a setup function.

    If use_markdown_readme is set,
    then README.md is added to setuptools READMES list.

    If use_stdeb is set on a Debian based system,
    then module stdeb is imported.
    Stdeb supports building deb packages on Debian based systems.
    The package should only be installed on the same system version
    it was built on, though. See http://github.com/astraw/stdeb.

    If use_distribute is set, then distribute_setup.py is imported.


model = copy.deepcopy(self)<EOL>model.coef_ += other.coef_<EOL>model.intercept_ += other.intercept_<EOL>return model<EOL>

Add method for Linear models with coef and intercept attributes.

        Parameters
        ----------
        other : fitted sklearn linear model
            Model to add.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


return self if other == <NUM_LIT:0> else self.__add__(other)<EOL>

Reverse add method for Linear models.

        Parameters
        ----------
        other : fitted sklearn linear model
            Model to add.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


self.coef_ /= other<EOL>self.intercept_ /= other<EOL>return self<EOL>

Division method for Linear models. Used for averaging.

        Parameters
        ----------
        other : integer
            Integer to divide with.

        Returns
        -------
        model : Linear model
            Model with updated coefficients.


def _spark_fit(self, cls, Z, *args, **kwargs):

mapper = lambda X_y: super(cls, self).fit(<EOL>X_y[<NUM_LIT:0>], X_y[<NUM_LIT:1>], *args, **kwargs<EOL>)<EOL>models = Z.map(mapper)<EOL>avg = models.reduce(operator.add) / models.count()<EOL>self.__dict__.update(avg.__dict__)<EOL>return self<EOL>

Wraps a Scikit-learn Linear model's fit method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : TupleRDD or DictRDD
            The distributed train data in a DictRDD.

        Returns
        -------
        self: the wrapped class


def _spark_predict(self, cls, X, *args, **kwargs):

return X.map(lambda X: super(cls, self).predict(X, *args, **kwargs))<EOL>

Wraps a Scikit-learn Linear model's predict method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : ArrayRDD
            The distributed data to predict in a DictRDD.

        Returns
        -------
        self: the wrapped class


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>return self._spark_fit(SparkLinearRegression, Z)<EOL>

Fit linear model.

Parameters
----------
Z : DictRDD with (X, y) values
    X containing numpy array or sparse matrix - The training data
    y containing the target values

Returns
-------
self : returns an instance of self.


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return self._spark_predict(SparkLinearRegression, X)<EOL>

Distributed method to predict class labels for samples in X.

        Parameters
        ----------
        X : ArrayRDD containing {array-like, sparse matrix}
            Samples.

        Returns
        -------
        C : ArrayRDD
            Predicted class label per sample.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>self._classes_ = np.unique(classes)<EOL>return self._spark_fit(SparkSGDClassifier, Z)<EOL>

Fit the model according to the given training data.

        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector
            y - Target labels
        classes : iterable
            The set of available classes

        Returns
        -------
        self : object
            Returns self.


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return self._spark_predict(SparkSGDClassifier, X)<EOL>


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>self._classes_ = np.unique(classes)<EOL>return self._spark_fit(SparkLogisticRegression, Z)<EOL>


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return self._spark_predict(SparkLogisticRegression, X)<EOL>


<EOL>c = blocked_rdd.map(lambda x: (x.T.dot(x), x.shape[<NUM_LIT:0>]))<EOL>prod, n = c.reduce(lambda x, y: (x[<NUM_LIT:0>] + y[<NUM_LIT:0>], x[<NUM_LIT:1>] + y[<NUM_LIT:1>]))<EOL>w, v = ln.eig(prod / n)<EOL>w = np.real(w)<EOL>v = np.real(v)<EOL>inds = np.argsort(w)[::-<NUM_LIT:1>]<EOL>s = np.sqrt(w[inds[<NUM_LIT:0>:k]]) * np.sqrt(n)<EOL>v = v[:, inds[<NUM_LIT:0>:k]].T<EOL>u = blocked_rdd.map(lambda x: np.inner(x, v) / s)<EOL>return u, s, v<EOL>

Calculate the SVD of a blocked RDD directly, returning only the leading k
singular vectors. Assumes n rows and d columns, efficient when n >> d
Must be able to fit d^2 within the memory of a single machine.
Parameters
----------
blocked_rdd : RDD
    RDD with data points in numpy array blocks
k : Int
    Number of singular vectors to return
Returns
----------
u : RDD of blocks
    Left eigenvectors
s : numpy array
    Singular values
v : numpy array
    Right eigenvectors


def svd_em(blocked_rdd, k, maxiter=<NUM_LIT:20>, tol=<NUM_LIT>, compute_u=True, seed=None):

n, m = blocked_rdd.shape[:<NUM_LIT:2>]<EOL>sc = blocked_rdd._rdd.context<EOL>def outerprod(x):<EOL><INDENT>return x.T.dot(x)<EOL><DEDENT>if seed is not None:<EOL><INDENT>rng = np.random.RandomState(seed)<EOL>c = rng.randn(k, m)<EOL><DEDENT>else:<EOL><INDENT>c = np.random.randn(k, m)<EOL><DEDENT>iter = <NUM_LIT:0><EOL>error = <NUM_LIT:100><EOL>while (iter < maxiter) & (error > tol):<EOL><INDENT>c_old = c<EOL>c_inv = np.dot(c.T, ln.inv(np.dot(c, c.T)))<EOL>premult1 = sc.broadcast(c_inv)<EOL>xx = blocked_rdd.map(lambda x: outerprod(safe_sparse_dot(x, premult1.value))).treeReduce(add)<EOL>xx_inv = ln.inv(xx)<EOL>premult2 = blocked_rdd.context.broadcast(np.dot(c_inv, xx_inv))<EOL>c = blocked_rdd.map(lambda x: safe_sparse_dot(x.T, safe_sparse_dot(x, premult2.value))).treeReduce(add)<EOL>c = c.T<EOL>error = np.sum((c - c_old) ** <NUM_LIT:2>)<EOL>iter += <NUM_LIT:1><EOL><DEDENT>c = ln.orth(c.T).T<EOL>cov = blocked_rdd.map(lambda x: safe_sparse_dot(x, c.T)).map(lambda x: outerprod(x)).treeReduce(add)<EOL>w, v = ln.eig(cov / n)<EOL>w = np.real(w)<EOL>v = np.real(v)<EOL>inds = np.argsort(w)[::-<NUM_LIT:1>]<EOL>s = np.sqrt(w[inds[<NUM_LIT:0>:k]]) * np.sqrt(n)<EOL>v = np.dot(v[:, inds[<NUM_LIT:0>:k]].T, c)<EOL>if compute_u:<EOL><INDENT>v_broadcasted = blocked_rdd.context.broadcast(v)<EOL>u = blocked_rdd.map(<EOL>lambda x: safe_sparse_dot(x, v_broadcasted.value.T) / s)<EOL>return u, s, v<EOL><DEDENT>else:<EOL><INDENT>return s, v<EOL><DEDENT>

Calculate the SVD of a blocked RDD using an expectation maximization
algorithm (from Roweis, NIPS, 1997) that avoids explicitly
computing the covariance matrix, returning only the leading k
singular vectors. Assumes n rows and d columns, does not require
d^2 to fit into memory on a single machine.
Parameters
----------
blocked_rdd : ArrayRDD
    ArrayRDD with data points in numpy array blocks
k : Int
    Number of singular vectors to return
maxiter : Int, optional, default = 20
    Number of iterations to perform
tol : Double, optional, default = 1e-5
    Tolerance for stopping iterative updates
seed : Int, optional, default = None
    Seed for random number generator for initializing subspace
Returns
----------
u : RDD of blocks
    Left eigenvectors
s : numpy array
    Singular values
v : numpy array
    Right eigenvectors


self.fit_transform(Z)<EOL>return self<EOL>

Fit LSI model on training data X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self : object
            Returns the transformer object.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>if self.algorithm == "<STR_LIT>":<EOL><INDENT>X = X.persist()  <EOL>Sigma, V = svd_em(X, k=self.n_components, maxiter=self.n_iter,<EOL>tol=self.tol, compute_u=False,<EOL>seed=self.random_state)<EOL>self.components_ = V<EOL>X.unpersist()<EOL>return self.transform(Z)<EOL><DEDENT>else:<EOL><INDENT>return super(SparkTruncatedSVD, self).fit_transform(X.tosparse())<EOL><DEDENT>

Fit LSI model to X and perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>mapper = self.broadcast(<EOL>super(SparkTruncatedSVD, self).transform, Z.context)<EOL>return Z.transform(mapper, column='<STR_LIT:X>', dtype=np.ndarray)<EOL>

Perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>mapper = self.broadcast(<EOL>super(SparkTruncatedSVD, self).inverse_transform, Z.context)<EOL>return Z.transform(mapper, column='<STR_LIT:X>', dtype=np.ndarray)<EOL>

Transform X back to its original space.

        Returns an array X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data.

        Returns
        -------
        X_original : array, shape (n_samples, n_features)
            Note that this is always a dense array.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>if self.init == '<STR_LIT>':<EOL><INDENT>self._mllib_model = MLlibKMeans.train(<EOL>X.unblock(),<EOL>self.n_clusters,<EOL>maxIterations=self.max_iter,<EOL>initializationMode="<STR_LIT>")<EOL>self.cluster_centers_ = self._mllib_model.centers<EOL><DEDENT>else:<EOL><INDENT>models = X.map(lambda X: super(SparkKMeans, self).fit(X))<EOL>models = models.map(lambda model: model.cluster_centers_).collect()<EOL>return super(SparkKMeans, self).fit(np.concatenate(models))<EOL><DEDENT>

Compute k-means clustering.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD containing array-like or sparse matrix
            Train data.

        Returns
        -------
        self


check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>if isinstance(X, ArrayRDD):<EOL><INDENT>X = X.unblock()<EOL><DEDENT>return X.map(lambda x: self._mllib_model.predict(x))<EOL><DEDENT>else:<EOL><INDENT>rdd = X.map(lambda X: super(SparkKMeans, self).predict(X))<EOL>return ArrayRDD(rdd)<EOL><DEDENT>

Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : ArrayRDD containing array-like, sparse matrix
            New data to predict.

        Returns
        -------
        labels : ArrayRDD with predictions
            Index of the cluster each sample belongs to.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>"""<STR_LIT>"""<EOL>class SetAccum(AccumulatorParam):<EOL><INDENT>def zero(self, initialValue):<EOL><INDENT>return set(initialValue)<EOL><DEDENT>def addInPlace(self, v1, v2):<EOL><INDENT>v1 |= v2<EOL>return v1<EOL><DEDENT><DEDENT>accum = X.context.accumulator(set(), SetAccum())<EOL>def mapper(X, separator=self.separator):<EOL><INDENT>feature_names = []<EOL>for x in X:<EOL><INDENT>for f, v in six.iteritems(x):<EOL><INDENT>if isinstance(v, six.string_types):<EOL><INDENT>f = "<STR_LIT>" % (f, self.separator, v)<EOL><DEDENT>feature_names.append(f)<EOL><DEDENT><DEDENT>accum.add(set(feature_names))<EOL><DEDENT>X.foreach(mapper)  <EOL>feature_names = list(accum.value)<EOL>if self.sort:<EOL><INDENT>feature_names.sort()<EOL><DEDENT>vocab = dict((f, i) for i, f in enumerate(feature_names))<EOL>self.feature_names_ = feature_names<EOL>self.vocabulary_ = vocab<EOL>return self<EOL>

Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        Z : DictRDD with column 'X'
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        self


mapper = self.broadcast(super(SparkDictVectorizer, self).transform,<EOL>Z.context)<EOL>dtype = sp.spmatrix if self.sparse else np.ndarray<EOL>return Z.transform(mapper, column='<STR_LIT:X>', dtype=dtype)<EOL>

Transform ArrayRDD's (or DictRDD's 'X' column's) feature->value dicts
        to array or sparse matrix.
        Named features not encountered during fit or fit_transform will be
        silently ignored.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD with column 'X' containing Mapping or
            iterable over Mappings, length = n_samples
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        Z : transformed, containing {array, sparse matrix}
            Feature vectors; always 2-d.


return self.fit(Z).transform(Z)<EOL>

Learn a list of feature name -> indices mappings and transform Z.
        Like fit(Z) followed by transform(Z).

        Parameters
        ----------
        Z : Z : ArrayRDD or DictRDD with column 'X' containing Mapping or
            iterable over Mappings
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        Z : transformed, containing {array, sparse matrix}
            Feature vectors; always 2-d.


class SetAccum(AccumulatorParam):<EOL><INDENT>def zero(self, initialValue):<EOL><INDENT>return set(initialValue)<EOL><DEDENT>def addInPlace(self, v1, v2):<EOL><INDENT>v1 |= v2<EOL>return v1<EOL><DEDENT><DEDENT>if not self.fixed_vocabulary_:<EOL><INDENT>accum = analyzed_docs._rdd.context.accumulator(set(), SetAccum())<EOL>analyzed_docs.foreach(<EOL>lambda x: accum.add(set(chain.from_iterable(x))))<EOL>vocabulary = {t: i for i, t in enumerate(accum.value)}<EOL><DEDENT>else:<EOL><INDENT>vocabulary = self.vocabulary_<EOL><DEDENT>if not vocabulary:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return vocabulary<EOL>


vocabulary = self.vocabulary_<EOL>j_indices = _make_int_array()<EOL>indptr = _make_int_array()<EOL>indptr.append(<NUM_LIT:0>)<EOL>for doc in analyzed_docs:<EOL><INDENT>for feature in doc:<EOL><INDENT>try:<EOL><INDENT>j_indices.append(vocabulary[feature])<EOL><DEDENT>except KeyError:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>indptr.append(len(j_indices))<EOL><DEDENT>j_indices = frombuffer_empty(j_indices, dtype=np.intc)<EOL>indptr = np.frombuffer(indptr, dtype=np.intc)<EOL>values = np.ones(len(j_indices))<EOL>X = sp.csr_matrix((values, j_indices, indptr),<EOL>shape=(len(indptr) - <NUM_LIT:1>, len(vocabulary)),<EOL>dtype=self.dtype)<EOL>X.sum_duplicates()<EOL>if self.binary:<EOL><INDENT>X.data.fill(<NUM_LIT:1>)<EOL><DEDENT>return X<EOL>

Create sparse feature matrix, and vocabulary where fixed_vocab=False


sorted_features = sorted(six.iteritems(vocabulary))<EOL>map_index = np.empty(len(sorted_features), dtype=np.int32)<EOL>for new_val, (term, old_val) in enumerate(sorted_features):<EOL><INDENT>map_index[new_val] = old_val<EOL>vocabulary[term] = new_val<EOL><DEDENT>return map_index<EOL>

Sort features by name

        Returns a reordered matrix and modifies the vocabulary in place


def _limit_features(self, X, vocabulary, high=None, low=None,<EOL>limit=None):

if high is None and low is None and limit is None:<EOL><INDENT>return X, set()<EOL><DEDENT>dfs = X.map(_document_frequency).sum()<EOL>tfs = X.map(lambda x: np.asarray(x.sum(axis=<NUM_LIT:0>))).sum().ravel()<EOL>mask = np.ones(len(dfs), dtype=bool)<EOL>if high is not None:<EOL><INDENT>mask &= dfs <= high<EOL><DEDENT>if low is not None:<EOL><INDENT>mask &= dfs >= low<EOL><DEDENT>if limit is not None and mask.sum() > limit:<EOL><INDENT>mask_inds = (-tfs[mask]).argsort()[:limit]<EOL>new_mask = np.zeros(len(dfs), dtype=bool)<EOL>new_mask[np.where(mask)[<NUM_LIT:0>][mask_inds]] = True<EOL>mask = new_mask<EOL><DEDENT>new_indices = np.cumsum(mask) - <NUM_LIT:1>  <EOL>removed_terms = set()<EOL>for term, old_index in list(six.iteritems(vocabulary)):<EOL><INDENT>if mask[old_index]:<EOL><INDENT>vocabulary[term] = new_indices[old_index]<EOL><DEDENT>else:<EOL><INDENT>del vocabulary[term]<EOL>removed_terms.add(term)<EOL><DEDENT><DEDENT>kept_indices = np.where(mask)[<NUM_LIT:0>]<EOL>if len(kept_indices) == <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return kept_indices, removed_terms<EOL>

Remove too rare or too common features.

        Prune features that are non zero in more samples than high or less
        documents than low, modifying the vocabulary, and restricting it to
        at most the limit most frequent.

        This does not prune samples with zero features.


Learn a vocabulary dictionary of all tokens in the raw documents in
        the DictRDD's 'X' column.

        Parameters
        ----------
        raw_documents : iterable or DictRDD with column 'X'
            An iterable which yields either str, unicode or file objects; or a
            DictRDD with column 'X' containing such iterables.

        Returns
        -------
        self


self._validate_vocabulary()<EOL>analyze = self.build_analyzer()<EOL>A = Z.transform(lambda X: list(map(analyze, X)), column='<STR_LIT:X>').persist()<EOL>X = A[:, '<STR_LIT:X>'] if isinstance(A, DictRDD) else A<EOL>self.vocabulary_ = self._init_vocab(X)<EOL>mapper = self.broadcast(self._count_vocab, A.context)<EOL>Z = A.transform(mapper, column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL>if not self.fixed_vocabulary_:<EOL><INDENT>X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>max_df = self.max_df<EOL>min_df = self.min_df<EOL>max_features = self.max_features<EOL>n_doc = X.shape[<NUM_LIT:0>]<EOL>max_doc_count = (max_df<EOL>if isinstance(max_df, numbers.Integral)<EOL>else max_df * n_doc)<EOL>min_doc_count = (min_df<EOL>if isinstance(min_df, numbers.Integral)<EOL>else min_df * n_doc)<EOL>if max_doc_count < min_doc_count:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>")<EOL><DEDENT>kept_indices, self.stop_words_ = self._limit_features(<EOL>X, self.vocabulary_, max_doc_count, min_doc_count, max_features)<EOL>map_index = self._sort_features(self.vocabulary_)<EOL>mask = kept_indices[map_index]<EOL>Z = Z.transform(lambda x: x[:, mask], column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL><DEDENT>A.unpersist()<EOL>return Z<EOL>

Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._validate_vocabulary()<EOL><DEDENT>self._check_vocabulary()<EOL>analyze = self.build_analyzer()<EOL>mapper = self.broadcast(self._count_vocab, Z.context)<EOL>Z = Z.transform(lambda X: list(map(analyze, X)), column='<STR_LIT:X>').transform(mapper, column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL>return Z<EOL>

Transform documents to document-term matrix.

        Extract token counts out of raw text documents using the vocabulary
        fitted with fit or the one provided to the constructor.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Document-term matrix.


mapper = super(SparkHashingVectorizer, self).transform<EOL>return Z.transform(mapper, column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL>

Transform an ArrayRDD (or DictRDD with column 'X') containing
        sequence of documents to a document-term matrix.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD with raw text documents
            Samples. Each sample must be a text document (either bytes or
            unicode strings) which will be tokenized and hashed.

        Returns
        -------
        Z : SparseRDD/DictRDD containg scipy.sparse matrix
            Document-term matrix.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>def mapper(X, use_idf=self.use_idf):<EOL><INDENT>if not sp.issparse(X):<EOL><INDENT>X = sp.csc_matrix(X)<EOL><DEDENT>if use_idf:<EOL><INDENT>return _document_frequency(X)<EOL><DEDENT><DEDENT>if self.use_idf:<EOL><INDENT>n_samples, n_features = X.shape<EOL>df = X.map(mapper).treeReduce(operator.add)<EOL>df += int(self.smooth_idf)<EOL>n_samples += int(self.smooth_idf)<EOL>idf = np.log(float(n_samples) / df) + <NUM_LIT:1.0><EOL>self._idf_diag = sp.spdiags(idf,<EOL>diags=<NUM_LIT:0>, m=n_features, n=n_features)<EOL><DEDENT>return self<EOL>

Learn the idf vector (global term weights)

        Parameters
        ----------
        Z : ArrayRDD or DictRDD containing (sparse matrices|ndarray)
            a matrix of term/token counts

        Returns
        -------
        self : TfidfVectorizer


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>mapper = super(SparkTfidfTransformer, self).transform<EOL>if self.use_idf:<EOL><INDENT>check_is_fitted(self, '<STR_LIT>', '<STR_LIT>')<EOL>mapper = self.broadcast(mapper, Z.context)<EOL><DEDENT>return Z.transform(mapper, column='<STR_LIT:X>', dtype=sp.spmatrix)<EOL>

Transform an ArrayRDD (or DictRDD's 'X' column) containing count
        matrices to a tf or tf-idf representation

        Parameters
        ----------

        Z : ArrayRDD/DictRDD with sparse matrices
            a matrix of term/token counts

        Returns
        -------
        Z : SparseRDD/DictRDD containing sparse matrices


def fit_transform(self, Z, **fit_params):

<EOL>return self.fit(Z, **fit_params).transform(Z)<EOL>

Fit to data, then transform it.

        Fits transformer to Z with optional parameters fit_params
        and returns a transformed version of Z.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD
            Training set.

        Returns
        -------
        Z_new : ArrayRDD or DictRDD
            Transformed array.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>mapper = lambda X_y: super(SparkRandomForestClassifier, self).fit(<EOL>X_y[<NUM_LIT:0>], X_y[<NUM_LIT:1>]<EOL>)<EOL>models = Z.map(mapper).collect()<EOL>self.__dict__ = models[<NUM_LIT:0>].__dict__<EOL>self.estimators_ = []<EOL>for m in models:<EOL><INDENT>self.estimators_ += m.estimators_<EOL><DEDENT>self.n_estimators = len(self.estimators_)<EOL>return self<EOL>

Fit the model according to the given training data.

        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector
            y - Target labels
        classes : iterable
            The set of available classes

        Returns
        -------
        self : object
            Returns self.


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return X.map(lambda X: super(SparkRandomForestClassifier, self).predict(X))<EOL>

Distributed method to predict class labels for samples in X.

        Parameters
        ----------
        X : ArrayRDD containing {array-like, sparse matrix}
            Samples.

        Returns
        -------
        C : ArrayRDD
            Predicted class label per sample.


self.scorer_ = check_scoring(self.estimator, scoring=self.scoring)<EOL>cv = self.cv<EOL>cv = _check_cv(cv, Z)<EOL>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>if isinstance(parameter_iterable, Sized):<EOL><INDENT>n_candidates = len(parameter_iterable)<EOL>print("<STR_LIT>"<EOL>"<STR_LIT>".format(len(cv), n_candidates,<EOL>n_candidates * len(cv)))<EOL><DEDENT><DEDENT>base_estimator = clone(self.estimator)<EOL>pre_dispatch = self.pre_dispatch<EOL>out = Parallel(<EOL>n_jobs=self.n_jobs, verbose=self.verbose,<EOL>pre_dispatch=pre_dispatch, backend="<STR_LIT>"<EOL>)(<EOL>delayed(_fit_and_score)(clone(base_estimator), Z, self.scorer_,<EOL>train, test, self.verbose, parameters,<EOL>self.fit_params, return_parameters=True,<EOL>error_score=self.error_score)<EOL>for parameters in parameter_iterable<EOL>for train, test in cv)<EOL>n_fits = len(out)<EOL>n_folds = len(cv)<EOL>scores = list()<EOL>grid_scores = list()<EOL>for grid_start in range(<NUM_LIT:0>, n_fits, n_folds):<EOL><INDENT>n_test_samples = <NUM_LIT:0><EOL>score = <NUM_LIT:0><EOL>all_scores = []<EOL>for this_score, this_n_test_samples, _, parameters inout[grid_start:grid_start + n_folds]:<EOL><INDENT>all_scores.append(this_score)<EOL>if self.iid:<EOL><INDENT>this_score *= this_n_test_samples<EOL>n_test_samples += this_n_test_samples<EOL><DEDENT>score += this_score<EOL><DEDENT>if self.iid:<EOL><INDENT>score /= float(n_test_samples)<EOL><DEDENT>else:<EOL><INDENT>score /= float(n_folds)<EOL><DEDENT>scores.append((score, parameters))<EOL>grid_scores.append(_CVScoreTuple(<EOL>parameters,<EOL>score,<EOL>np.array(all_scores)))<EOL><DEDENT>self.grid_scores_ = grid_scores<EOL>best = sorted(grid_scores, key=lambda x: x.mean_validation_score,<EOL>reverse=True)[<NUM_LIT:0>]<EOL>self.best_params_ = best.parameters<EOL>self.best_score_ = best.mean_validation_score<EOL>if self.refit:<EOL><INDENT>best_estimator = clone(base_estimator).set_params(<EOL>**best.parameters)<EOL>best_estimator.fit(Z, **self.fit_params)<EOL>self.best_estimator_ = best_estimator<EOL><DEDENT>return self<EOL>

Actual fitting,  performing the search over parameters.


return SparkFeatureUnion(_name_estimators(transformers))<EOL>

Construct a FeatureUnion from the given transformers.
    This is a shorthand for the FeatureUnion constructor; it does not require,
    and does not permit, naming the transformers. Instead, they will be given
    names automatically based on their types. It also does not allow weighting.
    Examples
    --------
    >>> from sklearn.decomposition import PCA, TruncatedSVD
    >>> make_union(PCA(), TruncatedSVD())    # doctest: +NORMALIZE_WHITESPACE
    FeatureUnion(n_jobs=1,
                 transformer_list=[('pca', PCA(copy=True, n_components=None,
                                               whiten=False)),
                                   ('truncatedsvd',
                                    TruncatedSVD(algorithm='randomized',
                                                 n_components=2, n_iter=5,
                                                 random_state=None, tol=0.0))],
                 transformer_weights=None)
    Returns
    -------
    f : FeatureUnion


Zt, fit_params = self._pre_transform(Z, **fit_params)<EOL>self.steps[-<NUM_LIT:1>][-<NUM_LIT:1>].fit(Zt, **fit_params)<EOL>Zt.unpersist()<EOL>return self<EOL>

Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.

        Parameters
        ----------
        Z : ArrayRDD, TupleRDD or DictRDD
            Input data in blocked distributed format.

        Returns
        -------
        self : SparkPipeline


Zt, fit_params = self._pre_transform(Z, **fit_params)<EOL>if hasattr(self.steps[-<NUM_LIT:1>][-<NUM_LIT:1>], '<STR_LIT>'):<EOL><INDENT>return self.steps[-<NUM_LIT:1>][-<NUM_LIT:1>].fit_transform(Zt, **fit_params)<EOL><DEDENT>else:<EOL><INDENT>return self.steps[-<NUM_LIT:1>][-<NUM_LIT:1>].fit(Zt, **fit_params).transform(Zt)<EOL><DEDENT>

Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator.


Zt = Z<EOL>for name, transform in self.steps[:-<NUM_LIT:1>]:<EOL><INDENT>Zt = transform.transform(Zt)<EOL><DEDENT>return self.steps[-<NUM_LIT:1>][-<NUM_LIT:1>].score(Zt)<EOL>

Applies transforms to the data, and the score method of the
        final estimator. Valid only if the final estimator implements
        score.


fit_params_steps = dict((step, {})<EOL>for step, _ in self.transformer_list)<EOL>for pname, pval in six.iteritems(fit_params):<EOL><INDENT>step, param = pname.split('<STR_LIT>', <NUM_LIT:1>)<EOL>fit_params_steps[step][param] = pval<EOL><DEDENT>transformers = Parallel(n_jobs=self.n_jobs, backend="<STR_LIT>")(<EOL>delayed(_fit_one_transformer)(trans, Z, **fit_params_steps[name])<EOL>for name, trans in self.transformer_list)<EOL>self._update_transformer_list(transformers)<EOL>return self<EOL>

TODO: rewrite docstring
        Fit all transformers using X.
        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data, used to fit transformers.


return self.fit(Z, **fit_params).transform(Z)<EOL>

TODO: rewrite docstring
        Fit all transformers using X, transform the data and concatenate
        results.
        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data to be transformed.
        Returns
        -------
        X_t : array-like or sparse matrix, shape (n_samples, sum_n_components)
            hstack of results of transformers. sum_n_components is the
            sum of n_components (output dimension) over transformers.


if isinstance(Z, DictRDD):<EOL><INDENT>X = Z[:, '<STR_LIT:X>']<EOL><DEDENT>else:<EOL><INDENT>X = Z<EOL><DEDENT>Zs = [_transform_one(trans, name, X, self.transformer_weights)<EOL>for name, trans in self.transformer_list]<EOL>X_rdd = reduce(lambda x, y: x.zip(y._rdd), Zs)<EOL>X_rdd = X_rdd.map(flatten)<EOL>mapper = np.hstack<EOL>for item in X_rdd.first():<EOL><INDENT>if sp.issparse(item):<EOL><INDENT>mapper = sp.hstack<EOL><DEDENT><DEDENT>X_rdd = X_rdd.map(lambda x: mapper(x))<EOL>if isinstance(Z, DictRDD):<EOL><INDENT>return DictRDD([X_rdd, Z[:, '<STR_LIT:y>']],<EOL>columns=Z.columns,<EOL>dtype=Z.dtype,<EOL>bsize=Z.bsize)<EOL><DEDENT>else:<EOL><INDENT>return X_rdd<EOL><DEDENT>

TODO: rewrite docstring
        Transform X separately by each transformer, concatenate results.
        Parameters
        ----------
        X : array-like or sparse matrix, shape (n_samples, n_features)
            Input data to be transformed.
        Returns
        -------
        X_t : array-like or sparse matrix, shape (n_samples, sum_n_components)
            hstack of results of transformers. sum_n_components is the
            sum of n_components (output dimension) over transformers.


if not isinstance(rdd, BlockRDD):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>.format(BlockRDD, type(rdd)))<EOL><DEDENT>if isinstance(rdd, DictRDD):<EOL><INDENT>if not isinstance(expected_dtype, dict):<EOL><INDENT>raise TypeError('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>.format(dict, type(expected_dtype)))<EOL><DEDENT>accept = True<EOL>types = dict(list(zip(rdd.columns, rdd.dtype)))<EOL>for key, values in expected_dtype.items():<EOL><INDENT>if not isinstance(values, (tuple, list)):<EOL><INDENT>values = [values]<EOL><DEDENT>accept = accept and types[key] in values<EOL><DEDENT>return accept<EOL><DEDENT>if not isinstance(expected_dtype, (tuple, list)):<EOL><INDENT>expected_dtype = [expected_dtype]<EOL><DEDENT>return rdd.dtype in expected_dtype<EOL>

Checks if the blocks in the RDD matches the expected types.

    Parameters:
    -----------
    rdd: splearn.BlockRDD
        The RDD to check
    expected_dtype: {type, list of types, tuple of types, dict of types}
        Expected type(s). If the RDD is a DictRDD the parameter type is
        restricted to dict.

    Returns:
    --------
    accept: bool
        Returns if the types are matched.


if not check_rdd_dtype(rdd, expected_dtype):<EOL><INDENT>raise TypeError("<STR_LIT>".format(rdd))<EOL><DEDENT>

Wrapper function to check_rdd_dtype. Raises TypeError in case of dtype
    mismatch.

    Parameters:
    -----------
    rdd: splearn.BlockRDD
        The RDD to check
    expected_dtype: {type, list of types, tuple of types, dict of types}
        Expected type(s). If the RDD is a DictRDD the parameter type is
        restricted to dict.


def _block_collection(iterator, dtype, bsize=-<NUM_LIT:1>):

i = <NUM_LIT:0><EOL>accumulated = []<EOL>for a in iterator:<EOL><INDENT>if (bsize > <NUM_LIT:0>) and (i >= bsize):<EOL><INDENT>yield _pack_accumulated(accumulated, dtype)<EOL>accumulated = []<EOL>i = <NUM_LIT:0><EOL><DEDENT>accumulated.append(a)<EOL>i += <NUM_LIT:1><EOL><DEDENT>if i > <NUM_LIT:0>:<EOL><INDENT>yield _pack_accumulated(accumulated, dtype)<EOL><DEDENT>

Pack rdd with a specific collection constructor.


def _block_tuple(iterator, dtypes, bsize=-<NUM_LIT:1>):

i = <NUM_LIT:0><EOL>blocked_tuple = None<EOL>for tuple_i in iterator:<EOL><INDENT>if blocked_tuple is None:<EOL><INDENT>blocked_tuple = tuple([] for _ in range(len(tuple_i)))<EOL><DEDENT>if (bsize > <NUM_LIT:0>) and (i >= bsize):<EOL><INDENT>yield tuple(_pack_accumulated(x, dtype)<EOL>for x, dtype in zip(blocked_tuple, dtypes))<EOL>blocked_tuple = tuple([] for _ in range(len(tuple_i)))<EOL>i = <NUM_LIT:0><EOL><DEDENT>for x_j, x in zip(tuple_i, blocked_tuple):<EOL><INDENT>x.append(x_j)<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>if i > <NUM_LIT:0>:<EOL><INDENT>yield tuple(_pack_accumulated(x, dtype)<EOL>for x, dtype in zip(blocked_tuple, dtypes))<EOL><DEDENT>

Pack rdd of tuples as tuples of arrays or scipy.sparse matrices.


def block(rdd, bsize=-<NUM_LIT:1>, dtype=None):

try:<EOL><INDENT>entry = rdd.first()<EOL><DEDENT>except IndexError:<EOL><INDENT>return rdd<EOL><DEDENT>if isinstance(entry, dict):<EOL><INDENT>rdd = rdd.map(lambda x: list(x.values()))<EOL>return DictRDD(rdd, list(entry.keys()), bsize, dtype)<EOL><DEDENT>elif isinstance(entry, tuple):<EOL><INDENT>return DictRDD(rdd, bsize=bsize, dtype=dtype)<EOL><DEDENT>elif sp.issparse(entry):<EOL><INDENT>return SparseRDD(rdd, bsize)<EOL><DEDENT>elif isinstance(entry, np.ndarray):<EOL><INDENT>return ArrayRDD(rdd, bsize)<EOL><DEDENT>else:<EOL><INDENT>return BlockRDD(rdd, bsize, dtype)<EOL><DEDENT>

Block an RDD

    Parameters
    ----------

    rdd : RDD
        RDD of data points to block into either numpy arrays,
        scipy sparse matrices, or pandas data frames.
        Type of data point will be automatically inferred
        and blocked accordingly.

    bsize : int, optional, default None
        Size of each block (number of elements), if None all data points
        from each partition will be combined in a block.

    Returns
    -------

    rdd : ArrayRDD or TupleRDD or DictRDD
        The transformed rdd with added functionality


return rdd.mapPartitions(<EOL>lambda x: _block_collection(x, dtype, bsize))<EOL>

Execute the blocking process on the given rdd.

        Parameters
        ----------
        rdd : pyspark.rdd.RDD
            Distributed data to block
        bsize : int or None
            The desired size of the blocks

        Returns
        -------
        rdd : pyspark.rdd.RDD
            Blocked rdd.


return "<STR_LIT>".format(self.__class__, repr(self._rdd))<EOL>

Returns a string representation of the ArrayRDD.


def bypass(*args, **kwargs):<EOL><INDENT>result = getattr(self._rdd, attr)(*args, **kwargs)<EOL>if isinstance(result, RDD):<EOL><INDENT>if result is self._rdd:<EOL><INDENT>return self<EOL><DEDENT>else:<EOL><INDENT>return BlockRDD(result, bsize=self.bsize, dtype=self.dtype,<EOL>noblock=True)<EOL><DEDENT><DEDENT>return result<EOL><DEDENT>if not hasattr(self._rdd, attr):<EOL><INDENT>raise AttributeError("<STR_LIT>".format(<EOL>self.__class__, attr))<EOL><DEDENT>return bypass<EOL>

Access pyspark.rdd.RDD methods or attributes.

        Parameters
        ----------
        attr : method or attribute
            The method/attribute to access

        Returns
        -------
        result : method or attribute
            The result of the requested method or attribute casted to ArrayRDD
            if method/attribute is available, else raises AttributeError.


if isinstance(index, tuple):<EOL><INDENT>raise IndexError("<STR_LIT>")<EOL><DEDENT>elif isinstance(index, slice) and index == slice(None, None, None):<EOL><INDENT>return self<EOL><DEDENT>indices = np.arange(self.blocks)[index]<EOL>indexed = self._rdd.zipWithIndex()<EOL>if isinstance(index, slice):<EOL><INDENT>ascending = index.step is None or index.step > <NUM_LIT:0><EOL>rdd = indexed.filter(lambda x_i: x_i[<NUM_LIT:1>] in indices)<EOL>if not ascending:<EOL><INDENT>rdd = rdd.sortBy(lambda x_i: x_i[<NUM_LIT:1>], ascending)<EOL><DEDENT><DEDENT>elif hasattr(index, "<STR_LIT>"):<EOL><INDENT>arg = indices.tolist()<EOL>rdd = indexed.filter(lambda x_i: x_i[<NUM_LIT:1>] in indices).sortBy(lambda x_i: arg.index(x_i[<NUM_LIT:1>]))<EOL><DEDENT>elif isinstance(index, int):<EOL><INDENT>rdd = indexed.filter(lambda x_i: x_i[<NUM_LIT:1>] == indices)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>".format(type(index)))<EOL><DEDENT>rdd = rdd.map(lambda x_i: x_i[<NUM_LIT:0>])<EOL>return self.__class__(rdd, noblock=True, **self.get_params())<EOL>

Returns the selected blocks defined by index parameter.

        Parameter
        ---------
        index : int or slice
            The key of the block or the range of the blocks.

        Returns
        -------
        block : ArrayRDD
            The selected block(s).


return self._rdd.map(len).sum()<EOL>

Returns the number of elements in all blocks.


@property<EOL><INDENT>def blocks(self):  <DEDENT>

return self._rdd.count()<EOL>


@property<EOL><INDENT>def partitions(self):  <DEDENT>

return self._rdd.getNumPartitions()<EOL>

Returns the number of partitions in the rdd.


return self._rdd.flatMap(lambda x: list(x))<EOL>


return self.unblock().collect()<EOL>


rdd = self._rdd.map(lambda x: np.array(x))<EOL>return np.concatenate(rdd.collect())<EOL>

Returns the data as numpy.array from each partition.


def transform(self, fn, dtype=None, *args, **kwargs):

rdd = self._rdd.map(fn)<EOL>if dtype is None:<EOL><INDENT>return self.__class__(rdd, noblock=True, **self.get_params())<EOL><DEDENT>if dtype is np.ndarray:<EOL><INDENT>return ArrayRDD(rdd, bsize=self.bsize, noblock=True)<EOL><DEDENT>elif dtype is sp.spmatrix:<EOL><INDENT>return SparseRDD(rdd, bsize=self.bsize, noblock=True)<EOL><DEDENT>else:<EOL><INDENT>return BlockRDD(rdd, bsize=self.bsize, dtype=dtype, noblock=True)<EOL><DEDENT>

Equivalent to map, compatibility purpose only.
        Column parameter ignored.


if isinstance(key, tuple):<EOL><INDENT>index, mask = key[<NUM_LIT:0>], key[<NUM_LIT:1>:]<EOL>return super(ArrayRDD, self).__getitem__(index).map(lambda x: x[mask])<EOL><DEDENT>else:<EOL><INDENT>return super(ArrayRDD, self).__getitem__(key)<EOL><DEDENT>

Access a specified block.

        Parameters
        ----------
        key : int or slice
            The key of the block or the range of the blocks.

        Returns
        -------
        block : ArrayRDD
            The selected block(s).


<EOL>first = self.first().shape<EOL>shape = self._rdd.map(lambda x: x.shape[<NUM_LIT:0>]).sum()<EOL>return (shape,) + first[<NUM_LIT:1>:]<EOL>


return np.prod(self.shape)<EOL>


rdd = self._rdd.map(lambda x: x.toarray())<EOL>return np.concatenate(rdd.collect())<EOL>


return rdd.mapPartitions(lambda x: _block_tuple(x, dtype, bsize))<EOL>


return super(DictRDD, self).__getitem__(key)<EOL>

Returns the selected blocks defined by index parameter.
        Parameter
        ---------
        index : int or slice
            The key of the block or the range of the blocks.
        Returns
        -------
        block : DictRDD
            The selected block(s).


if isinstance(key, tuple):  <EOL><INDENT>index, key = key<EOL>return super(DictRDD, self).__getitem__(index).get(key)<EOL><DEDENT>else:<EOL><INDENT>return super(DictRDD, self).__getitem__(key)<EOL><DEDENT>

Access a specified block.

        Parameters
        ----------
        key : int or slice
            The key of the block or the range of the blocks.

        Returns
        -------
        block : ArrayRDD or TupleRDD
            The selected block(s).


return self._rdd.flatMap(lambda cols: list(zip(*cols)))<EOL>


def transform(self, fn, column=None, dtype=None):

dtypes = self.dtype<EOL>if column is None:<EOL><INDENT>indices = list(range(len(self.columns)))<EOL><DEDENT>else:<EOL><INDENT>if not type(column) in (list, tuple):<EOL><INDENT>column = [column]<EOL><DEDENT>indices = [self.columns.index(c) for c in column]<EOL><DEDENT>if dtype is not None:<EOL><INDENT>if not type(dtype) in (list, tuple):<EOL><INDENT>dtype = [dtype]<EOL><DEDENT>dtypes = [dtype[indices.index(i)] if i in indices else t<EOL>for i, t in enumerate(self.dtype)]<EOL><DEDENT>def mapper(values):<EOL><INDENT>result = fn(*[values[i] for i in indices])<EOL>if len(indices) == <NUM_LIT:1>:<EOL><INDENT>result = (result,)<EOL><DEDENT>elif not isinstance(result, (tuple, list)):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif len(result) != len(indices):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>return tuple(result[indices.index(i)] if i in indices else v<EOL>for i, v in enumerate(values))<EOL><DEDENT>return DictRDD(self._rdd.map(mapper),<EOL>columns=self.columns, dtype=dtypes,<EOL>bsize=self.bsize, noblock=True)<EOL>

Execute a transformation on a column or columns. Returns the modified
        DictRDD.

        Parameters
        ----------
        f : function
            The function to execute on the columns.
        column : {str, list or None}
            The column(s) to transform. If None is specified the method is
            equivalent to map.
        column : {str, list or None}
            The dtype of the column(s) to transform.

        Returns
        -------
        result : DictRDD
            DictRDD with transformed column(s).

        TODO: optimize


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>def mapper(X):<EOL><INDENT>"""<STR_LIT>"""<EOL>X = check_array(X, ('<STR_LIT>', '<STR_LIT>'), dtype=np.float64)<EOL>if hasattr(X, "<STR_LIT>"):   <EOL><INDENT>mean, var = mean_variance_axis(X, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>mean, var = np.mean(X, axis=<NUM_LIT:0>), np.var(X, axis=<NUM_LIT:0>)<EOL><DEDENT>return X.shape[<NUM_LIT:0>], mean, var<EOL><DEDENT>def reducer(a, b):<EOL><INDENT>"""<STR_LIT>"""<EOL>n_a, mean_a, var_a = a<EOL>n_b, mean_b, var_b = b<EOL>n_ab = n_a + n_b<EOL>mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab<EOL>var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) +((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** <NUM_LIT:2>)<EOL>return (n_ab, mean_ab, var_ab)<EOL><DEDENT>_, _, self.variances_ = X.map(mapper).treeReduce(reducer)<EOL>if np.all(self.variances_ <= self.threshold):<EOL><INDENT>msg = "<STR_LIT>"<EOL>if X.shape[<NUM_LIT:0>] == <NUM_LIT:1>:<EOL><INDENT>msg += "<STR_LIT>"<EOL><DEDENT>raise ValueError(msg.format(self.threshold))<EOL><DEDENT>return self<EOL>

Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>mapper = self.broadcast(<EOL>super(SparkVarianceThreshold, self).transform, Z.context)<EOL>return Z.transform(mapper, column='<STR_LIT:X>')<EOL>

Reduce X to the selected features.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        Returns
        -------
        X_r : array of shape [n_samples, n_selected_features]
            The input samples with only the selected features.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray)})<EOL>self._classes_ = np.unique(classes)<EOL>return self._spark_fit(SparkLinearSVC, Z)<EOL>

Fit the model according to the given training data.

        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector
            y - Target labels
        classes : iterable
            The set of available classes

        Returns
        -------
        self : object
            Returns self.


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return self._spark_predict(SparkLinearSVC, X)<EOL>

Distributed method to predict class labels for samples in X.

        Parameters
        ----------
        X : ArrayRDD containing {array-like, sparse matrix}
            Samples.

        Returns
        -------
        C : ArrayRDD
            Predicted class label per sample.


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return X.map(<EOL>lambda X: super(SparkBaseNB, self).predict(X))<EOL>

Perform classification on an RDD containing arrays of test vectors X.

Parameters
----------
X : RDD containing array-like items, shape = [m_samples, n_features]

Returns
-------
C : RDD with arrays, shape = [n_samples]
    Predicted target values for X


check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return X.map(<EOL>lambda X: super(SparkBaseNB, self).predict_proba(X))<EOL>

Return probability estimates for the RDD containing test vector X.

Parameters
----------
X : RDD containing array-like items, shape = [m_samples, n_features]

Returns
-------
C : RDD with array-like items , shape = [n_samples, n_classes]
    Returns the probability of the samples for each class in
    the models for each RDD block. The columns correspond to the classes
    in sorted order, as they appear in the attribute `classes_`.


<EOL>if not isinstance(X, BlockRDD):<EOL><INDENT>return super(SparkBaseNB, self).predict_log_proba(X)<EOL><DEDENT>check_rdd(X, (sp.spmatrix, np.ndarray))<EOL>return X.map(<EOL>lambda X: super(SparkBaseNB, self).predict_log_proba(X))<EOL>

Return log-probability estimates for the RDD containing the
test vector X.

Parameters
----------
X : RDD containing array-like items, shape = [m_samples, n_features]

Returns
-------
C : RDD with array-like items, shape = [n_samples, n_classes]
    Returns the log-probability of the samples for each class in
    the model for each RDD block. The columns correspond to the classes
    in sorted order, as they appear in the attribute `classes_`.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray), '<STR_LIT:y>': (sp.spmatrix, np.ndarray)})<EOL>models = Z[:, ['<STR_LIT:X>', '<STR_LIT:y>']].map(<EOL>lambda X_y: self.partial_fit(X_y[<NUM_LIT:0>], X_y[<NUM_LIT:1>], classes))<EOL>avg = models.reduce(operator.add)<EOL>self.__dict__.update(avg.__dict__)<EOL>return self<EOL>

Fit Gaussian Naive Bayes according to X, y

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        Returns
        -------
        self : object
            Returns self.


<EOL>if other == <NUM_LIT:0>:<EOL><INDENT>return self<EOL><DEDENT>model = copy.deepcopy(self)<EOL>model.class_count_ += other.class_count_<EOL>model.feature_count_ += other.feature_count_<EOL>model._update_class_log_prior()<EOL>model._update_feature_log_prob()<EOL>return model<EOL>

Add method for DiscreteNB models.

Parameters
----------
other : fitted splearn multinomilal NB model with class_count_
        and feature_count_ attribute
    Model to add.

Returns
-------
model : splearn Naive Bayes model
    Model with updated coefficients.


check_rdd(Z, {'<STR_LIT:X>': (sp.spmatrix, np.ndarray), '<STR_LIT:y>': (sp.spmatrix, np.ndarray)})<EOL>if '<STR_LIT:w>' in Z.columns:<EOL><INDENT>models = Z[:, ['<STR_LIT:X>', '<STR_LIT:y>', '<STR_LIT:w>']].map(<EOL>lambda X_y_w: self.partial_fit(<EOL>X_y_w[<NUM_LIT:0>], X_y_w[<NUM_LIT:1>], classes, X_y_w[<NUM_LIT:2>]<EOL>)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>models = Z[:, ['<STR_LIT:X>', '<STR_LIT:y>']].map(<EOL>lambda X_y: self.partial_fit(X_y[<NUM_LIT:0>], X_y[<NUM_LIT:1>], classes))<EOL><DEDENT>avg = models.sum()<EOL>self.__dict__.update(avg.__dict__)<EOL>return self<EOL>

TODO fulibacsi fix docstring
Fit Multinomial Naive Bayes according to (X,y) pair
which is zipped into TupleRDD Z.

Parameters
----------
Z : TupleRDD containing X [array-like, shape (m_samples, n_features)]
    and y [array-like, shape (m_samples,)] tuple
    Training vectors, where ,_samples is the number of samples in the
    block and n_features is the number of features, and y contains
    the target values.

Returns
-------
self : object
    Returns self.


def mapper(y):<EOL><INDENT>y = column_or_1d(y, warn=True)<EOL>_check_numpy_unicode_bug(y)<EOL>return np.unique(y)<EOL><DEDENT>def reducer(a, b):<EOL><INDENT>return np.unique(np.concatenate((a, b)))<EOL><DEDENT>self.classes_ = y.map(mapper).reduce(reducer)<EOL>return self<EOL>

Fit label encoder
        Parameters
        ----------
        y : ArrayRDD (n_samples,)
            Target values.
        Returns
        -------
        self : returns an instance of self.


return self.fit(y).transform(y)<EOL>

Fit label encoder and return encoded labels
        Parameters
        ----------
        y : ArrayRDD [n_samples]
            Target values.
        Returns
        -------
        y : ArrayRDD [n_samples]


mapper = super(SparkLabelEncoder, self).transform<EOL>mapper = self.broadcast(mapper, y.context)<EOL>return y.transform(mapper)<EOL>

Transform labels to normalized encoding.
        Parameters
        ----------
        y : ArrayRDD [n_samples]
            Target values.
        Returns
        -------
        y : ArrayRDD [n_samples]


mapper = super(SparkLabelEncoder, self).inverse_transform<EOL>mapper = self.broadcast(mapper, y.context)<EOL>return y.transform(mapper)<EOL>

Transform labels back to original encoding.
        Parameters
        ----------
        y : numpy array of shape [n_samples]
            Target values.
        Returns
        -------
        y : ArrayRDD [n_samples]


<EOL>self._reset()<EOL>X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>def mapper(X):<EOL><INDENT>"""<STR_LIT>"""<EOL>X = check_array(X, ('<STR_LIT>', '<STR_LIT>'), dtype=np.float64)<EOL>if hasattr(X, "<STR_LIT>"):   <EOL><INDENT>mean, var = mean_variance_axis(X, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>mean, var = np.mean(X, axis=<NUM_LIT:0>), np.var(X, axis=<NUM_LIT:0>)<EOL><DEDENT>return X.shape[<NUM_LIT:0>], mean, var<EOL><DEDENT>def reducer(a, b):<EOL><INDENT>"""<STR_LIT>"""<EOL>n_a, mean_a, var_a = a<EOL>n_b, mean_b, var_b = b<EOL>n_ab = n_a + n_b<EOL>mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab<EOL>var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) +((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** <NUM_LIT:2>)<EOL>return (n_ab, mean_ab, var_ab)<EOL><DEDENT>if check_rdd_dtype(X, (sp.spmatrix)):<EOL><INDENT>if self.with_mean:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>self.n_samples_seen_, self.mean_, self.var_ = X.map(mapper).treeReduce(reducer)<EOL>if self.with_std:<EOL><INDENT>self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))<EOL><DEDENT>else:<EOL><INDENT>self.scale_ = None<EOL><DEDENT>return self<EOL>

Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.


X = Z[:, '<STR_LIT:X>'] if isinstance(Z, DictRDD) else Z<EOL>check_rdd(X, (np.ndarray, sp.spmatrix))<EOL>if check_rdd_dtype(X, (sp.spmatrix)):<EOL><INDENT>if self.with_mean:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if self.scale_ is not None:<EOL><INDENT>def mapper(X):<EOL><INDENT>inplace_column_scale(X, <NUM_LIT:1> / self.scale_)<EOL>return X<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self.with_mean:<EOL><INDENT>if self.with_std:<EOL><INDENT>def mapper(X):<EOL><INDENT>X -= self.mean_<EOL>X /= self.scale_<EOL>return X<EOL><DEDENT><DEDENT>else:<EOL><INDENT>def mapper(X):<EOL><INDENT>X -= self.mean_<EOL>return X<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self.with_std:<EOL><INDENT>def mapper(X):<EOL><INDENT>X /= self.scale_<EOL>return X<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>return Z.transform(mapper, column="<STR_LIT:X>")<EOL>

Perform standardization by centering and scaling
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector
            y - Target labels
        Returns
        -------
        C : DictRDD containing (X, y) pairs
            X - Training vector standardized
            y - Target labels


scaler = StandardScaler(with_mean=self.with_mean,<EOL>with_std=self.with_std,<EOL>copy=self.copy)<EOL>scaler.__dict__ = self.__dict__<EOL>return scaler<EOL>


def auth_decorator(func):<EOL><INDENT>def inner(self, *args, **kw):<EOL><INDENT>if self.get_authenticated_user(auth_func, realm):<EOL><INDENT>return func(self, *args, **kw)<EOL><DEDENT><DEDENT>return inner<EOL><DEDENT>return auth_decorator<EOL>

Decorator that protect methods with HTTP authentication.


lang = self.highlightlang<EOL>linenos = node.rawsource.count('<STR_LIT:\n>') >= self.highlightlinenothreshold - <NUM_LIT:1><EOL>fname = node['<STR_LIT:filename>']<EOL>highlight_args = node.get('<STR_LIT>', {})<EOL>if '<STR_LIT>' in node:<EOL><INDENT>lang = node['<STR_LIT>']<EOL>highlight_args['<STR_LIT>'] = True<EOL><DEDENT>if '<STR_LIT>' in node:<EOL><INDENT>linenos = node['<STR_LIT>']<EOL><DEDENT>def warner(msg):<EOL><INDENT>self.builder.warn(msg, (self.builder.current_docname, node.line))<EOL><DEDENT>highlighted = self.highlighter.highlight_block(node.rawsource, lang,<EOL>warn=warner,<EOL>linenos=linenos,<EOL>**highlight_args)<EOL>starttag = self.starttag(node, '<STR_LIT>', suffix='<STR_LIT>',<EOL>CLASS='<STR_LIT>' % lang)<EOL>self.body.append(starttag)<EOL>self.body.append('<STR_LIT>''<STR_LIT>' % (fname,))<EOL>self.body.append(highlighted)<EOL>self.body.append('<STR_LIT>')<EOL>raise nodes.SkipNode<EOL>


code = node.rawsource.rstrip('<STR_LIT:\n>')<EOL>lang = self.hlsettingstack[-<NUM_LIT:1>][<NUM_LIT:0>]<EOL>linenos = code.count('<STR_LIT:\n>') >= self.hlsettingstack[-<NUM_LIT:1>][<NUM_LIT:1>] - <NUM_LIT:1><EOL>fname = node['<STR_LIT:filename>']<EOL>highlight_args = node.get('<STR_LIT>', {})<EOL>if '<STR_LIT>' in node:<EOL><INDENT>lang = node['<STR_LIT>']<EOL>highlight_args['<STR_LIT>'] = True<EOL><DEDENT>if '<STR_LIT>' in node:<EOL><INDENT>linenos = node['<STR_LIT>']<EOL><DEDENT>def warner(msg):<EOL><INDENT>self.builder.warn(msg, (self.curfilestack[-<NUM_LIT:1>], node.line))<EOL><DEDENT>hlcode = self.highlighter.highlight_block(code, lang, warn=warner,<EOL>linenos=linenos,<EOL>**highlight_args)<EOL>self.body.append(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (<EOL>fname.replace('<STR_LIT:_>', r'<STR_LIT>'),<EOL>)<EOL>)<EOL>if self.table:<EOL><INDENT>hlcode = hlcode.replace('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>self.table.has_problematic = True<EOL>self.table.has_verbatim = True<EOL><DEDENT>hlcode = hlcode.rstrip()[:-<NUM_LIT>]  <EOL>hlcode = hlcode.rstrip() + '<STR_LIT:\n>'<EOL>self.body.append('<STR_LIT:\n>' + hlcode + '<STR_LIT>' %<EOL>(self.table and '<STR_LIT>' or '<STR_LIT>'))<EOL>raise nodes.SkipNode<EOL>


pass<EOL>

Latex document generator depart handler.


mems = {x[<NUM_LIT:0>]: x[<NUM_LIT:1>] for x in inspect.getmembers(thing_obj)}<EOL>if member_string in mems:<EOL><INDENT>return mems[member_string]<EOL><DEDENT>

Get a member from an object by (string) name


components = callable_str.split('<STR_LIT:.>')<EOL>func = None<EOL>if len(components) < <NUM_LIT:2>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>elif len(components) == <NUM_LIT:2>:<EOL><INDENT>mod_name = components[<NUM_LIT:0>]<EOL>func_name = components[<NUM_LIT:1>]<EOL>try:<EOL><INDENT>mod = import_module(mod_name)<EOL><DEDENT>except ModuleNotFoundError:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>func = get_member(mod, func_name)<EOL>if func is None:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>mod_name = '<STR_LIT:.>'.join(components[:-<NUM_LIT:1>])<EOL>func_name = components[-<NUM_LIT:1>]<EOL>try:<EOL><INDENT>mod = import_module(mod_name)<EOL><DEDENT>except ModuleNotFoundError:<EOL><INDENT>mod_name = '<STR_LIT:.>'.join(components[:-<NUM_LIT:2>])<EOL>class_name = components[-<NUM_LIT:2>]<EOL>try:<EOL><INDENT>mod = import_module(mod_name)<EOL><DEDENT>except ModuleNotFoundError:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>klass = get_member(mod, class_name)<EOL>if klass is None:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>func = get_member(klass, func_name)<EOL>if func is None:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT><DEDENT>if func is None:<EOL><INDENT>func = get_member(mod, func_name)<EOL>if func is None:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>if inspect.ismodule(func):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if inspect.isclass(func):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>sig = [x for x in inspect.signature(func).parameters]<EOL><DEDENT>except TypeError:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>if len(sig) == <NUM_LIT:1>:<EOL><INDENT>if sig[<NUM_LIT:0>] == '<STR_LIT:message>':<EOL><INDENT>return func<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>elif len(sig)==<NUM_LIT:2> and sig[<NUM_LIT:0>]=='<STR_LIT>' and sig[<NUM_LIT:1>]=='<STR_LIT:message>':<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Return a live function from a full dotted path.  Must be either a plain function
    directly in a module, a class function, or a static function.  (No modules, classes,
    or instance methods, since those can't be called as tasks.)


def task_with_callable(the_callable, label=None, schedule=DEFAULT_SCHEDULE, userdata=None, pk_override=None):

task = Task()<EOL>if isinstance(the_callable, str):<EOL><INDENT>if pk_override is not None:<EOL><INDENT>components = the_callable.split('<STR_LIT:.>')<EOL>info = dict(<EOL>func_type='<STR_LIT>',<EOL>module_name='<STR_LIT:.>'.join(components[:-<NUM_LIT:2>]),<EOL>class_name=components[-<NUM_LIT:2>],<EOL>class_path='<STR_LIT:.>'.join(components[:-<NUM_LIT:1>]),<EOL>model_pk=pk_override,<EOL>func_name=components[-<NUM_LIT:1>],<EOL>func_path=the_callable,<EOL>)<EOL>task.funcinfo = info<EOL><DEDENT>else:<EOL><INDENT>task.funcinfo = get_func_info(func_from_string(the_callable))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>task.funcinfo = get_func_info(the_callable)<EOL><DEDENT>if label is None:<EOL><INDENT>task.label = task.funcinfo['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>task.label = label<EOL><DEDENT>task.schedule = schedule<EOL>if not croniter.is_valid(task.schedule):<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>if userdata is None:<EOL><INDENT>task.userdata = dict()<EOL><DEDENT>else:<EOL><INDENT>if isinstance(userdata, dict):<EOL><INDENT>task.userdata = userdata<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>return task<EOL>

Factory function to create a properly initialized task.


task = Task.objects.get(label=label)<EOL>info = json.loads(task._func_info)<EOL>return info<EOL>

Return task info dictionary from task label.  Internal function,
    pretty much only used in migrations since the model methods aren't there.


info = self.funcinfo<EOL>functype = info['<STR_LIT>']<EOL>if functype in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>the_modelclass = get_module_member_by_dottedpath(info['<STR_LIT>'])<EOL>if functype == '<STR_LIT>':<EOL><INDENT>the_modelobject = the_modelclass.objects.get(pk=info['<STR_LIT>'])<EOL>the_callable = get_member(the_modelobject, info['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>the_callable = get_member(the_modelclass, info['<STR_LIT>'])<EOL><DEDENT>return the_callable<EOL><DEDENT>elif functype == '<STR_LIT>':<EOL><INDENT>mod = import_module(info['<STR_LIT>'])<EOL>the_callable = get_member(mod, info['<STR_LIT>'])<EOL>return the_callable<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>

Find and return a callable object from a task info dictionary


now = timezone.now()<EOL>tasks = cls.objects.filter(enabled=True)<EOL>for task in tasks:<EOL><INDENT>if task.next_run == HAS_NOT_RUN:<EOL><INDENT>task.calc_next_run()<EOL><DEDENT>if task.next_run < now:<EOL><INDENT>if (task.start_running < now):<EOL><INDENT>if (task.end_running > now):<EOL><INDENT>task.run_asap()<EOL><DEDENT>else:<EOL><INDENT>task.enabled = False<EOL>task.save()<EOL>Channel(KILL_TASK_CHANNEL).send({'<STR_LIT:id>': task.pk})<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Internal task-runner class method, called by :py:func:`sisy.consumers.run_heartbeat`