@path_required<EOL><INDENT>def dump_file(self, value, relativePath,<EOL>description=None,<EOL>dump=None, pull=None,<EOL>replace=False, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

<EOL>assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(replace, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>if description is None:<EOL><INDENT>description = '<STR_LIT>'<EOL><DEDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL>if pull is None and dump is not None:<EOL><INDENT>if dump.startswith('<STR_LIT>') or dump.startswith('<STR_LIT>') or dump.startswith('<STR_LIT>') or dump =='<STR_LIT>':<EOL><INDENT>pull = dump<EOL><DEDENT><DEDENT>dump = get_dump_method(dump, protocol=self._DEFAULT_PICKLE_PROTOCOL)<EOL>pull = get_pull_method(pull)<EOL>relativePath = self.to_repo_relative_path(path=relativePath, split=False)<EOL>savePath     = os.path.join(self.__path,relativePath)<EOL>fPath, fName = os.path.split(savePath)<EOL>success, reason = self.is_name_allowed(savePath)<EOL>if not success:<EOL><INDENT>assert not raiseError, reason<EOL>return False, reason<EOL><DEDENT>try:<EOL><INDENT>success, reason = self.add_directory(fPath, raiseError=False, ntrials=ntrials)<EOL><DEDENT>except Exception as err:<EOL><INDENT>reason  = "<STR_LIT>"%(str(err))<EOL>success = False<EOL><DEDENT>if not success:<EOL><INDENT>assert not raiseError, reason<EOL>return False, reason<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>if not acquired:<EOL><INDENT>m = "<STR_LIT>"%(code,)<EOL>assert raiseError, Exception(m)<EOL>return False,m<EOL><DEDENT>LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))<EOL>acquired, code = LF.acquire_lock()<EOL>if not acquired:<EOL><INDENT>LR.release_lock()<EOL>error = "<STR_LIT>"%(code,relativePath)<EOL>assert not raiseError, error<EOL>return False, error<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT>except Exception as err:<EOL><INDENT>error = str(err)<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is not None:<EOL><INDENT>LR.release_lock()<EOL>LF.release_lock()<EOL>assert not raiseError, Exception(error)<EOL>return False, error<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>isRepoFile, fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)<EOL>if isRepoFile:<EOL><INDENT>assert replace, "<STR_LIT>"<EOL><DEDENT>fileInfoPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%fName)<EOL>if isRepoFile and fileOnDisk:<EOL><INDENT>with open(fileInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>info = pickle.load(fd)<EOL><DEDENT>assert info['<STR_LIT>'] == self.__repo['<STR_LIT>'], "<STR_LIT>"<EOL>info['<STR_LIT>'] = time.time()<EOL><DEDENT>else:<EOL><INDENT>info = {'<STR_LIT>':self.__repo['<STR_LIT>']}<EOL>info['<STR_LIT>'] = info['<STR_LIT>'] = time.time()<EOL><DEDENT>info['<STR_LIT>'] = dump<EOL>info['<STR_LIT>'] = pull<EOL>info['<STR_LIT:description>'] = description<EOL>if not isRepoFile:<EOL><INDENT>dirList = self.__get_repository_directory(fPath)<EOL><DEDENT>my_exec( dump.replace("<STR_LIT>", str(savePath)), locals=locals(), globals=globals(), description='<STR_LIT>'  )<EOL>with open(fileInfoPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>pickle.dump( info,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL)<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT>fileClassPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%fName)<EOL>with open(fileClassPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>if value is None:<EOL><INDENT>klass = None<EOL><DEDENT>else:<EOL><INDENT>klass = value.__class__<EOL><DEDENT>pickle.dump(klass , fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT>if not isRepoFile:<EOL><INDENT>dirList.append(fName)<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%(str(err),)<EOL>try:<EOL><INDENT>if '<STR_LIT>' in dump:<EOL><INDENT>mi = get_pickling_errors(value)<EOL>if mi is not None:<EOL><INDENT>error += '<STR_LIT>'%str(mi)<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>error = None<EOL>break<EOL><DEDENT><DEDENT>if error is None:<EOL><INDENT>_, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)<EOL><DEDENT>LR.release_lock()<EOL>LF.release_lock()<EOL>assert not raiseError or error is None, "<STR_LIT>"%(relativePath, ntrials, error,)<EOL>return success, error<EOL>

Dump a file using its value to the system and creates its
attribute in the Repository with utc timestamp.

:Parameters:
    #. value (object): The value of a file to dump and add to the
       repository. It is any python object or file.
    #. relativePath (str): The relative to the repository path to where
       to dump the file.
    #. description (None, string): Any description about the file.
    #. dump (None, string): The dumping method.
       If None it will be set automatically to pickle and therefore the
       object must be pickleable. If a string is given, it can be a
       keyword ('json','pickle','dill') or a string compileable code to
       dump the data. The string code must include all the necessary
       imports and a '$FILE_PATH' that replaces the absolute file path
       when the dumping will be performed.\n
       e.g. "import numpy as np; np.savetxt(fname='$FILE_PATH', X=value, fmt='%.6e')"
    #. pull (None, string): The pulling method. If None it will be set
       automatically to pickle and therefore the object must be
       pickleable. If a string is given, it can be a keyword
       ('json','pickle','dill') or a string compileable code to pull
       the data. The string code must include all the necessary imports,
       a '$FILE_PATH' that replaces the absolute file path when the
       dumping will be performed and finally a PULLED_DATA variable.\n
       e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
    #. replace (boolean): Whether to replace any existing file.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (boolean): Whether renaming the directory was successful.
    #. message (None, string): Some explanatory message or error reason
       why directory was not dumped.


@path_required<EOL><INDENT>def save(self, description=None, raiseError=True, ntrials=<NUM_LIT:3>):<DEDENT>

assert isinstance(raiseError, bool), "<STR_LIT>"<EOL>assert isinstance(ntrials, int), "<STR_LIT>"<EOL>assert ntrials><NUM_LIT:0>, "<STR_LIT>"<EOL>if description is not None:<EOL><INDENT>assert isinstance(description, basestring), "<STR_LIT>"<EOL><DEDENT>dirInfoPath = os.path.join(self.__path, self.__dirInfo)<EOL>if description is None and not os.path.isfile(dirInfoPath):<EOL><INDENT>description = '<STR_LIT>'<EOL><DEDENT>LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))<EOL>acquired, code = LR.acquire_lock()<EOL>m = "<STR_LIT>"%(code,)<EOL>if not acquired:<EOL><INDENT>assert not raiseError, Exception(m)<EOL>return False, m<EOL><DEDENT>for _trial in range(ntrials):<EOL><INDENT>try:<EOL><INDENT>repoInfoPath = os.path.join(self.__path, self.__repoFile)<EOL>error = None<EOL>self.__save_dirinfo(description=description, dirInfoPath=dirInfoPath)<EOL>if os.path.isfile(repoInfoPath):<EOL><INDENT>with open(repoInfoPath, '<STR_LIT:rb>') as fd:<EOL><INDENT>repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))<EOL>self.__repo['<STR_LIT>'] = repo['<STR_LIT>']<EOL><DEDENT><DEDENT>with open(repoInfoPath, '<STR_LIT:wb>') as fd:<EOL><INDENT>self.__repo["<STR_LIT>"] = time.time()<EOL>pickle.dump( self.__repo,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )<EOL>fd.flush()<EOL>os.fsync(fd.fileno())<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>error = "<STR_LIT>"%err<EOL>if self.DEBUG_PRINT_FAILED_TRIALS: print("<STR_LIT>"%(_trial, inspect.stack()[<NUM_LIT:1>][<NUM_LIT:3>], str(error)))<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>LR.release_lock()<EOL>assert error is None or not raiseError, error<EOL>return error is None, error<EOL>

Save repository '.pyreprepo' to disk and create (if missing) or
update (if description is not None) '.pyrepdirinfo'.

:Parameters:
    #. description (None, str): Repository main directory information.
       If given will be replaced.
    #. raiseError (boolean): Whether to raise encountered error instead
       of returning failure.
    #. ntrials (int): After aquiring all locks, ntrials is the maximum
       number of trials allowed before failing.
       In rare cases, when multiple processes
       are accessing the same repository components, different processes
       can alter repository components between successive lock releases
       of some other process. Bigger number of trials lowers the
       likelyhood of failure due to multiple processes same time
       alteration.

:Returns:
    #. success (bool): Whether saving was successful.
    #. error (None, string): Fail to save repository message in case
       saving is not successful. If success is True, error will be None.


def get_file_relative_path_by_name(self, name, skip=<NUM_LIT:0>):

if skip is None:<EOL><INDENT>paths = []<EOL><DEDENT>else:<EOL><INDENT>paths = None<EOL><DEDENT>for path, info in self.walk_files_info():<EOL><INDENT>_, n = os.path.split(path)<EOL>if n==name:<EOL><INDENT>if skip is None:<EOL><INDENT>paths.append(path)<EOL><DEDENT>elif skip><NUM_LIT:0>:<EOL><INDENT>skip -= <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>paths = path<EOL>break<EOL><DEDENT><DEDENT><DEDENT>return paths<EOL>

Get file relative path given the file name. If file name is redundant in different
directories in the repository, this method ensures to return all or some of the
files according to skip value.

Parameters:
    #. name (string): The file name.
    #. skip (None, integer): As file names can be identical, skip determines
       the number of satisfying files name to skip before returning.\n
       If None is given, a list of all files relative path will be returned.

:Returns:
    #. relativePath (string, list): The file relative path.
       If None, it means file was not found.\n
       If skip is None a list of all found files relative paths will be returned.


@acquire_lock<EOL><INDENT>@sync_required<EOL>def add_directory(self, relativePath, info=None):<DEDENT>

path = os.path.normpath(relativePath)<EOL>currentDir  = self.path<EOL>currentDict = self<EOL>if path in ("<STR_LIT>","<STR_LIT:.>"):<EOL><INDENT>return currentDict<EOL><DEDENT>save = False<EOL>for dir in path.split(os.sep):<EOL><INDENT>dirPath = os.path.join(currentDir, dir)<EOL>if not os.path.exists(dirPath):<EOL><INDENT>os.mkdir(dirPath)<EOL><DEDENT>currentDict = dict.__getitem__(currentDict, "<STR_LIT>")<EOL>if currentDict.get(dir, None) is None:<EOL><INDENT>save = True<EOL>currentDict[dir] = {"<STR_LIT>":{}, "<STR_LIT>":{},<EOL>"<STR_LIT>":datetime.utcnow(),<EOL>"<STR_LIT:id>":str(uuid.uuid1()),<EOL>"<STR_LIT:info>": info} <EOL><DEDENT>currentDict = currentDict[dir]<EOL>currentDir  = dirPath<EOL><DEDENT>if save:<EOL><INDENT>self.save()<EOL><DEDENT>return currentDict<EOL>

Adds a directory in the repository and creates its
attribute in the Repository with utc timestamp.
It insures adding all the missing directories in the path.

:Parameters:
    #. relativePath (string): The relative to the repository path of the directory to add in the repository.
    #. info (None, string, pickable object): Any random info about the folder.

:Returns:
    #. info (dict): The directory info dict.


@acquire_lock<EOL><INDENT>@sync_required<EOL>def move_directory(self, relativePath, relativeDestination, replace=False, verbose=True):<DEDENT>

<EOL>relativePath    = os.path.normpath(relativePath)<EOL>relativeDestination = os.path.normpath(relativeDestination)<EOL>filesInfo = list( self.walk_files_info(relativePath=relativePath) )<EOL>dirsPath  = list( self.walk_directories_relative_path(relativePath=relativePath) )<EOL>dirInfoDict, errorMessage = self.get_directory_info(relativePath)<EOL>assert dirInfoDict is not None, errorMessage<EOL>self.remove_directory(relativePath=relativePath, removeFromSystem=False)<EOL>self.add_directory(relativeDestination)<EOL>for RP, info in filesInfo:<EOL><INDENT>source      = os.path.join(self.__path, relativePath, RP)<EOL>destination = os.path.join(self.__path, relativeDestination, RP)<EOL>newDirRP, fileName = os.path.split(os.path.join(relativeDestination, RP))<EOL>dirInfoDict = self.add_directory( newDirRP )<EOL>if os.path.isfile(destination):<EOL><INDENT>if replace:<EOL><INDENT>os.remove(destination)<EOL>if verbose:<EOL><INDENT>warnings.warn("<STR_LIT>"%(fileName, newDirRP))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>warnings.warn("<STR_LIT>"%(fileName,destination))<EOL><DEDENT>continue<EOL><DEDENT><DEDENT>os.rename(source, destination)<EOL>dict.__getitem__(dirInfoDict, "<STR_LIT>")[fileName] = info<EOL><DEDENT>self.save()<EOL>

Move a directory in the repository from one place to another. It insures moving all the
files and subdirectories in the system.

:Parameters:
    #. relativePath (string): The relative to the repository path of the directory to be moved.
    #. relativeDestination (string): The new relative to the repository path of the directory.
    #. replace (boolean): Whether to replace existing files with the same name in the new created directory.
    #. verbose (boolean): Whether to be warn and informed about any abnormalities.


if self.__path is None:<EOL><INDENT>return<EOL><DEDENT>for dirPath in sorted(list(self.walk_directories_relative_path())):<EOL><INDENT>realPath = os.path.join(self.__path, dirPath)<EOL>if os.path.isdir(realPath):<EOL><INDENT>continue<EOL><DEDENT>if verbose: warnings.warn("<STR_LIT>"%realPath)<EOL>keys = dirPath.split(os.sep)<EOL>dirInfoDict = self<EOL>for idx in range(len(keys)-<NUM_LIT:1>):<EOL><INDENT>dirs = dict.get(dirInfoDict, '<STR_LIT>', None)<EOL>if dirs is None: break<EOL>dirInfoDict = dict.get(dirs, keys[idx], None)<EOL>if dirInfoDict is None: break<EOL><DEDENT>if dirInfoDict is not None:<EOL><INDENT>dirs = dict.get(dirInfoDict, '<STR_LIT>', None)<EOL>if dirs is not None:<EOL><INDENT>dict.pop( dirs, keys[-<NUM_LIT:1>], None )<EOL><DEDENT><DEDENT><DEDENT>for filePath in sorted(list(self.walk_files_relative_path())):<EOL><INDENT>realPath = os.path.join(self.__path, filePath)<EOL>if os.path.isfile( realPath ):<EOL><INDENT>continue<EOL><DEDENT>if verbose: warnings.warn("<STR_LIT>"%realPath)<EOL>keys = filePath.split(os.sep)<EOL>dirInfoDict = self<EOL>for idx in range(len(keys)-<NUM_LIT:1>):<EOL><INDENT>dirs = dict.get(dirInfoDict, '<STR_LIT>', None)<EOL>if dirs is None: break<EOL>dirInfoDict = dict.get(dirs, keys[idx], None)<EOL>if dirInfoDict is None: break<EOL><DEDENT>if dirInfoDict is not None:<EOL><INDENT>files = dict.get(dirInfoDict, '<STR_LIT>', None)<EOL>if files is not None:<EOL><INDENT>dict.pop( files, keys[-<NUM_LIT:1>], None )<EOL><DEDENT><DEDENT><DEDENT>

Synchronizes the Repository information with the directory.
All registered but missing files and directories in the directory,
will be automatically removed from the Repository.

:parameters:
    #. verbose (boolean): Whether to be warn and inform about any abnormalities.


@path_required<EOL><INDENT>def create_package(self, path=None, name=None, mode=None):<DEDENT>

<EOL>assert mode in (None, '<STR_LIT:w>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'), '<STR_LIT>'%str(mode)<EOL>if mode is None:<EOL><INDENT>mode = '<STR_LIT>'<EOL>mode = '<STR_LIT>'<EOL><DEDENT>if path is None:<EOL><INDENT>root = os.path.split(self.__path)[<NUM_LIT:0>]<EOL><DEDENT>elif path.strip() in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>root = os.getcwd()<EOL><DEDENT>else:<EOL><INDENT>root = os.path.realpath( os.path.expanduser(path) )<EOL><DEDENT>assert os.path.isdir(root), '<STR_LIT>'%path<EOL>if name is None:<EOL><INDENT>ext = mode.split("<STR_LIT::>")<EOL>if len(ext) == <NUM_LIT:2>:<EOL><INDENT>if len(ext[<NUM_LIT:1>]):<EOL><INDENT>ext = "<STR_LIT:.>"+ext[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT>name = os.path.split(self.__path)[<NUM_LIT:1>]+ext<EOL><DEDENT>self.save()<EOL>tarfilePath = os.path.join(root, name)<EOL>try:<EOL><INDENT>tarHandler = tarfile.TarFile.open(tarfilePath, mode=mode)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise Exception("<STR_LIT>"%e)<EOL><DEDENT>for directory in sorted(list(self.walk_directories_relative_path())):<EOL><INDENT>t = tarfile.TarInfo( directory )<EOL>t.type = tarfile.DIRTYPE<EOL>tarHandler.addfile(t)<EOL><DEDENT>for file in self.walk_files_relative_path():<EOL><INDENT>tarHandler.add(os.path.join(self.__path,file), arcname=file)<EOL><DEDENT>tarHandler.add(os.path.join(self.__path,"<STR_LIT>"), arcname="<STR_LIT>")<EOL>tarHandler.close()<EOL>

Create a tar file package of all the repository files and directories.
Only files and directories that are stored in the repository info
are stored in the package tar file.

**N.B. On some systems packaging requires root permissions.**

:Parameters:
    #. path (None, string): The real absolute path where to create the package.
       If None, it will be created in the same directory as the repository
       If '.' or an empty string is passed, the current working directory will be used.
    #. name (None, string): The name to give to the package file
       If None, the package directory name will be used with the appropriate extension added.
    #. mode (None, string): The writing mode of the tarfile.
       If None, automatically the best compression mode will be chose.
       Available modes are ('w', 'w:', 'w:gz', 'w:bz2')


def get_repository(self, path, info=None, verbose=True):

<EOL>if path.strip() in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>realPath = os.path.realpath( os.path.expanduser(path) )<EOL>if not os.path.isdir(realPath):<EOL><INDENT>os.makedirs(realPath)<EOL><DEDENT>if not self.is_repository(realPath):<EOL><INDENT>self.create_repository(realPath, info=info, verbose=verbose)<EOL><DEDENT>else:<EOL><INDENT>self.load_repository(realPath)<EOL><DEDENT>

Create a repository at given real path or load any existing one.
This method insures the creation of the directory in the system if it is missing.\n
Unlike create_repository, this method doesn't erase any existing repository
in the path but loads it instead.

**N.B. On some systems and some paths, creating a directory may requires root permissions.**

:Parameters:
    #. path (string): The real absolute path where to create the Repository.
       If '.' or an empty string is passed, the current working directory will be used.
    #. info (None, object): Any information that can identify the repository.
    #. verbose (boolean): Whether to be warn and informed about any abnormalities.


def pull_file(self, relativePath, name=None, pull=None, update=True):

<EOL>relativePath = os.path.normpath(relativePath)<EOL>if relativePath == '<STR_LIT:.>':<EOL><INDENT>relativePath = '<STR_LIT>'<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL><DEDENT>if name is None:<EOL><INDENT>assert len(relativePath), "<STR_LIT>"<EOL>relativePath,name = os.path.split(relativePath)<EOL><DEDENT>fileInfo, errorMessage = self.get_file_info(relativePath, name)<EOL>assert fileInfo is not None, errorMessage<EOL>realPath = os.path.join(self.__path, relativePath)<EOL>assert os.path.exists(realPath), "<STR_LIT>"%(relativePath, self.__path)<EOL>fileAbsPath = os.path.join(realPath, name)<EOL>assert os.path.isfile(fileAbsPath), "<STR_LIT>"%(name,realPath)<EOL>if pull is None:<EOL><INDENT>pull = fileInfo["<STR_LIT>"]<EOL><DEDENT>try:<EOL><INDENT>namespace = {}<EOL>namespace.update( globals() )<EOL>exec( pull.replace("<STR_LIT>", str(os.path.join(realPath,name)) ), namespace )<EOL><DEDENT>except Exception as e:<EOL><INDENT>m = pull.replace("<STR_LIT>", str(os.path.join(realPath,name)) )<EOL>raise Exception( "<STR_LIT>"%(m,e) )<EOL><DEDENT>if update:<EOL><INDENT>fileInfo["<STR_LIT>"] = pull<EOL><DEDENT>return namespace['<STR_LIT>']<EOL>

Pull a file's data from the Repository.

:Parameters:
    #. relativePath (string): The relative to the repository path of the directory where the file should be pulled.
    #. name (string): The file name.
       If None is given, name will be split from relativePath.
    #. pull (None, string): The pulling method.
       If None, the pull method saved in the file info will be used.
       If a string is given, the string should include all the necessary imports,
       a '$FILE_PATH' that replaces the absolute file path when the dumping will be performed
       and finally a PULLED_DATA variable.
       e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
    #. update (boolean): If pull is not None, Whether to update the pull method stored in the file info by the given pull method.

:Returns:
    #. data (object): The pulled data from the file.


return self.__state<EOL>


@wraps(func)<EOL>def wrapper(self, *args, **kwargs):<EOL><INDENT>if self.path is None:<EOL><INDENT>warnings.warn('<STR_LIT>')<EOL>return<EOL><DEDENT>return func(self, *args, **kwargs)<EOL><DEDENT>return wrapper<EOL>

Decorate methods when repository path is required.


return self.__path<EOL>

The repository instance path which points to the folder and
        directory where .pyrepinfo is.


@wraps(func)<EOL>def wrapper(self, *args, **kwargs):<EOL><INDENT>with self.locker as r:<EOL><INDENT>acquired, code, _  = r<EOL>if acquired:<EOL><INDENT>try:<EOL><INDENT>r = func(self, *args, **kwargs)<EOL><DEDENT>except Exception as err:<EOL><INDENT>e = str(err)<EOL><DEDENT>else:<EOL><INDENT>e = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>warnings.warn("<STR_LIT>"%(code,func.__name__) )<EOL>e = None<EOL>r = None<EOL><DEDENT><DEDENT>if e is not None:<EOL><INDENT>traceback.print_stack()<EOL>raise Exception(e)<EOL><DEDENT>return r<EOL><DEDENT>return wrapper<EOL>

Decorate methods when locking repository is required.


def walk_directories(directory, relativePath):<EOL><INDENT>directories = dict.__getitem__(directory, '<STR_LIT>')<EOL>dirNames = dict.keys(directories)<EOL>for d in sorted(dirNames):<EOL><INDENT>yield os.path.join(relativePath, d)<EOL><DEDENT>for k in sorted(dict.keys(directories)):<EOL><INDENT>path = os.path.join(relativePath, k)<EOL>dir  = dict.__getitem__(directories, k)<EOL>for e in walk_directories(dir, path):<EOL><INDENT>yield e<EOL><DEDENT><DEDENT><DEDENT>dir, errorMessage = self.get_directory_info(relativePath)<EOL>assert dir is not None, errorMessage<EOL>return walk_directories(dir, relativePath='<STR_LIT>')<EOL>

Walk repository and yield all found directories relative path

:parameters:
    #. relativePath (str): The relative path from which start the walk.


@acquire_lock<EOL><INDENT>@sync_required<EOL>def dump_file(self, value, relativePath, name=None,<EOL>description=None, klass=None,<EOL>dump=None, pull=None,<EOL>replace=False, ACID=None, verbose=False):<DEDENT>

<EOL>if ACID is None:<EOL><INDENT>ACID = self.__ACID<EOL><DEDENT>assert isinstance(ACID, bool), "<STR_LIT>"<EOL>relativePath = os.path.normpath(relativePath)<EOL>if relativePath == '<STR_LIT:.>':<EOL><INDENT>relativePath = '<STR_LIT>'<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL><DEDENT>if name is None:<EOL><INDENT>assert len(relativePath), "<STR_LIT>"<EOL>relativePath,name = os.path.split(relativePath)<EOL><DEDENT>self.add_directory(relativePath)<EOL>realPath = os.path.join(self.__path, relativePath)<EOL>dirInfoDict, errorMessage = self.get_directory_info(relativePath)<EOL>assert dirInfoDict is not None, errorMessage<EOL>if name in dict.__getitem__(dirInfoDict, "<STR_LIT>"):<EOL><INDENT>if not replace:<EOL><INDENT>if verbose:<EOL><INDENT>warnings.warn("<STR_LIT>"%(name))<EOL><DEDENT>return<EOL><DEDENT><DEDENT>if dump is None:<EOL><INDENT>dump=DEFAULT_DUMP<EOL><DEDENT>if pull is None:<EOL><INDENT>pull=DEFAULT_PULL<EOL><DEDENT>if ACID:<EOL><INDENT>savePath = os.path.join(tempfile.gettempdir(), str(uuid.uuid1()))<EOL><DEDENT>else:<EOL><INDENT>savePath = os.path.join(realPath,name)<EOL><DEDENT>try:<EOL><INDENT>exec( dump.replace("<STR_LIT>", str(savePath)) )<EOL><DEDENT>except Exception as e:<EOL><INDENT>message = "<STR_LIT>"%e<EOL>if '<STR_LIT>' in dump:<EOL><INDENT>message += '<STR_LIT>'%str(get_pickling_errors(value))<EOL><DEDENT>raise Exception( message )<EOL><DEDENT>if ACID:<EOL><INDENT>try:<EOL><INDENT>shutil.copyfile(savePath, os.path.join(realPath,name))<EOL><DEDENT>except Exception as e:<EOL><INDENT>os.remove(savePath)<EOL>if verbose:<EOL><INDENT>warnings.warn(e)<EOL><DEDENT>return<EOL><DEDENT>os.remove(savePath)<EOL><DEDENT>if klass is None and value is not None:<EOL><INDENT>klass = value.__class__<EOL><DEDENT>if klass is not None:<EOL><INDENT>assert inspect.isclass(klass), "<STR_LIT>"<EOL><DEDENT>dict.__getitem__(dirInfoDict, "<STR_LIT>")[name] = {"<STR_LIT>":dump,<EOL>"<STR_LIT>":pull,<EOL>"<STR_LIT>":datetime.utcnow(),<EOL>"<STR_LIT:id>":str(uuid.uuid1()),<EOL>"<STR_LIT:class>": klass,<EOL>"<STR_LIT:description>":description}<EOL>self.save()<EOL>

Dump a file using its value to the system and creates its
attribute in the Repository with utc timestamp.

:Parameters:
    #. value (object): The value of a file to dump and add to the repository. It is any python object or file.
    #. relativePath (str): The relative to the repository path of the directory where the file should be dumped.
       If relativePath does not exist, it will be created automatically.
    #. name (string): The file name.
       If None is given, name will be split from relativePath.
    #. description (None, string, pickable object): Any random description about the file.
    #. klass (None, class): The dumped object class. If None is given
       klass will be automatically set to the following value.__class__
    #. dump (None, string): The dumping method.
       If None it will be set automatically to pickle and therefore the object must be pickleable.
       If a string is given, the string should include all the necessary imports
       and a '$FILE_PATH' that replaces the absolute file path when the dumping will be performed.\n
       e.g. "import numpy as np; np.savetxt(fname='$FILE_PATH', X=value, fmt='%.6e')"
    #. pull (None, string): The pulling method.
       If None it will be set automatically to pickle and therefore the object must be pickleable.
       If a string is given, the string should include all the necessary imports,
       a '$FILE_PATH' that replaces the absolute file path when the dumping will be performed
       and finally a PULLED_DATA variable.\n
       e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
    #. replace (boolean): Whether to replace any existing file with the same name if existing.
    #. ACID (None, boolean): Whether to ensure the ACID (Atomicity, Consistency, Isolation, Durability)
       properties of the repository upon dumping a file. This is ensured by dumping the file in
       a temporary path first and then moving it to the desired path.
       If None is given, repository ACID property will be used.
    #. verbose (boolean): Whether to be warn and informed about any abnormalities.


<EOL>errorMessage = "<STR_LIT>"<EOL>relativePath = os.path.normpath(relativePath)<EOL>dirInfoDict, errorMessage = self.get_directory_info(relativePath)<EOL>assert dirInfoDict is not None, errorMessage<EOL>for dname in dict.__getitem__(dirInfoDict, "<STR_LIT>"):<EOL><INDENT>yield os.path.join(relativePath, dname)<EOL><DEDENT>

Walk a certain directory in repository and yield all found directories relative path.

:parameters:
    #. relativePath (str): The relative path of the directory.


return self.__locker<EOL>


@acquire_lock<EOL><INDENT>@sync_required<EOL>def update_file(self, value, relativePath, name=None,<EOL>description=False, klass=False,<EOL>dump=False, pull=False,<EOL>ACID=None, verbose=False):<DEDENT>

<EOL>if ACID is None:<EOL><INDENT>ACID = self.__ACID<EOL><DEDENT>assert isinstance(ACID, bool), "<STR_LIT>"<EOL>relativePath = os.path.normpath(relativePath)<EOL>if relativePath == '<STR_LIT:.>':<EOL><INDENT>relativePath = '<STR_LIT>'<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL>assert name != '<STR_LIT>', "<STR_LIT>"<EOL><DEDENT>if name is None:<EOL><INDENT>assert len(relativePath), "<STR_LIT>"<EOL>relativePath,name = os.path.split(relativePath)<EOL><DEDENT>fileInfoDict, errorMessage = self.get_file_info(relativePath, name)<EOL>assert fileInfoDict is not None, errorMessage<EOL>realPath = os.path.join(self.__path, relativePath)<EOL>if verbose:<EOL><INDENT>if not os.path.isfile( os.path.join(realPath, name) ):<EOL><INDENT>warnings.warn("<STR_LIT>"%os.path.join(realPath, name))<EOL><DEDENT><DEDENT>if not dump:<EOL><INDENT>dump = fileInfoDict["<STR_LIT>"]<EOL><DEDENT>if not pull:<EOL><INDENT>pull = fileInfoDict["<STR_LIT>"]<EOL><DEDENT>if ACID:<EOL><INDENT>savePath = os.path.join(tempfile.gettempdir(), name)<EOL><DEDENT>else:<EOL><INDENT>savePath = os.path.join(realPath,name)<EOL><DEDENT>try:<EOL><INDENT>exec( dump.replace("<STR_LIT>", str(savePath)) )<EOL><DEDENT>except Exception as e:<EOL><INDENT>message = "<STR_LIT>"%e<EOL>if '<STR_LIT>' in dump:<EOL><INDENT>message += '<STR_LIT>'%str(get_pickling_errors(value))<EOL><DEDENT>raise Exception( message )<EOL><DEDENT>if ACID:<EOL><INDENT>try:<EOL><INDENT>shutil.copyfile(savePath, os.path.join(realPath,name))<EOL><DEDENT>except Exception as e:<EOL><INDENT>os.remove(savePath)<EOL>if verbose:<EOL><INDENT>warnings.warn(e)<EOL><DEDENT>return<EOL><DEDENT>os.remove(savePath)<EOL><DEDENT>fileInfoDict["<STR_LIT>"] = datetime.utcnow()<EOL>if description is not False:<EOL><INDENT>fileInfoDict["<STR_LIT:description>"] = description<EOL><DEDENT>if klass is not False:<EOL><INDENT>assert inspect.isclass(klass), "<STR_LIT>"<EOL>fileInfoDict["<STR_LIT:class>"] = klass<EOL><DEDENT>self.save()<EOL>

Update the value and the utc timestamp of a file that is already in the Repository.\n
If file is not registered in repository, and error will be thrown.\n
If file is missing in the system, it will be regenerated as dump method is called.

:Parameters:
    #. value (object): The value of the file to update. It is any python object or a file.
    #. relativePath (str): The relative to the repository path of the directory where the file should be dumped.
    #. name (None, string): The file name.
       If None is given, name will be split from relativePath.
    #. description (False, string, pickable object): Any random description about the file.
       If False is given, the description info won't be updated,
       otherwise it will be update to what description argument value is.
    #. klass (False, class): The dumped object class. If False is given,
       the class info won't be updated, otherwise it will be update to what klass argument value is.
    #. dump (False, string): The new dump method. If False is given, the old one will be used.
    #. pull (False, string): The new pull method. If False is given, the old one will be used.
    #. ACID (boolean): Whether to ensure the ACID (Atomicity, Consistency, Isolation, Durability)
       properties of the repository upon dumping a file. This is ensured by dumping the file in
       a temporary path first and then moving it to the desired path.
       If None is given, repository ACID property will be used.
    #. verbose (boolean): Whether to be warn and informed about any abnormalities.


<EOL>relativePath = os.path.normpath(relativePath)<EOL>dirInfoDict, errorMessage = self.get_directory_info(relativePath)<EOL>assert dirInfoDict is not None, errorMessage<EOL>for fname in dict.__getitem__(dirInfoDict, "<STR_LIT>"):<EOL><INDENT>yield os.path.join(relativePath, fname), dict.__getitem__(dirInfoDict, "<STR_LIT>")[fname]<EOL><DEDENT>

Walk a certain directory in repository and yield tuples as the following:\n
(relative path joined with directory name, file info dict).

:parameters:
    #. relativePath (str): The relative path of the directory.


@acquire_lock<EOL><INDENT>@sync_required<EOL>def dump_copy(self, path, relativePath, name=None,<EOL>description=None,<EOL>replace=False, verbose=False):<DEDENT>

relativePath = os.path.normpath(relativePath)<EOL>if relativePath == '<STR_LIT:.>':<EOL><INDENT>relativePath = '<STR_LIT>'<EOL><DEDENT>if name is None:<EOL><INDENT>_,name = os.path.split(path)<EOL><DEDENT>self.add_directory(relativePath)<EOL>realPath = os.path.join(self.__path, relativePath)<EOL>dirInfoDict, errorMessage = self.get_directory_info(relativePath)<EOL>assert dirInfoDict is not None, errorMessage<EOL>if name in dict.__getitem__(dirInfoDict, "<STR_LIT>"):<EOL><INDENT>if not replace:<EOL><INDENT>if verbose:<EOL><INDENT>warnings.warn("<STR_LIT>"%(name))<EOL><DEDENT>return<EOL><DEDENT><DEDENT>dump = "<STR_LIT>"<EOL>pull = "<STR_LIT>"<EOL>try:<EOL><INDENT>shutil.copyfile(path, os.path.join(realPath,name))<EOL><DEDENT>except Exception as e:<EOL><INDENT>if verbose:<EOL><INDENT>warnings.warn(e)<EOL><DEDENT>return<EOL><DEDENT>klass = None<EOL>dict.__getitem__(dirInfoDict, "<STR_LIT>")[name] = {"<STR_LIT>":dump,<EOL>"<STR_LIT>":pull,<EOL>"<STR_LIT>":datetime.utcnow(),<EOL>"<STR_LIT:id>":str(uuid.uuid1()),<EOL>"<STR_LIT:class>": klass,<EOL>"<STR_LIT:description>":description}<EOL>self.save()<EOL>

Copy an exisitng system file to the repository.
attribute in the Repository with utc timestamp.

:Parameters:
    #. path (str): The full path of the file to copy into the repository.
    #. relativePath (str): The relative to the repository path of the directory where the file should be dumped.
       If relativePath does not exist, it will be created automatically.
    #. name (string): The file name.
       If None is given, name will be split from path.
    #. description (None, string, pickable object): Any random description about the file.
    #. replace (boolean): Whether to replace any existing file with the same name if existing.
    #. verbose (boolean): Whether to be warn and informed about any abnormalities.


relativePath = os.path.normpath(relativePath)<EOL>if relativePath in ('<STR_LIT>','<STR_LIT:.>'):<EOL><INDENT>return self, "<STR_LIT>"<EOL><DEDENT>currentDir  = self.__path<EOL>dirInfoDict = self<EOL>for dir in relativePath.split(os.sep):<EOL><INDENT>dirInfoDict = dict.__getitem__(dirInfoDict, "<STR_LIT>")<EOL>currentDir = os.path.join(currentDir, dir)<EOL>if not os.path.exists(currentDir):<EOL><INDENT>return None,  "<STR_LIT>"%currentDir<EOL><DEDENT>val = dirInfoDict.get(dir, None)<EOL>if val is None:<EOL><INDENT>return None,  "<STR_LIT>"%currentDir<EOL><DEDENT>dirInfoDict = val<EOL><DEDENT>return dirInfoDict, "<STR_LIT>"<EOL>

get directory info from the Repository.

:Parameters:
    #. relativePath (string): The relative to the repository path of the directory.

:Returns:
    #. info (None, dictionary): The directory information dictionary.
       If None, it means an error has occurred.
    #. error (string): The error message if any error occurred.


@acquire_lock<EOL><INDENT>@sync_required<EOL>def remove_directory(self, relativePath, removeFromSystem=False):<DEDENT>

<EOL>relativePath = os.path.normpath(relativePath)<EOL>parentDirInfoDict, errorMessage = self.get_parent_directory_info(relativePath)<EOL>assert parentDirInfoDict is not None, errorMessage<EOL>path, name = os.path.split(relativePath)<EOL>if dict.__getitem__(parentDirInfoDict, '<STR_LIT>').get(name, None) is None:<EOL><INDENT>raise Exception("<STR_LIT>"%(name, path))<EOL><DEDENT>if removeFromSystem:<EOL><INDENT>for rp in self.walk_files_relative_path(relativePath=relativePath):<EOL><INDENT>ap = os.path.join(self.__path, relativePath, rp)<EOL>if not os.path.isfile(ap):<EOL><INDENT>continue<EOL><DEDENT>if not os.path.exists(ap):<EOL><INDENT>continue<EOL><DEDENT>if os.path.isfile(ap):<EOL><INDENT>os.remove( ap )<EOL><DEDENT><DEDENT>for rp in self.walk_directories_relative_path(relativePath=relativePath):<EOL><INDENT>ap = os.path.join(self.__path, relativePath, rp)<EOL>if not os.path.isdir(ap):<EOL><INDENT>continue<EOL><DEDENT>if not os.path.exists(ap):<EOL><INDENT>continue<EOL><DEDENT>if not len(os.listdir(ap)):<EOL><INDENT>os.rmdir(ap)<EOL><DEDENT><DEDENT><DEDENT>dict.__getitem__(parentDirInfoDict, '<STR_LIT>').pop(name, None)<EOL>ap = os.path.join(self.__path, relativePath)<EOL>if not os.path.isdir(ap):<EOL><INDENT>if not len(os.listdir(ap)):<EOL><INDENT>os.rmdir(ap)<EOL><DEDENT><DEDENT>self.save()<EOL>

Remove directory from repository.

:Parameters:
    #. relativePath (string): The relative to the repository path of the directory to remove from the repository.
    #. removeFromSystem (boolean): Whether to also remove directory and all files from the system.\n
       Only files saved in the repository will be removed and empty left directories.


realPath = os.path.realpath( os.path.expanduser(path) )<EOL>if not os.path.isdir(realPath):<EOL><INDENT>return False<EOL><DEDENT>if "<STR_LIT>" not in os.listdir(realPath):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Check if there is a Repository in path.

:Parameters:
    #. path (string): The real path of the directory where to check if there is a repository.

:Returns:
    #. result (boolean): Whether its a repository or not.


assert isinstance(ACID, bool), "<STR_LIT>"<EOL>self.__ACID = ACID<EOL>

Set the gobal ACID poperty of the repository.

:parameters:
    #. ACID (boolean): Whether to ensure the ACID (Atomicity, Consistency, Isolation, Durability)
       properties of the repository upon dumping a file. This is ensured by dumping the file in
       a temporary path first and then moving it to the desired path.


def walk_files(directory, relativePath):<EOL><INDENT>directories = dict.__getitem__(directory, '<STR_LIT>')<EOL>files       = dict.__getitem__(directory, '<STR_LIT>')<EOL>for f in sorted(files):<EOL><INDENT>yield os.path.join(relativePath, f)<EOL><DEDENT>for k in sorted(dict.keys(directories)):<EOL><INDENT>path = os.path.join(relativePath, k)<EOL>dir  = directories.__getitem__(k)<EOL>for e in walk_files(dir, path):<EOL><INDENT>yield e<EOL><DEDENT><DEDENT><DEDENT>dir, errorMessage = self.get_directory_info(relativePath)<EOL>assert dir is not None, errorMessage<EOL>return walk_files(dir, relativePath='<STR_LIT>')<EOL>

Walk the repository and yield all found files relative path joined with file name.

:parameters:
    #. relativePath (str): The relative path from which start the walk.


return __onlinedoc__<EOL>

Get pyrep's official online documentation link.


if(html):<EOL><INDENT>payload = {<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': subject,<EOL>'<STR_LIT:html>': html,<EOL>'<STR_LIT:text>': text<EOL>}<EOL><DEDENT>else:<EOL><INDENT>payload = {<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': subject,<EOL>'<STR_LIT:text>': text<EOL>}<EOL><DEDENT>if locale:<EOL><INDENT>url = self.TEMPLATES_SPECIFIC_LOCALE_VERSIONS_ENDPOINT % (<EOL>template_id,<EOL>locale<EOL>)<EOL><DEDENT>else:<EOL><INDENT>url = self.TEMPLATES_NEW_VERSION_ENDPOINT % template_id<EOL><DEDENT>return self._api_request(<EOL>url,<EOL>self.HTTP_POST,<EOL>payload=payload,<EOL>timeout=timeout<EOL>)<EOL>

API call to create a new version of a template


payload = {<EOL>'<STR_LIT>': locale,<EOL>'<STR_LIT:name>': version_name,<EOL>'<STR_LIT>': subject<EOL>}<EOL>if html:<EOL><INDENT>payload['<STR_LIT:html>'] = html<EOL><DEDENT>if text:<EOL><INDENT>payload['<STR_LIT:text>'] = text<EOL><DEDENT>return self._api_request(<EOL>self.TEMPLATES_LOCALES_ENDPOINT % template_id,<EOL>self.HTTP_POST,<EOL>payload=payload,<EOL>timeout=timeout<EOL>)<EOL>

API call to create a new locale and version of a template


if not self._raise_errors:<EOL><INDENT>return response<EOL><DEDENT>is_4xx_error = str(response.status_code)[<NUM_LIT:0>] == '<STR_LIT:4>'<EOL>is_5xx_error = str(response.status_code)[<NUM_LIT:0>] == '<STR_LIT:5>'<EOL>content = response.content<EOL>if response.status_code == <NUM_LIT>:<EOL><INDENT>raise AuthenticationError(content)<EOL><DEDENT>elif is_4xx_error:<EOL><INDENT>raise APIError(content)<EOL><DEDENT>elif is_5xx_error:<EOL><INDENT>raise ServerError(content)<EOL><DEDENT>return response<EOL>

Parses the API response and raises appropriate errors if
        raise_errors was set to True


if not email_data:<EOL><INDENT>email_data = {}<EOL><DEDENT>if isinstance(recipient, string_types):<EOL><INDENT>warnings.warn(<EOL>"<STR_LIT>",<EOL>DeprecationWarning)<EOL>recipient = {'<STR_LIT:address>': recipient}<EOL><DEDENT>payload = {<EOL>'<STR_LIT>': email_id,<EOL>'<STR_LIT>': recipient,<EOL>'<STR_LIT>': email_data<EOL>}<EOL>if sender:<EOL><INDENT>payload['<STR_LIT>'] = sender<EOL><DEDENT>if cc:<EOL><INDENT>if not type(cc) == list:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % type(cc))<EOL><DEDENT>payload['<STR_LIT>'] = cc<EOL><DEDENT>if bcc:<EOL><INDENT>if not type(bcc) == list:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % type(bcc))<EOL><DEDENT>payload['<STR_LIT>'] = bcc<EOL><DEDENT>if tags:<EOL><INDENT>if not type(tags) == list:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % (type(tags)))<EOL><DEDENT>payload['<STR_LIT>'] = tags<EOL><DEDENT>if headers:<EOL><INDENT>if not type(headers) == dict:<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % (<EOL>type(headers)<EOL>)<EOL>)<EOL><DEDENT>payload['<STR_LIT>'] = headers<EOL><DEDENT>if esp_account:<EOL><INDENT>if not isinstance(esp_account, string_types):<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % (<EOL>type(esp_account)<EOL>)<EOL>)<EOL><DEDENT>payload['<STR_LIT>'] = esp_account<EOL><DEDENT>if locale:<EOL><INDENT>if not isinstance(locale, string_types):<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % (type(locale))<EOL>)<EOL><DEDENT>payload['<STR_LIT>'] = locale<EOL><DEDENT>if email_version_name:<EOL><INDENT>if not isinstance(email_version_name, string_types):<EOL><INDENT>logger.error(<EOL>'<STR_LIT>' % (<EOL>type(email_version_name)))<EOL><DEDENT>payload['<STR_LIT>'] = email_version_name<EOL><DEDENT>if inline:<EOL><INDENT>payload['<STR_LIT>'] = self._make_file_dict(inline)<EOL><DEDENT>if files:<EOL><INDENT>payload['<STR_LIT>'] = [self._make_file_dict(f) for f in files]<EOL><DEDENT>return self._api_request(<EOL>self.SEND_ENDPOINT,<EOL>self.HTTP_POST,<EOL>payload=payload,<EOL>timeout=timeout<EOL>)<EOL>


return self._api_request(<EOL>self.TEMPLATES_ENDPOINT,<EOL>self.HTTP_GET,<EOL>timeout=timeout<EOL>)<EOL>


def _api_request(self, endpoint, http_method, *args, **kwargs):

logger.debug('<STR_LIT>' % endpoint)<EOL>path = self._build_request_path(endpoint, absolute=False)<EOL>logger.debug('<STR_LIT>' % path)<EOL>data = None<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>data = kwargs['<STR_LIT>']<EOL><DEDENT>logger.debug('<STR_LIT>' % data)<EOL>command = {<EOL>"<STR_LIT:path>": path,<EOL>"<STR_LIT>": http_method<EOL>}<EOL>if data:<EOL><INDENT>command['<STR_LIT:body>'] = data<EOL><DEDENT>self._commands.append(command)<EOL>


if not api_key:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self.API_KEY = api_key<EOL>self.DEFAULT_TIMEOUT = default_timeout<EOL>self._json_encoder = json_encoder<EOL>self._raise_errors = raise_errors<EOL>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.API_HOST = kwargs['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.API_PROTO = kwargs['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.API_PORT = kwargs['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.API_VERSION = kwargs['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>self.DEBUG = kwargs['<STR_LIT>']<EOL><DEDENT>if self.DEBUG:<EOL><INDENT>logging.basicConfig(format=LOGGER_FORMAT, level=logging.DEBUG)<EOL>logger.debug('<STR_LIT>')<EOL>logger.propagate = True<EOL><DEDENT>


def create_snippet(self, name, body, timeout=None):

payload = {<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT:body>': body<EOL>}<EOL>return self._api_request(<EOL>self.SNIPPETS_ENDPOINT,<EOL>self.HTTP_POST,<EOL>payload=payload,<EOL>timeout=timeout<EOL>)<EOL>


def get_template(self, template_id, version=None, timeout=None):

if (version):<EOL><INDENT>return self._api_request(<EOL>self.TEMPLATES_VERSION_ENDPOINT % (template_id, version),<EOL>self.HTTP_GET,<EOL>timeout=timeout<EOL>)<EOL><DEDENT>else:<EOL><INDENT>return self._api_request(<EOL>self.TEMPLATES_SPECIFIC_ENDPOINT % template_id,<EOL>self.HTTP_GET,<EOL>timeout=timeout<EOL>)<EOL><DEDENT>


logger.debug('<STR_LIT>' % len(self._commands))<EOL>auth = self._build_http_auth()<EOL>headers = self._build_request_headers()<EOL>logger.debug('<STR_LIT>' % headers)<EOL>logger.debug('<STR_LIT>' % len(self._commands))<EOL>path = self._build_request_path(self.BATCH_ENDPOINT)<EOL>data = json.dumps(self._commands, cls=self._json_encoder)<EOL>r = requests.post(<EOL>path,<EOL>auth=auth,<EOL>headers=headers,<EOL>data=data,<EOL>timeout=(self.DEFAULT_TIMEOUT if timeout is None else timeout)<EOL>)<EOL>self._commands = []<EOL>logger.debug('<STR_LIT>' % r.status_code)<EOL>try:<EOL><INDENT>logger.debug('<STR_LIT>' % r.json())<EOL><DEDENT>except:<EOL><INDENT>logger.debug('<STR_LIT>' % r.content)<EOL><DEDENT>return r<EOL>

Execute all currently queued batch commands


payload = {<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': subject,<EOL>'<STR_LIT:html>': html,<EOL>'<STR_LIT:text>': text<EOL>}<EOL>return self._api_request(<EOL>self.TEMPLATES_ENDPOINT,<EOL>self.HTTP_POST,<EOL>payload=payload,<EOL>timeout=timeout<EOL>)<EOL>


def get_log(self, log_id, timeout=None):

return self._api_request(<EOL>self.GET_LOG_ENDPOINT % log_id,<EOL>self.HTTP_GET,<EOL>timeout=timeout<EOL>)<EOL>


if isinstance(f, dict):<EOL><INDENT>file_obj = f['<STR_LIT:file>']<EOL>if '<STR_LIT:filename>' in f:<EOL><INDENT>file_name = f['<STR_LIT:filename>']<EOL><DEDENT>else:<EOL><INDENT>file_name = file_obj.name<EOL><DEDENT><DEDENT>else:<EOL><INDENT>file_obj = f<EOL>file_name = f.name<EOL><DEDENT>b64_data = base64.b64encode(file_obj.read())<EOL>return {<EOL>'<STR_LIT:id>': file_name,<EOL>'<STR_LIT:data>': b64_data.decode() if six.PY3 else b64_data,<EOL>}<EOL>

Make a dictionary with filename and base64 file data


logger.debug('<STR_LIT>' % endpoint)<EOL>auth = self._build_http_auth()<EOL>headers = self._build_request_headers(kwargs.get('<STR_LIT>'))<EOL>logger.debug('<STR_LIT>' % headers)<EOL>path = self._build_request_path(endpoint)<EOL>logger.debug('<STR_LIT>' % path)<EOL>data = self._build_payload(kwargs.get('<STR_LIT>'))<EOL>if not data:<EOL><INDENT>data = kwargs.get('<STR_LIT:data>')<EOL><DEDENT>logger.debug('<STR_LIT>' % data)<EOL>req_kw = dict(<EOL>auth=auth,<EOL>headers=headers,<EOL>timeout=kwargs.get('<STR_LIT>', self.DEFAULT_TIMEOUT)<EOL>)<EOL>if (http_method == self.HTTP_POST):<EOL><INDENT>if (data):<EOL><INDENT>r = requests.post(path, data=data, **req_kw)<EOL><DEDENT>else:<EOL><INDENT>r = requests.post(path, **req_kw)<EOL><DEDENT><DEDENT>elif http_method == self.HTTP_PUT:<EOL><INDENT>if (data):<EOL><INDENT>r = requests.put(path, data=data, **req_kw)<EOL><DEDENT>else:<EOL><INDENT>r = requests.put(path, **req_kw)<EOL><DEDENT><DEDENT>elif http_method == self.HTTP_DELETE:<EOL><INDENT>r = requests.delete(path, **req_kw)<EOL><DEDENT>else:<EOL><INDENT>r = requests.get(path, **req_kw)<EOL><DEDENT>logger.debug('<STR_LIT>' % r.status_code)<EOL>try:<EOL><INDENT>logger.debug('<STR_LIT>' % r.json())<EOL><DEDENT>except:<EOL><INDENT>logger.debug('<STR_LIT>' % r.content)<EOL><DEDENT>return self._parse_response(r)<EOL>


if n == <NUM_LIT:0>:<EOL><INDENT>return n<EOL><DEDENT>else:<EOL><INDENT>return foo(n - <NUM_LIT:1>)<EOL><DEDENT>


def _get_namedrange(book, rangename, sheetname=None):

def cond(namedef):<EOL><INDENT>if namedef.type.upper() == "<STR_LIT>":<EOL><INDENT>if namedef.name.upper() == rangename.upper():<EOL><INDENT>if sheetname is None:<EOL><INDENT>if not namedef.localSheetId:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>sheet_id = [sht.upper() for sht in book.sheetnames].index(<EOL>sheetname.upper()<EOL>)<EOL>if namedef.localSheetId == sheet_id:<EOL><INDENT>return True<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return False<EOL><DEDENT>def get_destinations(name_def):<EOL><INDENT>"""<STR_LIT>"""<EOL>from openpyxl.formula import Tokenizer<EOL>from openpyxl.utils.cell import SHEETRANGE_RE<EOL>if name_def.type == "<STR_LIT>":<EOL><INDENT>tok = Tokenizer("<STR_LIT:=>" + name_def.value)<EOL>for part in tok.items:<EOL><INDENT>if part.subtype == "<STR_LIT>":<EOL><INDENT>m = SHEETRANGE_RE.match(part.value)<EOL>if m.group("<STR_LIT>"):<EOL><INDENT>sheet_name = m.group("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>sheet_name = m.group("<STR_LIT>")<EOL><DEDENT>yield sheet_name, m.group("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>namedef = next(<EOL>(item for item in book.defined_names.definedName if cond(item)), None<EOL>)<EOL>if namedef is None:<EOL><INDENT>return None<EOL><DEDENT>dests = get_destinations(namedef)<EOL>xlranges = []<EOL>sheetnames_upper = [name.upper() for name in book.sheetnames]<EOL>for sht, addr in dests:<EOL><INDENT>if sheetname:<EOL><INDENT>sht = sheetname<EOL><DEDENT>index = sheetnames_upper.index(sht.upper())<EOL>xlranges.append(book.worksheets[index][addr])<EOL><DEDENT>if len(xlranges) == <NUM_LIT:1>:<EOL><INDENT>return xlranges[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return xlranges<EOL><DEDENT>

Get range from a workbook.

    A workbook can contain multiple definitions for a single name,
    as a name can be defined for the entire book or for
    a particular sheet.

    If sheet is None, the book-wide def is searched,
    otherwise sheet-local def is looked up.

    Args:
        book: An openpyxl workbook object.
        rangename (str): Range expression, such as "A1", "$G4:$K10",
            named range "NamedRange1".
        sheetname (str, optional): None for book-wide name def,
            sheet name for sheet-local named range.

    Returns:
        Range object specified by the name.


def read_range(filepath, range_expr, sheet=None, dict_generator=None):

def default_generator(cells):<EOL><INDENT>for row_ind, row in enumerate(cells):<EOL><INDENT>for col_ind, cell in enumerate(row):<EOL><INDENT>yield (row_ind, col_ind), cell.value<EOL><DEDENT><DEDENT><DEDENT>book = opxl.load_workbook(filepath, data_only=True)<EOL>if _is_range_address(range_expr):<EOL><INDENT>sheet_names = [name.upper() for name in book.sheetnames]<EOL>index = sheet_names.index(sheet.upper())<EOL>cells = book.worksheets[index][range_expr]<EOL><DEDENT>else:<EOL><INDENT>cells = _get_namedrange(book, range_expr, sheet)<EOL><DEDENT>if isinstance(cells, opxl.cell.Cell):<EOL><INDENT>return cells.value<EOL><DEDENT>if dict_generator is None:<EOL><INDENT>dict_generator = default_generator<EOL><DEDENT>gen = dict_generator(cells)<EOL>return {keyval[<NUM_LIT:0>]: keyval[<NUM_LIT:1>] for keyval in gen}<EOL>

Read values from an Excel range into a dictionary.

    `range_expr` ie either a range address string, such as "A1", "$C$3:$E$5",
    or a defined name string for a range, such as "NamedRange1".
    If a range address is provided, `sheet` argument must also be provided.
    If a named range is provided and `sheet` is not, book level defined name
    is searched. If `sheet` is also provided, sheet level defined name for the
    specified `sheet` is searched.
    If range_expr points to a single cell, its value is returned.

    `dictgenerator` is a generator function that yields keys and values of
    the returned dictionary. the excel range, as a nested tuple of openpyxl's
    Cell objects, is passed to the generator function as its single argument.
    If not specified, default generator is used, which maps tuples of row and
    column indexes, both starting with 0, to their values.

    Args:
        filepath (str): Path to an Excel file.
        range_epxr (str): Range expression, such as "A1", "$G4:$K10",
            or named range "NamedRange1"
        sheet (str): Sheet name (case ignored).
            None if book level defined range name is passed as `range_epxr`.
        dict_generator: A generator function taking a nested tuple of cells
            as a single parameter.

    Returns:
        Nested list containing range values.


index = string.ascii_uppercase.index<EOL>col = <NUM_LIT:0><EOL>for c in name.upper():<EOL><INDENT>col = col * <NUM_LIT> + index(c) + <NUM_LIT:1><EOL><DEDENT>return col<EOL>


super().restore_state(system)<EOL>BaseSpaceContainerImpl.restore_state(self, system)<EOL>for cells in self._cells.values():<EOL><INDENT>cells.restore_state(system)<EOL><DEDENT>

Called after unpickling to restore some attributes manually.


return self._impl.is_derived<EOL>

True if the space is a derived space, False otherwise.


return self._impl.cells.interfaces<EOL>

A mapping of cells names to the cells objects in the space.


def get_dynspace(self, args, kwargs=None):

node = get_node(self, *convert_args(args, kwargs))<EOL>key = node[KEY]<EOL>if key in self.param_spaces:<EOL><INDENT>return self.param_spaces[key]<EOL><DEDENT>else:<EOL><INDENT>last_self = self.system.self<EOL>self.system.self = self<EOL>try:<EOL><INDENT>space_args = self.eval_formula(node)<EOL><DEDENT>finally:<EOL><INDENT>self.system.self = last_self<EOL><DEDENT>if space_args is None:<EOL><INDENT>space_args = {"<STR_LIT>": [self]}  <EOL><DEDENT>else:<EOL><INDENT>if "<STR_LIT>" in space_args:<EOL><INDENT>bases = get_impls(space_args["<STR_LIT>"])<EOL>if isinstance(bases, StaticSpaceImpl):<EOL><INDENT>space_args["<STR_LIT>"] = [bases]<EOL><DEDENT>elif bases is None:<EOL><INDENT>space_args["<STR_LIT>"] = [self]  <EOL><DEDENT>else:<EOL><INDENT>space_args["<STR_LIT>"] = bases<EOL><DEDENT><DEDENT>else:<EOL><INDENT>space_args["<STR_LIT>"] = [self]<EOL><DEDENT><DEDENT>space_args["<STR_LIT>"] = node_get_args(node)<EOL>space = self._new_dynspace(**space_args)<EOL>self.param_spaces[key] = space<EOL>space.inherit(clear_value=False)<EOL>return space<EOL><DEDENT>

Create a dynamic root space

        Called from interface methods


args = [repr(arg) for arg in get_interfaces(self.argvalues)]<EOL>param = "<STR_LIT:U+002CU+0020>".join(args)<EOL>return "<STR_LIT>" % (self.parent.evalrepr, param)<EOL>


return isinstance(self._impl, RootDynamicSpaceImpl)<EOL>

True if ths space is a dynamic space, False otherwise.


return self._impl.formula<EOL>


return self._impl.is_base(other._impl)<EOL>

True if the space is a base space of ``other``, False otherwise.


return self._impl.derived_spaces.interfaces<EOL>

A mapping associating names to derived spaces.


return self._impl.add_bases(get_impls(bases))<EOL>


if isinstance(item, str):<EOL><INDENT>return item in self._impl.namespace<EOL><DEDENT>elif isinstance(item, Cells):<EOL><INDENT>return item._impl in self._impl.cells.values()<EOL><DEDENT>elif isinstance(item, StaticSpace):<EOL><INDENT>return item._impl in self._impl.spaces.values()<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if item is in the space.

        item can be either a cells or space.

        Args:
            item: a cells or space to check.

        Returns:
            True if item is a direct child of the space, False otherwise.


<EOL>newcells = self._impl.new_cells_from_module(module)<EOL>return get_interfaces(newcells)<EOL>

Create a cells from a module.

        Alias to :py:meth:`import_funcs`.


return self._impl.derived_cells.interfaces<EOL>

A mapping associating names to derived cells.


return get_interfaces(self._impl.bases)<EOL>


parts = name.split("<STR_LIT:.>")<EOL>child = parts.pop(<NUM_LIT:0>)<EOL>if parts:<EOL><INDENT>return self.spaces[child].get_object("<STR_LIT:.>".join(parts))<EOL><DEDENT>else:<EOL><INDENT>return self._namespace_impl[child]<EOL><DEDENT>

Retrieve an object by a dotted name relative to the space.


return self._impl.remove_bases(bases)<EOL>


<EOL>return bool(self._impl.formula)<EOL>


lines = source.splitlines()<EOL>atok = asttokens.ASTTokens(source, parse=True)<EOL>for node in ast.walk(atok.tree):<EOL><INDENT>if isinstance(node, ast.FunctionDef):<EOL><INDENT>break<EOL><DEDENT><DEDENT>if node.decorator_list:<EOL><INDENT>deco_first = node.decorator_list[<NUM_LIT:0>]<EOL>deco_last = node.decorator_list[-<NUM_LIT:1>]<EOL>line_first = atok.tokens[deco_first.first_token.index - <NUM_LIT:1>].start[<NUM_LIT:0>]<EOL>line_last = atok.tokens[deco_last.last_token.index + <NUM_LIT:1>].start[<NUM_LIT:0>]<EOL>lines = lines[:line_first - <NUM_LIT:1>] + lines[line_last:]<EOL><DEDENT>return "<STR_LIT:\n>".join(lines) + "<STR_LIT:\n>"<EOL>

Remove decorators from function definition


try:<EOL><INDENT>module_node = compile(<EOL>source, "<STR_LIT>", mode="<STR_LIT>", flags=ast.PyCF_ONLY_AST<EOL>)<EOL><DEDENT>except SyntaxError:<EOL><INDENT>return find_funcdef(fix_lamdaline(source))<EOL><DEDENT>for node in ast.walk(module_node):<EOL><INDENT>if isinstance(node, ast.FunctionDef) or isinstance(node, ast.Lambda):<EOL><INDENT>return node<EOL><DEDENT><DEDENT>raise ValueError("<STR_LIT>")<EOL>

Find the first FuncDef ast object in source


return {"<STR_LIT:source>": self.source, "<STR_LIT>": self.module}<EOL>


if self.module is None:<EOL><INDENT>raise RuntimeError<EOL><DEDENT>elif module is None:<EOL><INDENT>import importlib<EOL>module = ModuleSource(importlib.reload(module))<EOL><DEDENT>elif module.name != self.module:<EOL><INDENT>raise RuntimeError<EOL><DEDENT>if self.name in module.funcs:<EOL><INDENT>func = module.funcs[self.name]<EOL>self.__init__(func=func)<EOL><DEDENT>else:<EOL><INDENT>self.__init__(func=NULL_FORMULA)<EOL><DEDENT>return self<EOL>

Reload the source function from the source module.

        **Internal use only**
        Update the source function of the formula.
        This method is used to updated the underlying formula
        when the source code of the module in which the source function
        is read from is modified.

        If the formula was not created from a module, an error is raised.
        If ``module_`` is not given, the source module of the formula is
        reloaded. If ``module_`` is given and matches the source module,
        then the module_ is used without being reloaded.
        If ``module_`` is given and does not match the source module of
        the formula, an error is raised.

        Args:
            module_: A ``ModuleSource`` object

        Returns:
            self


funcdef = find_funcdef(source)<EOL>params = []<EOL>for node in ast.walk(funcdef.args):<EOL><INDENT>if isinstance(node, ast.arg):<EOL><INDENT>if node.arg not in params:<EOL><INDENT>params.append(node.arg)<EOL><DEDENT><DEDENT><DEDENT>return params<EOL>

Extract parameters from a function definition


def clear_descendants(self, source, clear_source=True):

desc = nx.descendants(self, source)<EOL>if clear_source:<EOL><INDENT>desc.add(source)<EOL><DEDENT>self.remove_nodes_from(desc)<EOL>return desc<EOL>

Remove all descendants of(reachable from) `source`.

        Args:
            source: Node descendants
            clear_source(bool): Remove origin too if True.
        Returns:
            set: The removed nodes.


self._impl.system.rename_model(new_name=name, old_name=self.name)<EOL>


try:<EOL><INDENT>self.add_node("<STR_LIT>")<EOL>for base in bases:<EOL><INDENT>nx.DiGraph.add_edge(self, base, "<STR_LIT>")<EOL><DEDENT>result = self.get_mro("<STR_LIT>")[<NUM_LIT:1>:]<EOL><DEDENT>finally:<EOL><INDENT>self.remove_node("<STR_LIT>")<EOL><DEDENT>return result<EOL>

Check if C3 MRO is possible with given bases


obj_nodes = self.get_nodes_with(obj)<EOL>removed = set()<EOL>for node in obj_nodes:<EOL><INDENT>if self.has_node(node):<EOL><INDENT>removed.update(self.clear_descendants(node))<EOL><DEDENT><DEDENT>return removed<EOL>

Remove all nodes with `obj` and their descendants.


parts = name.split("<STR_LIT:.>")<EOL>space = self.spaces[parts.pop(<NUM_LIT:0>)]<EOL>if parts:<EOL><INDENT>return space.get_object("<STR_LIT:.>".join(parts))<EOL><DEDENT>else:<EOL><INDENT>return space<EOL><DEDENT>

Retrieve an object by a dotted name relative to the model.


removed = self.cellgraph.clear_descendants(source, clear_source)<EOL>for node in removed:<EOL><INDENT>del node[OBJ].data[node[KEY]]<EOL><DEDENT>

Clear values and nodes calculated from `source`.


return self._impl.global_refs.interfaces<EOL>


sys.setrecursionlimit(<NUM_LIT:10>**<NUM_LIT:6>)<EOL>warnings.showwarning = custom_showwarning<EOL>threading.stack_size(<NUM_LIT>)<EOL>

Configure Python settings for modelx

        The error handler is configured later.


def excepthook(self, except_type, exception, traceback):

if except_type is DeepReferenceError:<EOL><INDENT>print(exception.msg)<EOL><DEDENT>else:<EOL><INDENT>self.default_excepthook(except_type, exception, traceback)<EOL><DEDENT>

Not Used: Custom exception hook to replace sys.excepthook

    This is for CPython's default shell. IPython does not use sys.exepthook.

    https://stackoverflow.com/questions/27674602/hide-traceback-unless-a-debug-flag-is-set


parts = name.split("<STR_LIT:.>")<EOL>model_name = parts.pop(<NUM_LIT:0>)<EOL>return self.models[model_name].get_object("<STR_LIT:.>".join(parts))<EOL>

Retrieve an object by its absolute name.


def custom_showwarning(<EOL>message, category, filename="<STR_LIT>", lineno=-<NUM_LIT:1>, file=None, line=None<EOL>):

if file is None:<EOL><INDENT>file = sys.stderr<EOL>if file is None:<EOL><INDENT>return<EOL><DEDENT><DEDENT>text = "<STR_LIT>" % (category.__name__, message)<EOL>try:<EOL><INDENT>file.write(text)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>

Hook to override default showwarning.

    https://stackoverflow.com/questions/2187269/python-print-only-the-message-on-warnings


result = "<STR_LIT>"<EOL>for i, value in enumerate(self):<EOL><INDENT>result += "<STR_LIT>".format(i, get_node_repr(value))<EOL><DEDENT>result = result.strip("<STR_LIT:\n>")<EOL>lines = result.split("<STR_LIT:\n>")<EOL>if maxlen and len(lines) > maxlen:<EOL><INDENT>i = int(maxlen / <NUM_LIT:2>)<EOL>lines = lines[:i] + ["<STR_LIT>"] + lines[-(maxlen - i) :]<EOL>result = "<STR_LIT:\n>".join(lines)<EOL><DEDENT>return result<EOL>

if maxlen > 0, the message is shortened to maxlen traces.


self.default_showtraceback(<EOL>exc_tuple,<EOL>filename,<EOL>tb_offset,<EOL>exception_only=True,<EOL>running_compiled_code=running_compiled_code,<EOL>)<EOL>

Custom showtraceback for monkey-patching IPython's InteractiveShell

    https://stackoverflow.com/questions/1261668/cannot-override-sys-excepthook


return self + -other<EOL>


if self.has_value:<EOL><INDENT>return self._impl[OBJ].get_value(self._impl[KEY])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>


return self.__add__(other)<EOL>


return self._impl.clear_value(*args, **kwargs)<EOL>


return self._impl.single_value * other<EOL>


return self._impl.successors(args, kwargs)<EOL>

Return a list of successors of a cell.

        This method returns a list of CellNode objects, whose elements are
        successors of (i.e. referencing in their formulas)
        the cell specified by the given arguments.


return self.__eq__(other) or self.__gt__(other)<EOL>


return CellNode(get_node(self._impl, *convert_args(args, kwargs)))<EOL>

Return a :class:`CellNode` object for the given arguments.


return self._impl[OBJ].has_cell(self._impl[KEY])<EOL>

Return ``True`` if the cell has a value.


return -self + other<EOL>


return self._impl.to_frame(())<EOL>


return self._impl.single_value > other<EOL>


return self._impl.single_value / other<EOL>

self / other: Should promote to float when necessary.


raise NotImplementedError<EOL>

Returns the Real distance from 0. Called for abs(self).


return other / self._impl.single_value<EOL>


return self._impl.formula<EOL>


return self.__eq__(other) or self.__lt__(other)<EOL>


return self._impl[KEY]<EOL>


return self._impl.find_match(args, kwargs)<EOL>

Returns the best matching args and their value.

        If the cells returns None for the given arguments,
        continue to get a value by passing arguments
        masking the given arguments with Nones.
        The search of non-None value starts from the given arguments
        to the all None arguments in the lexicographical order.
        The masked arguments that returns non-None value
        first is returned with the value.


self._impl.set_value(tuplize_key(self, key), value)<EOL>


self.system = system<EOL>

Called after unpickling to restore some attributes manually.


if self._impl.parent is None:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>return self._impl.parent.interface<EOL><DEDENT>

The parent of this object. None for models.

        The parent object of a cells is a space that contains the cells.
        The parent object of a space is either a model or another space
        that contains the space.


if self.lazy_evals is None:<EOL><INDENT>return<EOL><DEDENT>elif isinstance(self.lazy_evals, LazyEval):<EOL><INDENT>self.lazy_evals.get_updated()<EOL><DEDENT>else:<EOL><INDENT>for lz in self.lazy_evals:<EOL><INDENT>lz.get_updated()<EOL><DEDENT><DEDENT>

Update all LazyEvals in self

        self.lzy_evals must be set to LazyEval object(s) enough to
        update all owned LazyEval objects.


result = {<EOL>"<STR_LIT:type>": type(self).__name__,<EOL>"<STR_LIT:id>": id(self),<EOL>"<STR_LIT:name>": self.name,<EOL>"<STR_LIT>": self.fullname,<EOL>"<STR_LIT>": self._get_repr(),<EOL>}<EOL>return result<EOL>


func = self.owner.formula.func<EOL>codeobj = func.__code__<EOL>name = func.__name__  <EOL>namespace_impl = self.owner._namespace_impl.get_updated()<EOL>namespace = namespace_impl.interfaces<EOL>selfnode = get_node(self.owner, None, None)<EOL>for name in self.owner.formula.srcnames:<EOL><INDENT>if name in namespace_impl and isinstance(<EOL>namespace_impl[name], ReferenceImpl<EOL>):<EOL><INDENT>refnode = get_node(namespace_impl[name], None, None)<EOL>self.owner.model.lexdep.add_path([selfnode, refnode])<EOL><DEDENT><DEDENT>closure = func.__closure__  <EOL>if closure is not None:  <EOL><INDENT>closure = create_closure(self.owner.interface)<EOL><DEDENT>self.altfunc = FunctionType(<EOL>codeobj, namespace, name=name, closure=closure<EOL>)<EOL>


result = {"<STR_LIT:type>": type(self).__name__}<EOL>try:<EOL><INDENT>result["<STR_LIT>"] = {<EOL>name: item._baseattrs<EOL>for name, item in self.items()<EOL>if name[<NUM_LIT:0>] != "<STR_LIT:_>"<EOL>}<EOL><DEDENT>except:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % self)<EOL><DEDENT>return result<EOL>


result = self._baseattrs<EOL>for attr in attrs:<EOL><INDENT>if hasattr(self, attr):<EOL><INDENT>result[attr] = getattr(self, attr)._to_attrdict(attrs)<EOL><DEDENT><DEDENT>return result<EOL>


from modelx.core.space import StaticSpaceImpl<EOL>if name is None:<EOL><INDENT>name = self.spacenamer.get_next(self.namespace, prefix)<EOL><DEDENT>if name in self.namespace:<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>if not prefix and not is_valid_name(name):<EOL><INDENT>raise ValueError("<STR_LIT>" % name)<EOL><DEDENT>space = self._new_space(<EOL>name=name,<EOL>formula=formula,<EOL>refs=refs,<EOL>source=source,<EOL>is_derived=is_derived,<EOL>)<EOL>self._set_space(space)<EOL>self.model.spacegraph.add_space(space)<EOL>if bases is not None:<EOL><INDENT>if isinstance(bases, StaticSpaceImpl):<EOL><INDENT>bases = [bases]<EOL><DEDENT>space.add_bases(bases)<EOL><DEDENT>return space<EOL>

Create a new child space.

        Args:
            name (str): Name of the space. If omitted, the space is
                created automatically.
            bases: If specified, the new space becomes a derived space of
                the `base` space.
            formula: Function whose parameters used to set space parameters.
            refs: a mapping of refs to be added.
            arguments: ordered dict of space parameter names to their values.
            source: A source module from which cell definitions are read.
            prefix: Prefix to the autogenerated name when name is None.


for space in self._spaces.values():<EOL><INDENT>space.restore_state(system)<EOL><DEDENT>


Create a child space from an Excel range.

        To use this method, ``openpyxl`` package must be installed.

        Args:
            book (str): Path to an Excel file.
            range_ (str): Range expression, such as "A1", "$G4:$K10",
                or named range "NamedRange1".
            sheet (str): Sheet name (case ignored).
            name (str, optional): Name of the space. Defaults to ``SpaceN``,
                where ``N`` is a number determined automatically.
            names_row (optional): an index number indicating
                what row contains the names of cells and parameters.
                Defaults to the top row (0).
            param_cols (optional): a sequence of index numbers
                indicating parameter columns.
                Defaults to only the leftmost column ([0]).
            names_col (optional): an index number, starting from 0,
                indicating what column contains additional parameters.
            param_rows (optional): a sequence of index numbers, starting from
                0, indicating rows of additional parameters, in case cells are
                defined in two dimensions.
            transpose (optional): Defaults to ``False``.
                If set to ``True``, "row(s)" and "col(s)" in the parameter
                names are interpreted inversely, i.e.
                all indexes passed to "row(s)" parameters are interpreted
                as column indexes,
                and all indexes passed to "col(s)" parameters as row indexes.
            space_param_order: a sequence to specify space parameters and
                their orders. The elements of the sequence denote the indexes
                of ``param_cols`` elements, and optionally the index of
                ``param_rows`` elements shifted by the length of
                ``param_cols``. The elements of this parameter and
                ``cell_param_order`` must not overlap.
            cell_param_order (optional): a sequence to reorder the parameters.
                The elements of the sequence denote the indexes of
                ``param_cols`` elements, and optionally the index of
                ``param_rows`` elements shifted by the length of
                ``param_cols``. The elements of this parameter and
                ``cell_space_order`` must not overlap.

        Returns:
            The new child space created from the Excel range.


result = super()._baseattrs<EOL>result["<STR_LIT>"] = self.spaces._baseattrs<EOL>return result<EOL>


def new_space(self, name=None, bases=None, formula=None, refs=None):

space = self._impl.model.currentspace = self._impl.new_space(<EOL>name=name, bases=get_impls(bases), formula=formula, refs=refs<EOL>)<EOL>return space.interface<EOL>

Create a child space.

        Args:
            name (str, optional): Name of the space. Defaults to ``SpaceN``,
                where ``N`` is a number determined automatically.
            bases (optional): A space or a sequence of spaces to be the base
                space(s) of the created space.
            formula (optional): Function to specify the parameters of
                dynamic child spaces. The signature of this function is used
                for setting parameters for dynamic child spaces.
                This function should return a mapping of keyword arguments
                to be passed to this method when the dynamic child spaces
                are created.

        Returns:
            The new child space.


def new_space(name=None, bases=None, formula=None):

return cur_model().new_space(name, bases, formula)<EOL>

Create and return a new space in the current model.

    The ``currentspace`` of the current model is set to the created model.

    Args:
        name (:obj:`str`, optional): The name of the space to create.
            Defaults to ``SpaceN``, with ``N``
            being an automatically assigned integer.

    Returns:
        The new space.


obj = get_object(name)<EOL>args = ast.literal_eval(args)<EOL>if not isinstance(args, tuple):<EOL><INDENT>args = (args,)<EOL><DEDENT>return obj.node(*args)<EOL>

Get node from object name and arg string

    Not Used. Left for future reference purpose.


Configure Python ``sys`` settings for modelx.

    This function is called implicitly when importing modelx.
    To restore the Python settings, call :py:func:`restore_python`


<EOL>elms = name.split("<STR_LIT:.>")<EOL>parent = get_models()[elms.pop(<NUM_LIT:0>)]<EOL>while len(elms) > <NUM_LIT:0>:<EOL><INDENT>obj = elms.pop(<NUM_LIT:0>)<EOL>parent = getattr(parent, obj)<EOL><DEDENT>return parent<EOL>


Set up IPython shell for modelx.

    Suppress IPython's default traceback messages upon error.


return _get_interfaces(_system.models)<EOL>

Returns a dict that maps model names to models.


Restore IPython' default error message.

    Bring back IPython's default traceback message upon error for debugging.


def tuplize_key(obj, key, remove_extra=False):

paramlen = len(obj.formula.parameters)<EOL>if isinstance(key, str):<EOL><INDENT>key = (key,)<EOL><DEDENT>elif not isinstance(key, Sequence):<EOL><INDENT>key = (key,)<EOL><DEDENT>if not remove_extra:<EOL><INDENT>return key<EOL><DEDENT>else:<EOL><INDENT>arglen = len(key)<EOL>if arglen:<EOL><INDENT>return key[: min(arglen, paramlen)]<EOL><DEDENT>else:<EOL><INDENT>return key<EOL><DEDENT><DEDENT>


obj = node[OBJ]<EOL>key = node[KEY]<EOL>boundargs = obj.formula.signature.bind(*key)<EOL>boundargs.apply_defaults()<EOL>return boundargs.arguments<EOL>

Return an ordered mapping from params to args


if model is None:<EOL><INDENT>model = mx.cur_model()<EOL><DEDENT>view = get_modeltree(model)<EOL>app = QApplication.instance()<EOL>if not app:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>view.show()<EOL>app.exec_()<EOL>

Display the model tree window.

    Args:
        model: :class:`Model <modelx.core.model.Model>` object.
            Defaults to the current model.

    Warnings:
        For this function to work with Spyder, *Graphics backend* option
        of Spyder must be set to *inline*.


if model is None:<EOL><INDENT>model = mx.cur_model()<EOL><DEDENT>treemodel = ModelTreeModel(model._baseattrs)<EOL>view = QTreeView()<EOL>view.setModel(treemodel)<EOL>view.setWindowTitle("<STR_LIT>" % model.name)<EOL>view.setAlternatingRowColors(True)<EOL>return view<EOL>

Get QTreeView object containing the model tree.

    Args:
        model: :class:`Model <modelx.core.model.Model>` object.
            Defaults to the current model.


return self.predecessors(node)<EOL>


if n == <NUM_LIT:1>:<EOL><INDENT>return <NUM_LIT:1> / (<NUM_LIT:1> + interest_rate)<EOL><DEDENT>else:<EOL><INDENT>return pv_payments(n - <NUM_LIT:1>) + <NUM_LIT:1> / (<NUM_LIT:1> + interest_rate) ** n<EOL><DEDENT>

Present value of repayments of 1 for n years.


def __init__(self, hostname: str = '<STR_LIT>',<EOL>port: int = <NUM_LIT>, retries: int = <NUM_LIT:10>,<EOL>log_errors: bool = False) -> None:

self.hostname = hostname<EOL>self.port = port<EOL>self.retries = retries<EOL>self.log_errors = log_errors<EOL>self._log_fp = Path(tempfile.gettempdir(), '<STR_LIT>')<EOL>if self._log_fp.is_file():<EOL><INDENT>self._log_fp.open('<STR_LIT:w>').close()<EOL><DEDENT>

:param hostname: The IP address of the TweeboParser API server.
:param port: The Port that the TweeboParser API server is attached to.
:param retries: Number of times to retry json decoding the
                returned data.


if self.log_errors:<EOL><INDENT>with self._log_fp.open('<STR_LIT>') as log_file:<EOL><INDENT>log_file.write(f'<STR_LIT>')<EOL><DEDENT><DEDENT>

Given some error text it will log the text if self.log_errors is True

:param text: Error text to log


problematic_hyphens = [(r'<STR_LIT>', r'<STR_LIT>'),<EOL>(r'<STR_LIT>', '<STR_LIT>'),<EOL>(r'<STR_LIT>', '<STR_LIT>')]<EOL>for problem_case in problematic_hyphens:<EOL><INDENT>self._regex_replacement(*problem_case)<EOL><DEDENT>

Transform hyphens to various kinds of dashes


not_sentence_end_chars = ['<STR_LIT:U+0020>']<EOL>abbreviations = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>']<EOL>titles = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for abbrev in abbreviations:<EOL><INDENT>for x in not_sentence_end_chars:<EOL><INDENT>self._str_replacement(abbrev + x, abbrev + '<STR_LIT>')<EOL><DEDENT><DEDENT>for title in titles:<EOL><INDENT>for x in not_sentence_end_chars:<EOL><INDENT>self._str_replacement(title + x, title + '<STR_LIT>')<EOL><DEDENT><DEDENT>

Fix common spacing errors caused by LaTeX's habit
        of using an inter-sentence space after any full stop.


return self.data<EOL>


return LatexFixer(*args, **kwargs).tostring()<EOL>

Transform a unicode string into another more compatible with latex,
    fixing some common typographical errors


pass<EOL>

Fix errors where inter-sentence spacing
        is not used after after a word ending with a capital letter.


for f, callback, transferred, key, ov in events:<EOL><INDENT>try:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(callback))<EOL>value = callback(transferred, key, ov)<EOL><DEDENT>except OSError:<EOL><INDENT>self._logger.warning('<STR_LIT>', exc_info=sys.exc_info())<EOL><DEDENT>else:<EOL><INDENT>f.set_result(value)<EOL><DEDENT><DEDENT>


raise NotImplementedError<EOL>

Implement abstract method even though we don't need it.


if isinstance(fileobj, int):<EOL><INDENT>fd = fileobj<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>fd = int(fileobj.fileno())<EOL><DEDENT>except (AttributeError, TypeError, ValueError) as ex:<EOL><INDENT>raise ValueError("<STR_LIT>".format(fileobj)) from ex<EOL><DEDENT><DEDENT>if fd < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>".format(fd))<EOL><DEDENT>return fd<EOL>

Return a file descriptor from a file object.

Parameters:
fileobj -- file object or file descriptor

Returns:
corresponding file descriptor

Raises:
ValueError if the object is invalid


try:<EOL><INDENT>return _fileobj_to_fd(fileobj)<EOL><DEDENT>except ValueError:<EOL><INDENT>for key in self._fd_to_key.values():<EOL><INDENT>if key.fileobj is fileobj:<EOL><INDENT>return key.fd<EOL><DEDENT><DEDENT>raise<EOL><DEDENT>

Return a file descriptor from a file object.

        This wraps _fileobj_to_fd() to do an exhaustive search in case
        the object is invalid but we still have it in our map.  This
        is used by unregister() so we can unregister an object that
        was previously registered even if it is closed.  It is also
        used by _SelectorMapping.


self._logger.debug('<STR_LIT>'.format(key, mask))<EOL>fileobj, (reader, writer) = key.fileobj, key.data<EOL>if mask & selectors.EVENT_READ and reader is not None:<EOL><INDENT>if reader._cancelled:<EOL><INDENT>self.remove_reader(fileobj)<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(reader))<EOL>reader._run()<EOL><DEDENT><DEDENT>if mask & selectors.EVENT_WRITE and writer is not None:<EOL><INDENT>if writer._cancelled:<EOL><INDENT>self.remove_writer(fileobj)<EOL><DEDENT>else:<EOL><INDENT>self._logger.debug('<STR_LIT>'.format(writer))<EOL>writer._run()<EOL><DEDENT><DEDENT>


self._logger.debug('<STR_LIT>')<EOL>message = context.get('<STR_LIT:message>')<EOL>if not message:<EOL><INDENT>message = '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>exception = context['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>exc_info = False<EOL><DEDENT>else:<EOL><INDENT>exc_info = (type(exception), exception, exception.__traceback__)<EOL><DEDENT>log_lines = [message]<EOL>for key in [k for k in sorted(context) if k not in {'<STR_LIT:message>', '<STR_LIT>'}]:<EOL><INDENT>log_lines.append('<STR_LIT>'.format(key, context[key]))<EOL><DEDENT>self.__log_error('<STR_LIT:\n>'.join(log_lines), exc_info=exc_info)<EOL>

Handle exceptions.

        This is the default exception handler.

        This is called when an exception occurs and no exception
        handler is set, and can be called by a custom exception
        handler that wants to defer to the default behavior.

        context parameter has the same meaning as in
        `call_exception_handler()`.


self.__is_running = True<EOL>self._before_run_forever()<EOL>try:<EOL><INDENT>self._logger.debug('<STR_LIT>')<EOL>rslt = self.__app.exec_()<EOL>self._logger.debug('<STR_LIT>'.format(rslt))<EOL>return rslt<EOL><DEDENT>finally:<EOL><INDENT>self._after_run_forever()<EOL>self.__is_running = False<EOL><DEDENT>


def call_at(self, when, callback, *args, context=None):

return self.call_later(when - self.time(), callback, *args, context=context)<EOL>

Register callback to be invoked at a certain time.


def add_writer(self, fd, callback, *args):

self._check_closed()<EOL>try:<EOL><INDENT>existing = self._write_notifiers[fd]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>existing.setEnabled(False)<EOL>existing.activated.disconnect()<EOL><DEDENT>notifier = QtCore.QSocketNotifier(fd, QtCore.QSocketNotifier.Write)<EOL>notifier.setEnabled(True)<EOL>self._logger.debug('<STR_LIT>'.format(fd))<EOL>notifier.activated.connect(<EOL>lambda: self.__on_notifier_ready(<EOL>self._write_notifiers, notifier, fd, callback, args)  <EOL>)<EOL>self._write_notifiers[fd] = notifier<EOL>

Register a callback for when a file descriptor is ready for writing.


if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>'.format(fd))<EOL>try:<EOL><INDENT>notifier = self._write_notifiers.pop(fd)<EOL><DEDENT>except KeyError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>notifier.setEnabled(False)<EOL>return True<EOL><DEDENT>


return self.__is_running<EOL>

Return True if the event loop is running, False otherwise.


if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>'.format(fd))<EOL>try:<EOL><INDENT>notifier = self._read_notifiers.pop(fd)<EOL><DEDENT>except KeyError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>notifier.setEnabled(False)<EOL>return True<EOL><DEDENT>


if self.is_running():<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if self.is_closed():<EOL><INDENT>return<EOL><DEDENT>self._logger.debug('<STR_LIT>')<EOL>if self.__default_executor is not None:<EOL><INDENT>self.__default_executor.shutdown()<EOL><DEDENT>super().close()<EOL>self._timer.stop()<EOL>self.__app = None<EOL>for notifier in itertools.chain(self._read_notifiers.values(), self._write_notifiers.values()):<EOL><INDENT>notifier.setEnabled(False)<EOL><DEDENT>self._read_notifiers = None<EOL>self._write_notifiers = None<EOL>

Release all resources used by the event loop.

The loop cannot be restarted after it has been closed.


@functools.wraps(fn)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>f = asyncio.ensure_future(fn(*args, **kwargs))<EOL>while not f.done():<EOL><INDENT>QApplication.instance().processEvents()<EOL><DEDENT><DEDENT>return wrapper<EOL>

Allow to run async code before application is closed.


video = Video()<EOL>video._embed_code = self.get_embed_code(node)<EOL>video._embed_type = self.get_embed_type(node)<EOL>video._width = self.get_width(node)<EOL>video._height = self.get_height(node)<EOL>video._src = self.get_src(node)<EOL>video._provider = self.get_provider(video.src)<EOL>return video<EOL>

Create a video object from a video embed


def update_score(self, node, add_to_score):

current_score = <NUM_LIT:0><EOL>score_string = self.parser.getAttribute(node, '<STR_LIT>')<EOL>if score_string:<EOL><INDENT>current_score = int(score_string)<EOL><DEDENT>new_score = current_score + int(add_to_score)<EOL>self.parser.setAttribute(node, "<STR_LIT>", str(new_score))<EOL>

adds a score to the gravityScore Attribute we put on divs
we'll get the current score then add the score
we're passing in to the current


def update_node_count(self, node, add_to_count):

current_score = <NUM_LIT:0><EOL>count_string = self.parser.getAttribute(node, '<STR_LIT>')<EOL>if count_string:<EOL><INDENT>current_score = int(count_string)<EOL><DEDENT>new_score = current_score + add_to_count<EOL>self.parser.setAttribute(node, "<STR_LIT>", str(new_score))<EOL>

\
        stores how many decent nodes are under a parent node


base = <NUM_LIT><EOL>paragraphs_number = <NUM_LIT:0><EOL>paragraphs_score = <NUM_LIT:0><EOL>nodes_to_check = self.parser.getElementsByTag(top_node, tag='<STR_LIT:p>')<EOL>for node in nodes_to_check:<EOL><INDENT>text_node = self.parser.getText(node)<EOL>word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text_node)<EOL>high_link_density = self.is_highlink_density(node)<EOL>if word_stats.get_stopword_count() > <NUM_LIT:2> and not high_link_density:<EOL><INDENT>paragraphs_number += <NUM_LIT:1><EOL>paragraphs_score += word_stats.get_stopword_count()<EOL><DEDENT><DEDENT>if paragraphs_number > <NUM_LIT:0>:<EOL><INDENT>base = paragraphs_score // paragraphs_number<EOL><DEDENT>return base<EOL>

we could have long articles that have tons of paragraphs
so if we tried to calculate the base score against
the total text score of those paragraphs it would be unfair.
So we need to normalize the score based on the average scoring
of the paragraphs within the top node.
For example if our total score of 10 paragraphs was 1000
but each had an average value of 100 then 100 should be our base.


return self.get_node_gravity_score(node) or <NUM_LIT:0><EOL>

returns the gravityScore as an integer from this node


links = self.parser.getElementsByTag(element, tag='<STR_LIT:a>')<EOL>if not links:<EOL><INDENT>return False<EOL><DEDENT>text = self.parser.getText(element)<EOL>words = text.split('<STR_LIT:U+0020>')<EOL>words_number = float(len(words))<EOL>link_text_parts = []<EOL>for link in links:<EOL><INDENT>link_text_parts.append(self.parser.getText(link))<EOL><DEDENT>link_text = '<STR_LIT>'.join(link_text_parts)<EOL>link_words = link_text.split('<STR_LIT:U+0020>')<EOL>number_of_link_words = float(len(link_words))<EOL>number_of_links = float(len(links))<EOL>link_divisor = float(number_of_link_words / words_number)<EOL>score = float(link_divisor * number_of_links)<EOL>if score >= <NUM_LIT:1.0>:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

checks the density of links within a node,
is there not much text and most of it contains linky shit?
if so it's no good


<EOL>if self.config.use_meta_language:<EOL><INDENT>if self.article.meta_lang:<EOL><INDENT>return self.article.meta_lang[:<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return self.config.target_language<EOL>

Returns the language is by the article or
the configuration language


para = "<STR_LIT:p>"<EOL>steps_away = <NUM_LIT:0><EOL>minimum_stopword_count = <NUM_LIT:5><EOL>max_stepsaway_from_node = <NUM_LIT:3><EOL>nodes = self.walk_siblings(node)<EOL>for current_node in nodes:<EOL><INDENT>current_node_tag = self.parser.getTag(current_node)<EOL>if current_node_tag == para:<EOL><INDENT>if steps_away >= max_stepsaway_from_node:<EOL><INDENT>return False<EOL><DEDENT>para_text = self.parser.getText(current_node)<EOL>word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(para_text)<EOL>if word_stats.get_stopword_count() > minimum_stopword_count:<EOL><INDENT>return True<EOL><DEDENT>steps_away += <NUM_LIT:1><EOL><DEDENT><DEDENT>return False<EOL>

\
        alot of times the first paragraph might be the caption under an image
        so we'll want to make sure if we're going to boost a parent node that
        it should be connected to other paragraphs,
        at least for the first n paragraphs so we'll want to make sure that
        the next sibling is a paragraph and has at
        least some substatial weight to it


if '<STR_LIT:image>' in self.article.opengraph:<EOL><INDENT>return self.get_image(self.article.opengraph["<STR_LIT:image>"],<EOL>extraction_type='<STR_LIT>')<EOL><DEDENT>elif (self.article.schema and '<STR_LIT:image>' in self.article.schema and<EOL>"<STR_LIT:url>" in self.article.schema["<STR_LIT:image>"]):<EOL><INDENT>return self.get_image(self.article.schema["<STR_LIT:image>"]["<STR_LIT:url>"],<EOL>extraction_type='<STR_LIT>')<EOL><DEDENT>return None<EOL>

\
        checks to see if we were able to find the image via known schemas:

        Supported Schemas
         - Open Graph
         - schema.org


src = self.parser.getAttribute(image_node, attr='<STR_LIT:src>')<EOL>if not src:<EOL><INDENT>return False<EOL><DEDENT>if self.badimages_names_re.search(src):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

\
        will check the image src against a list
        of bad image files we know of like buttons, etc...


o = urlparse(src)<EOL>if o.netloc != '<STR_LIT>':<EOL><INDENT>return o.geturl()<EOL><DEDENT>return urljoin(self.article.final_url, src)<EOL>

\
        This method will take an image path and build
        out the absolute path to that image
        * using the initial url we crawled
          so we can find a link to the image
          if they use relative urls like ../myimage.jpg


node = self.article.raw_doc<EOL>meta = self.parser.getElementsByTag(node, tag='<STR_LIT>', attr='<STR_LIT>', value='<STR_LIT>')<EOL>for item in meta:<EOL><INDENT>src = self.parser.getAttribute(item, attr='<STR_LIT>')<EOL>if src:<EOL><INDENT>return self.get_image(src, extraction_type='<STR_LIT>')<EOL><DEDENT><DEDENT>return None<EOL>

\
        checks to see if we were able to
        find open link_src on this page


good_images = []<EOL>for image in images:<EOL><INDENT>if self.is_valid_filename(image):<EOL><INDENT>good_images.append(image)<EOL><DEDENT><DEDENT>return good_images if len(good_images) > <NUM_LIT:0> else None<EOL>

\
        takes a list of image elements
        and filters out the ones with bad names


return ImageUtils.store_image(self.fetcher, self.article.link_hash, src, self.config)<EOL>

\
        returns the bytes of the image file on disk


kwargs = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>', '<STR_LIT:value>': '<STR_LIT>'}<EOL>meta = self.parser.getElementsByTag(self.article.doc, **kwargs)<EOL>if meta:<EOL><INDENT>favicon = self.parser.getAttribute(meta[<NUM_LIT:0>], '<STR_LIT>')<EOL>return favicon<EOL><DEDENT>return '<STR_LIT>'<EOL>

Extract the favicon from a website
http://en.wikipedia.org/wiki/Favicon
<link rel="shortcut icon" type="image/png" href="favicon.png" />
<link rel="icon" type="image/png" href="favicon.png" />


return self.get_meta_content("<STR_LIT>")<EOL>

if the article has meta keywords set in the source, use that


meta = self.parser.css_select(self.article.doc, meta_name)<EOL>content = None<EOL>if meta is not None and len(meta) > <NUM_LIT:0>:<EOL><INDENT>content = self.parser.getAttribute(meta[<NUM_LIT:0>], '<STR_LIT:content>')<EOL><DEDENT>if content:<EOL><INDENT>return content.strip()<EOL><DEDENT>return '<STR_LIT>'<EOL>

Extract a given meta content form document


return self._extraction_type<EOL>

str: The extraction type used

            Note:
                Read only


all_nodes = self.parser.getElementsByTags(self.get_top_node(), ['<STR_LIT:*>'])<EOL>all_nodes.reverse()<EOL>for elm in all_nodes:<EOL><INDENT>tag = self.parser.getTag(elm)<EOL>text = self.parser.getText(elm)<EOL>stop_words = self.stopwords_class(language=self.get_language()).get_stopword_count(text)<EOL>if ((tag != '<STR_LIT>' or text != '<STR_LIT>') and stop_words.get_stopword_count() < <NUM_LIT:3> and<EOL>len(self.parser.getElementsByTag(elm, tag='<STR_LIT:object>')) == <NUM_LIT:0> and<EOL>len(self.parser.getElementsByTag(elm, tag='<STR_LIT>')) == <NUM_LIT:0>):<EOL><INDENT>self.parser.remove(elm)<EOL><DEDENT>else:<EOL><INDENT>trimmed = self.parser.getText(elm)<EOL>if trimmed.startswith("<STR_LIT:(>") and trimmed.endswith("<STR_LIT:)>"):<EOL><INDENT>self.parser.remove(elm)<EOL><DEDENT><DEDENT><DEDENT>

\
        remove paragraphs that have less than x number of words,
        would indicate that it's some sort of link


self.parser.stripTags(self.get_top_node(), '<STR_LIT:a>')<EOL>

\
        cleans up and converts any nodes that
        should be considered text into text


for elm in self.parser.getElementsByTag(self.top_node, tag='<STR_LIT>'):<EOL><INDENT>elm.text = r'<STR_LIT>'.format(elm.text)<EOL><DEDENT>


def smart_unicode(string, encoding='<STR_LIT:utf-8>', strings_only=False, errors='<STR_LIT:strict>'):

<EOL>return force_unicode(string, encoding, strings_only, errors)<EOL>

Returns a unicode object representing 's'. Treats bytestrings using the
'encoding' codec.

If strings_only is True, don't convert (some) non-string-like objects.


return self._top_image<EOL>

Image: The top image object that likely represents the article

            Returns:
                Image: See more information on the goose3.Image class
            Note:
                Read only


return self._meta_lang<EOL>

str: Contents of the meta-lang field from the HTML source

            Note:
                Read only


return self._authors<EOL>

list(str): A listing of authors as parsed from the meta tags

            Note:
                Read only


return self._opengraph<EOL>

dict: All opengraph tag data

            Note:
                Read only


return self._meta_encoding<EOL>

str: Contents of the encoding/charset field from the HTML source

            Note:
                Read only


return self._domain<EOL>

str: Domain of the article parsed

            Note:
                Read only


return self._links<EOL>

list(str): A listing of URL links within the article

            Note:
                Read only


return self._link_hash<EOL>

str: The MD5 of the final url to be used for various identification tasks

            Note:
                Read only


data = {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:description>": self.meta_description,<EOL>"<STR_LIT>": self.meta_lang,<EOL>"<STR_LIT>": self.meta_keywords,<EOL>"<STR_LIT>": self.meta_favicon,<EOL>"<STR_LIT>": self.canonical_link,<EOL>"<STR_LIT>": self.meta_encoding<EOL>},<EOL>"<STR_LIT:image>": None,<EOL>"<STR_LIT>": self.domain,<EOL>"<STR_LIT:title>": self.title,<EOL>"<STR_LIT>": self.cleaned_text,<EOL>"<STR_LIT>": self.opengraph,<EOL>"<STR_LIT>": self.tags,<EOL>"<STR_LIT>": self.tweets,<EOL>"<STR_LIT>": [],<EOL>"<STR_LIT>": self.links,<EOL>"<STR_LIT>": self.authors,<EOL>"<STR_LIT>": self.publish_date<EOL>}<EOL>if self.top_image is not None:<EOL><INDENT>data['<STR_LIT:image>'] = {<EOL>'<STR_LIT:url>': self.top_image.src,<EOL>'<STR_LIT:width>': self.top_image.width,<EOL>'<STR_LIT>': self.top_image.height,<EOL>'<STR_LIT:type>': '<STR_LIT:image>'<EOL>}<EOL><DEDENT>for movie in self.movies:<EOL><INDENT>data['<STR_LIT>'].append({<EOL>'<STR_LIT>': movie.embed_type,<EOL>'<STR_LIT>': movie.provider,<EOL>'<STR_LIT:width>': movie.width,<EOL>'<STR_LIT>': movie.height,<EOL>'<STR_LIT>': movie.embed_code,<EOL>'<STR_LIT:src>': movie.src,<EOL>})<EOL><DEDENT>return data<EOL>

dict: The summation of all data available about the extracted article

            Note:
                Read only


return self._schema<EOL>

dict: All schema tag data

            Note:
                Read only


return self._meta_description<EOL>

str: Contents of the meta-description field from the HTML source

            Note:
                Read only


return self._doc<EOL>

etree: lxml document that is being processed

            Note:
                Read only


return self._tweets<EOL>

list(str): A listing of embeded tweets in the article

            Note:
                Read only


return self._top_node<EOL>

etree: The top Element that is a candidate for the main body of the article

            Note:
                Read only


return self._meta_favicon<EOL>

str: Contents of the meta-favicon field from the HTML source

            Note:
                Read only


return self._final_url<EOL>

str: The URL that was used to pull and parsed; `None` if raw_html was used
            and no url element was found.

            Note:
                Read only


return self._http_timeout<EOL>

float: The time delay to pass to `requests` to wait for the response
            in seconds

            Note:
                Defaults to 30.0


return self._http_headers<EOL>

dict: Custom headers to pass directly to the supporting `requests` object

            See Also:
                `Requests Custom Headers <http://docs.python-requests.org/en/master/user/quickstart/#custom-headers>`__


return self._parser_class<EOL>

str: The key of the parser to use

            Note:
                Defaults to `lxml`


return self._parse_headers<EOL>

bool: Specify if headers should be pulled or not in the cleaned_text
            output

            Note:
                Defaults to `True`


self._browser_user_agent = val<EOL>


self._images_min_bytes = int(val)<EOL>


self._parse_headers = bool(val)<EOL>


return self._images_min_bytes<EOL>

int: Minimum number of bytes for an image to be evaluated to be the
            main image of the site

            Note:
                Defaults to 4500 bytes


self._debug = bool(val)<EOL>


return self._use_meta_language<EOL>

bool: Determine if language should be extracted from the meta tags
            or not. If this is set to `False` then the target_language will be
            used. Also, if extraction fails then the target_language will be
            utilized.

            Note:
                Defaults to `True`


self._local_storage_path = val<EOL>


return self._http_auth<EOL>

tuple: Authentication class and information to pass to the requests
            library

            See Also:
                `Requests Authentication <http://docs.python-requests.org/en/master/user/authentication/>`__


self._http_proxies = val<EOL>