#NEWLINE# def record_corpus( base_dir , encoding = 'utf8' , ignore = [ '.json' , '.log' , '.pickle' ] , nltk_stop = True , stop_freq = 1 , add_stop = None , decode = False , verbose = 1 ) : #NEWLINE# #INDENT# "\n    `record_corpus` is a convenience function for generating Corpus\n    objects from a directory of plain text files.\n\n    It will also strip punctuation and arabic numerals outside the\n    range 1-29. All letters are made lowercase.\n\n    :param base_dir: Directory containing a recordections of books\n        which contain pages as plain-text files.\n    :type base_dir: string-like\n    \n    :param encoding: A string indicating the file encoding or 'detect',\n        in which case `chardet` is used to automatically guess the encoding.\n        Default is `utf8`.\n    :type encoding: string, optional\n    \n    :param ignore: The list containing suffixes of files to be filtered.\n        The suffix strings are normally file types. Default is ['.json',\n        '.log','.pickle'].\n    :type ignore: list of strings, optional\n\n    :param nltk_stop: If `True` then the corpus object is masked \n        using the NLTK English stop words. Default is `False`.\n    :type nltk_stop: boolean, optional\n    \n    :param stop_freq: The upper bound for a word to be masked on \n        the basis of its recordection frequency. Default is 1.\n    :type stop_freq: int, optional\n    \n    :param add_stop: A list of stop words. Default is `None`.\n    :type add_stop: array-like, optional\n    \n    :param decode: If `True` then unicode characters are converted to\n        ASCII. Default is `False`.\n    :type decode: boolean, optional\n\n    :param verbose: Verbosity level. 1 prints a progress bar.\n    :type verbose: int, default 1 \n\n    :returns: c : a Corpus object\n        Contains the tokenized corpus built from the plain-text files\n        in `record_dir` corpus. Document tokens are named `documents`.\n    " #NEWLINE# records = [ ] #NEWLINE# record_names = os . listdir ( base_dir ) #NEWLINE# record_names = filter_by_suffix ( record_names , ignore ) #NEWLINE# record_names . sort ( ) #NEWLINE# record_book_names = [ ] #NEWLINE# for record_dir in record_names : #NEWLINE# #INDENT# books = [ ] #NEWLINE# book_names = os . listdir ( record_dir ) #NEWLINE# book_names = filter_by_suffix ( book_names , ignore ) #NEWLINE# book_names . sort ( ) #NEWLINE# for book_name in book_names : #NEWLINE# #INDENT# pages = [ ] #NEWLINE# book_path = os . path . join ( record_dir , book_name ) #NEWLINE# page_names = os . listdir ( book_path ) #NEWLINE# page_names = filter_by_suffix ( page_names , ignore ) #NEWLINE# page_names . sort ( ) #NEWLINE# for page_name in page_names : #NEWLINE# #INDENT# page_file = ( ( book_name + '/' ) + page_name ) #NEWLINE# page_name = os . path . join ( book_path , page_name ) #NEWLINE# if ( encoding == 'detect' ) : #NEWLINE# #INDENT# encoding = detect_encoding ( page_name ) #NEWLINE# #UNINDENT# if decode : #NEWLINE# #INDENT# with open ( page_name , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# pages . append ( ( unidecode ( f . read ( ) ) , page_file ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# with open ( page_name , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# pages . append ( ( f . read ( ) , page_file ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# books . append ( pages ) #NEWLINE# #UNINDENT# record_book_names . append ( book_names ) #NEWLINE# records . append ( books ) #NEWLINE# #UNINDENT# ( words , tok ) = record_tokenize ( records , record_names , record_book_names ) #NEWLINE# ( names , data ) = zip ( * tok . items ( ) ) #NEWLINE# c = Corpus ( words , context_data = data , context_types = names ) #NEWLINE# if ( nltk_stop or stop_freq or add_stop ) : #NEWLINE# #INDENT# c = apply_stoplist ( c , nltk_stop = nltk_stop , freq = stop_freq , add_stop = add_stop ) #NEWLINE# #UNINDENT# return c