#NEWLINE# def dir_corpus( plain_dir , chunk_name = 'article' , encoding = 'utf8' , paragraphs = True , ignore = [ '.json' , '.log' , '.pickle' ] , nltk_stop = True , stop_freq = 1 , add_stop = None , decode = False , verbose = 1 , simple = False , tokenizer = word_tokenize ) : #NEWLINE# #INDENT# "\n    `dir_corpus` is a convenience function for generating Corpus\n    objects from a directory of plain text files.\n\n    `dir_corpus` will retain file-level tokenization and perform\n    sentence and word tokenizations. Optionally, it will provide\n    paragraph-level tokenizations.\n\n    It will also strip punctuation and arabic numerals outside the\n    range 1-29. All letters are made lowercase.\n\n    :param plain_dir: String containing directory containing a \n        plain-text corpus.\n    :type plain_dir: string-like\n    \n    :param chunk_name: The name of the tokenization corresponding \n        to individual files. For example, if the files are pages \n        of a book, one might set `chunk_name` to `pages`. Default \n        is `articles`.\n    :type chunk_name: string-like, optional\n    \n    :param encoding: A string indicating the file encoding or 'detect',\n        in which case `chardet` is used to automatically guess the encoding.\n        Default is `utf8`.\n    :type encoding: string, optional\n    \n    :param paragraphs: If `True`, a paragraph-level tokenization \n        is included. Defaults to `True`.\n    :type paragraphs: boolean, optional\n    \n    :param ignore: The list containing suffixes of files to be filtered.\n        The suffix strings are normally file types. Default is ['.json',\n        '.log','.pickle'].\n    :type ignore: list of strings, optional\n\n    :param nltk_stop: If `True` then the corpus object is masked \n        using the NLTK English stop words. Default is `False`.\n    :type nltk_stop: boolean, optional\n    \n    :param stop_freq: The upper bound for a word to be masked on \n        the basis of its collection frequency. Default is 1.\n    :type stop_freq: int, optional\n    \n    :param add_stop: A list of stop words. Default is `None`.\n    :type add_stop: array-like, optional\n    \n    :param decode: If `True` then unicode characters are converted to\n        ASCII. Default is `False`.\n    :type decode: boolean, optional\n\n    :param verbose: Verbosity level. 1 prints a progress bar.\n    :type verbose: int, default 1 \n\n    :returns: c : a Corpus object\n        Contains the tokenized corpus built from the input plain-text\n        corpus. Document tokens are named `documents`.\n    \n    :See Also: :class:`vsm.corpus.Corpus`, \n            :meth:`dir_tokenize`, \n            :meth:`vsm.corpus.util.apply_stoplist`\n    " #NEWLINE# chunks = [ ] #NEWLINE# filenames = os . listdir ( plain_dir ) #NEWLINE# filenames = filter_by_suffix ( filenames , ignore ) #NEWLINE# filenames . sort ( ) #NEWLINE# for filename in filenames : #NEWLINE# #INDENT# filename = os . path . join ( plain_dir , filename ) #NEWLINE# if ( encoding == 'detect' ) : #NEWLINE# #INDENT# encoding = detect_encoding ( filename ) #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# if decode : #NEWLINE# #INDENT# with open ( filename , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# chunks . append ( unidecode ( f . read ( ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# with open ( filename , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# chunks . append ( f . read ( ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# except UnicodeDecodeError : #NEWLINE# #INDENT# encoding = detect_encoding ( filename ) #NEWLINE# if decode : #NEWLINE# #INDENT# with open ( filename , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# chunks . append ( unidecode ( f . read ( ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# with open ( filename , mode = 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# chunks . append ( f . read ( ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# ( words , tok ) = dir_tokenize ( chunks , filenames , chunk_name = chunk_name , paragraphs = paragraphs , verbose = verbose , simple = simple , tokenizer = tokenizer ) #NEWLINE# ( names , data ) = zip ( * tok . items ( ) ) #NEWLINE# c = Corpus ( words , context_data = data , context_types = names ) #NEWLINE# if ( nltk_stop or stop_freq or add_stop ) : #NEWLINE# #INDENT# c = apply_stoplist ( c , nltk_stop = nltk_stop , freq = stop_freq , add_stop = add_stop ) #NEWLINE# #UNINDENT# return c