#NEWLINE# def toy_corpus( plain_corpus , is_filename = False , encoding = 'utf8' , nltk_stop = False , stop_freq = 0 , add_stop = None , decode = False , metadata = None , autolabel = False , tokenizer = word_tokenize , simple = False ) : #NEWLINE# #INDENT# "\n    `toy_corpus` is a convenience function for generating Corpus\n    objects from a given string or a single file.\n\n    `toy_corpus` will perform both word and document-level\n    tokenization. It will also strip punctuation and arabic numerals\n    outside the range 1-29. All letters are made lowercase.\n\n    Document tokens are delimited by two or more line breaks. E.g.,\n\n        <document 0>\n\n        <document 1>\n\n        ...\n\n        <document n>\n\n    where <document i> is any chunk of text to be tokenized by word.\n\n    :param plain_corpus: String containing a plain-text corpus or a \n        filename of a file containing one.\n    :type plain_corpus: string-like\n    \n    :param is_filename: If `True` then `plain_corpus` is treated like\n        a filename. Otherwise, `plain_corpus` is presumed to contain \n        the corpus. Default is `False`.\n    :type is_filename: boolean, optional\n\n    :param encoding: A string indicating the file encoding or 'detect',\n        in which case `chardet` is used to automatically guess the encoding.\n        Default is `utf8`.\n    :type encoding: string, optional\n    \n    :param nltk_stop: If `True` then the corpus object is masked using\n        the NLTK English stop words. Default is `False`.\n    :type nltk_stop: boolean, optional\n\n    :param stop_freq: The upper bound for a word to be masked on the basis of its\n        collection frequency. Default is 0.\n    :type stop_freq: int, optional\n\n    :param add_stop: A list of stop words. Default is `None`.\n    :type add_stop: array-like, optional\n    \n    :param decode: If `True` then unicode characters are converted to\n        ASCII. Default is `False`.\n    :type decode: boolean, optional\n\n    :param metadata: A list of strings providing metadata about the documents. If\n        provided, must have length equal to the number of documents.\n        Default is `None`.\n    :type metadata: array-like, optional\n    \n    :param autolabel: A boolean specifying whether to automatically label\n        documents by position in file. Default is False\n    :type metadata: boolean, optional\n    \n    :param tokenizer: word tokenization function. Defaults to `vsm.extensions.corpusbuilders.util.word_tokenize`.\n    :type tokenizer: lambda s -> tokens\n    \n    :returns: c : a Corpus object\n        Contains the tokenized corpus built from the input plain-text\n        corpus. Document tokens are named `documents`.\n\n    :See Also: :class:`vsm.corpus.Corpus`, \n        :meth:`vsm.corpus.util.paragraph_tokenize`, \n        :meth:`vsm.corpus.util.apply_stoplist`\n    " #NEWLINE# if is_filename : #NEWLINE# #INDENT# if ( encoding == 'detect' ) : #NEWLINE# #INDENT# encoding = detect_encoding ( plain_corpus ) #NEWLINE# #UNINDENT# with open ( plain_corpus , 'rb' , encoding = encoding ) as f : #NEWLINE# #INDENT# plain_corpus = f . read ( ) #NEWLINE# #UNINDENT# #UNINDENT# if decode : #NEWLINE# #INDENT# plain_corpus = unidecode ( plain_corpus ) #NEWLINE# #UNINDENT# docs = paragraph_tokenize ( plain_corpus ) #NEWLINE# docs = [ tokenizer ( d ) for d in docs ] #NEWLINE# corpus = sum ( docs , [ ] ) #NEWLINE# tok = np . cumsum ( np . array ( [ len ( d ) for d in docs ] ) ) #NEWLINE# if ( ( not metadata ) and autolabel ) : #NEWLINE# #INDENT# metadata = [ 'Document {0}' . format ( i ) for i in range ( len ( tok ) ) ] #NEWLINE# #UNINDENT# if metadata : #NEWLINE# #INDENT# if ( not ( len ( metadata ) == len ( tok ) ) ) : #NEWLINE# #INDENT# msg = 'Metadata mismatch: metadata length is {0} and numberof documents is {1}' . format ( len ( metadata ) , len ( tok ) ) #NEWLINE# raise Exception ( msg ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# dtype = [ ( 'idx' , np . array ( tok ) . dtype ) , ( 'document_label' , np . array ( metadata ) . dtype ) ] #NEWLINE# tok = np . array ( zip ( tok , metadata ) , dtype = dtype ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# dtype = [ ( 'idx' , np . array ( tok ) . dtype ) ] #NEWLINE# tok = np . array ( [ ( i , ) for i in tok ] , dtype = dtype ) #NEWLINE# #UNINDENT# c = Corpus ( corpus , context_data = [ tok ] , context_types = [ 'document' ] ) #NEWLINE# if ( nltk_stop or stop_freq or add_stop ) : #NEWLINE# #INDENT# c = apply_stoplist ( c , nltk_stop = nltk_stop , freq = stop_freq , add_stop = add_stop ) #NEWLINE# #UNINDENT# return c