#NEWLINE# def corpus_from_strings( strings , metadata = [ ] , decode = False , nltk_stop = True , stop_freq = 0 , add_stop = None , tokenizer = word_tokenize ) : #NEWLINE# #INDENT# '\n    Takes a list of strings and returns a Corpus object whose document\n    tokens are the strings.\n    :param tokenizer: word tokenization function. Defaults to `vsm.extensions.corpusbuilders.util.word_tokenize`.\n    :type tokenizer: lambda s -> tokens\n\n    ' #NEWLINE# if decode : #NEWLINE# #INDENT# for i in xrange ( len ( strings ) ) : #NEWLINE# #INDENT# if isinstance ( strings [ i ] , unicode ) : #NEWLINE# #INDENT# strings [ i ] = unidecode ( strings [ i ] ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# documents = [ tokenizer ( s ) for s in strings ] #NEWLINE# corpus = sum ( documents , [ ] ) #NEWLINE# indices = np . cumsum ( [ len ( d ) for d in documents ] ) #NEWLINE# del documents #NEWLINE# if ( len ( metadata ) == 0 ) : #NEWLINE# #INDENT# metadata = [ 'document_{0}' . format ( i ) for i in xrange ( len ( strings ) ) ] #NEWLINE# #UNINDENT# md_type = np . array ( metadata ) . dtype #NEWLINE# dtype = [ ( 'idx' , np . int ) , ( 'document_label' , md_type ) ] #NEWLINE# context_data = [ np . array ( zip ( indices , metadata ) , dtype = dtype ) ] #NEWLINE# c = Corpus ( corpus , context_data = context_data , context_types = [ 'document' ] ) #NEWLINE# if ( nltk_stop or stop_freq or add_stop ) : #NEWLINE# #INDENT# c = apply_stoplist ( c , nltk_stop = nltk_stop , freq = stop_freq , add_stop = add_stop ) #NEWLINE# #UNINDENT# return c