#NEWLINE# def get_texts( self ) : #NEWLINE# #INDENT# '\n        Iterate over the dump, returning text version of each article as a list\n        of tokens.\n\n        Only articles of sufficient length are returned (short articles & redirects\n        etc are ignored).\n\n        Note that this iterates over the **texts**; if you want vectors, just use\n        the standard corpus interface instead of this function::\n\n        >>> for vec in wiki_corpus:\n        >>>     print(vec)\n        ' #NEWLINE# ( articles , articles_all ) = ( 0 , 0 ) #NEWLINE# ( positions , positions_all ) = ( 0 , 0 ) #NEWLINE# texts = ( ( text , self . lemmatize , title , pageid ) for ( title , text , pageid ) in extract_pages ( bz2 . BZ2File ( self . fname ) , self . filter_namespaces ) ) #NEWLINE# pool = multiprocessing . Pool ( self . processes ) #NEWLINE# ignore_namespaces = 'Wikipedia Category File Portal Template MediaWiki User Help Book Draft' . split ( ) #NEWLINE# for group in utils . chunkize ( texts , chunksize = ( 10 * self . processes ) , maxsize = 1 ) : #NEWLINE# #INDENT# for ( tokens , title , pageid ) in pool . imap ( process_article , group ) : #NEWLINE# #INDENT# articles_all += 1 #NEWLINE# positions_all += len ( tokens ) #NEWLINE# if ( ( len ( tokens ) < ARTICLE_MIN_WORDS ) or any ( ( title . startswith ( ( ignore + ':' ) ) for ignore in ignore_namespaces ) ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# articles += 1 #NEWLINE# positions += len ( tokens ) #NEWLINE# if self . metadata : #NEWLINE# #INDENT# ( yield ( tokens , ( pageid , title ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# ( yield tokens ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# pool . terminate ( ) #NEWLINE# logger . info ( ( 'finished iterating over Wikipedia corpus of %i documents with %i positions (total %i articles, %i positions before pruning articles shorter than %i words)' % ( articles , positions , articles_all , positions_all , ARTICLE_MIN_WORDS ) ) ) #NEWLINE# self . length = articles