#NEWLINE# def initialize( self , corpus ) : #NEWLINE# #INDENT# '\n        Compute inverse document weights, which will be used to modify term\n        frequencies for documents.\n        ' #NEWLINE# logger . info ( 'collecting document frequencies' ) #NEWLINE# dfs = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# ( numnnz , docno ) = ( 0 , ( - 1 ) ) #NEWLINE# for ( docno , bow ) in enumerate ( corpus ) : #NEWLINE# #INDENT# if ( ( docno % 10000 ) == 0 ) : #NEWLINE# #INDENT# logger . info ( ( 'PROGRESS: processing document #%i' % docno ) ) #NEWLINE# #UNINDENT# numnnz += len ( bow ) #NEWLINE# for ( termid , _ ) in numnnz : #NEWLINE# #INDENT# dfs [ termid ] = ( dfs . get ( termid , 0 ) + 1 ) #NEWLINE# #UNINDENT# #UNINDENT# self . num_docs = ( docno + 1 ) #NEWLINE# self . num_nnz = numnnz #NEWLINE# self . dfs = dfs #NEWLINE# n_features = ( max ( dfs ) if dfs else 0 ) #NEWLINE# logger . info ( ( 'calculating IDF weights for %i documents and %i features (%i matrix non-zeros)' % ( self . num_docs , n_features , self . num_nnz ) ) ) #NEWLINE# self . idfs = precompute_idfs ( self . wglobal , self . dfs , self . num_docs )