#NEWLINE# @ app . route ( '/<username>/make_subset/<int:folder_id>/<int:ex_id>' , methods = [ 'POST' ] ) #NEWLINE# def make_subset( username = None , folder_id = None , ex_id = None ) : #NEWLINE# #INDENT# dOld = Folder . query . get ( folder_id ) #NEWLINE# [ topW , topic_probs , doc_tops , lms ] = Experiment . query . get ( ex_id ) . getExInfo ( ) . viz_data #NEWLINE# nTops = len ( doc_tops [ 0 ] ) #NEWLINE# nDox = int ( request . form [ 'nDox' ] ) #NEWLINE# searchwords = request . form [ 'searchwords' ] . split ( ' ' ) #NEWLINE# topicblocks = [ int ( x ) for x in request . form . getlist ( 'blocks[]' ) ] #NEWLINE# blockDist = [ 0.0 for _ in xrange ( nTops ) ] #NEWLINE# tot = 0.0 #NEWLINE# for v in topicblocks : #NEWLINE# #INDENT# if ( v >= 0 ) : #NEWLINE# #INDENT# blockDist [ v ] += 1.0 #NEWLINE# tot += 1.0 #NEWLINE# #UNINDENT# #UNINDENT# if ( tot > 0 ) : #NEWLINE# #INDENT# blockDist = [ ( x / tot ) for x in blockDist ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# blockDist = [ 0.0 for x in blockDist ] #NEWLINE# #UNINDENT# '\n    get target KL distribution from search terms\n    sort by kl - below\n    ' #NEWLINE# vocabfile = dOld . vocab_path ( ) #NEWLINE# vocab = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# idx = 0 #NEWLINE# vv = [ x . strip ( ) for x in open ( vocabfile , 'r' ) ] #NEWLINE# for v in vv : #NEWLINE# #INDENT# vocab [ v ] = idx #NEWLINE# idx += 1 #NEWLINE# #UNINDENT# searchSmooth = 1e-07 #NEWLINE# searchDist = [ searchSmooth for _ in xrange ( nTops ) ] #NEWLINE# sdTotal = ( searchSmooth * nTops ) #NEWLINE# for sw in searchwords : #NEWLINE# #INDENT# if ( sw in vocab ) : #NEWLINE# #INDENT# idx = vocab [ sw ] #NEWLINE# for tidx in xrange ( nTops ) : #NEWLINE# #INDENT# for ( i , p ) in topW [ tidx ] : #NEWLINE# #INDENT# if ( i == idx ) : #NEWLINE# #INDENT# searchDist [ tidx ] += p #NEWLINE# sdTotal += p #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# searchDist = [ ( x / sdTotal ) for x in searchDist ] #NEWLINE# if ( len ( request . form [ 'searchwords' ] . strip ( ) ) > 0 ) : #NEWLINE# #INDENT# targDist = [ ( ( a + b ) / 2.0 ) for ( a , b ) in zip ( searchDist , blockDist ) ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# targDist = blockDist #NEWLINE# #UNINDENT# targTotal = sum ( targDist ) #NEWLINE# if ( targTotal == 0 ) : #NEWLINE# #INDENT# targDist = [ ( 1.0 / float ( nTops ) ) for x in xrange ( nTops ) ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# targDist = [ ( x / targTotal ) for x in targDist ] #NEWLINE# #UNINDENT# L = np . array ( lms , dtype = np . float16 ) #NEWLINE# searchLM = np . log ( np . transpose ( np . array ( targDist , dtype = np . float16 ) . dot ( L ) ) ) #NEWLINE# D = sp . sparse . dok_matrix ( ( dOld . N ( ) , dOld . vocabSize ) , dtype = np . float16 ) #NEWLINE# wordcounts = open ( dOld . wordcount_path ( ) , 'r' ) #NEWLINE# totalW = np . zeros ( dOld . N ( ) ) #NEWLINE# for line in wordcounts : #NEWLINE# #INDENT# [ d , w , c ] = [ int ( x ) for x in line . strip ( ) . split ( ',' ) ] #NEWLINE# D [ ( d , w ) ] = c #NEWLINE# totalW [ d ] += c #NEWLINE# #UNINDENT# wordcounts . close ( ) #NEWLINE# D = D . asformat ( 'csr' ) #NEWLINE# lmScores = D . dot ( searchLM ) #NEWLINE# lmScores = np . divide ( lmScores , totalW ) #NEWLINE# lmScores = np . divide ( np . power ( 2.0 , np . multiply ( lmScores , ( - 1.0 ) ) ) , totalW ) #NEWLINE# inds = np . argsort ( lmScores ) [ : nDox ] . tolist ( ) #NEWLINE# probs = [ doc_tops [ idx ] for idx in inds ] #NEWLINE# topDocs = zip ( inds , probs ) #NEWLINE# aD = [ d for d in dOld . all_docs ( ) ] #NEWLINE# def makedata( ) : #NEWLINE# #INDENT# for td in topDocs : #NEWLINE# #INDENT# d = aD [ td [ 0 ] ] #NEWLINE# ( yield [ d . id , d . name , td [ 1 ] ] ) #NEWLINE# #UNINDENT# #UNINDENT# dd = [ x for x in makedata ( ) ] #NEWLINE# return jsonify ( docs = dd , targDist = targDist )