#NEWLINE# def cooccurrence( iterable , window = ( ( - 1 ) , ( - 1 ) ) , term1 = ( lambda x : True ) , term2 = ( lambda x : True ) , normalize = ( lambda x : x ) , matrix = None , update = None ) : #NEWLINE# #INDENT# ' Returns the co-occurence matrix of terms in the given iterable, string, file or file list,\n        as a dictionary: {term1: {term2: count, term3: count, ...}}.\n        The window specifies the size of the co-occurence window.\n        The term1() function defines anchors.\n        The term2() function defines co-occurring terms to count.\n        The normalize() function can be used to remove punctuation, lowercase words, etc.\n        Optionally, a user-defined matrix to update can be given.\n        Optionally, a user-defined update(matrix, term1, term2, index2) function can be given.\n    ' #NEWLINE# if ( not isinstance ( matrix , dict ) ) : #NEWLINE# #INDENT# matrix = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# if isinstance ( iterable , basestring ) : #NEWLINE# #INDENT# iterable = isplit ( iterable ) #NEWLINE# #UNINDENT# if ( isinstance ( iterable , ( list , tuple ) ) and all ( ( hasattr ( f , 'read' ) for f in iterable ) ) ) : #NEWLINE# #INDENT# iterable = chain ( * ( isplit ( chain ( * x ) ) for x in iterable ) ) #NEWLINE# #UNINDENT# if hasattr ( iterable , 'read' ) : #NEWLINE# #INDENT# iterable = isplit ( chain ( * iterable ) ) #NEWLINE# #UNINDENT# q = deque ( ) #NEWLINE# n = ( ( - min ( 0 , window [ 0 ] ) ) + max ( window [ 1 ] , 0 ) ) #NEWLINE# m = matrix #NEWLINE# for x in chain ( ( [ Sentinel ( ) ] * term1 ) , iterable , ( [ Sentinel ( ) ] * n ) ) : #NEWLINE# #INDENT# q . append ( x ) #NEWLINE# if ( len ( q ) > n ) : #NEWLINE# #INDENT# if ( window [ 1 ] >= 0 ) : #NEWLINE# #INDENT# i = ( ( - 1 ) - window [ 1 ] ) #NEWLINE# #UNINDENT# if ( window [ 1 ] < 0 ) : #NEWLINE# #INDENT# i = ( len ( q ) - 1 ) #NEWLINE# #UNINDENT# if ( i < 0 ) : #NEWLINE# #INDENT# i = ( len ( q ) + i ) #NEWLINE# #UNINDENT# x1 = q [ i ] #NEWLINE# if ( not isinstance ( x1 , Sentinel ) ) : #NEWLINE# #INDENT# x1 = normalize ( x1 ) #NEWLINE# if term1 ( x1 ) : #NEWLINE# #INDENT# for ( j , x2 ) in enumerate ( list ( q ) . __getslice__ ( ( i + window [ 0 ] ) , ( ( i + window [ 1 ] ) + 1 ) ) ) : #NEWLINE# #INDENT# if ( not isinstance ( x2 , Sentinel ) ) : #NEWLINE# #INDENT# x2 = normalize ( x2 ) #NEWLINE# if term2 ( x2 ) : #NEWLINE# #INDENT# if update : #NEWLINE# #INDENT# update ( matrix , x1 , x2 , j ) #NEWLINE# continue #NEWLINE# #UNINDENT# if ( x1 not in m ) : #NEWLINE# #INDENT# m [ x1 ] = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# if ( x2 not in m [ x1 ] ) : #NEWLINE# #INDENT# m [ x1 ] [ x2 ] = 0 #NEWLINE# #UNINDENT# m [ x1 ] [ x2 ] += 1 #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# q . popleft ( ) #NEWLINE# #UNINDENT# #UNINDENT# return m