#NEWLINE# def record_tokenize( records , record_names , verbose = 1 ) : #NEWLINE# #INDENT# '\n    `record_tokenize` is a helper function for :meth:`record_corpus`.\n\n    Takes a list of books and `book_names`, and returns words \n    and corpus data.\n\n    :param books: List of books.\n    :type books: list\n\n    :param book_names: List of book names.\n    :type book_names: list\n\n    :param verbose: Verbosity level. 1 prints a progress bar.\n    :type verbose: int, default 1 \n\n    :returns: words : List of words.\n        words in the `books` tokenized by :meth:`word_tokenize`.\n        corpus_data : Dictionary with context type as keys and\n        corresponding tokenizations as values. The tokenizations\n        are np.arrays.\n    ' #NEWLINE# ( words , record_tokens , book_tokens , page_tokens , sent_tokens ) = ( [ ] , [ ] , [ ] , [ ] , [ ] ) #NEWLINE# ( sent_break , record_n , book_n , page_n , sent_n ) = ( 0 , 0 , 0 , 0 , 0 ) #NEWLINE# if ( verbose == 1 ) : #NEWLINE# #INDENT# pbar = ProgressBar ( widgets = [ Percentage ( ) , Bar ( ) ] , maxval = len ( books ) ) . start ( ) #NEWLINE# #UNINDENT# for ( record , record_label , book_labels ) in zip ( records , record_names , record_book_names ) : #NEWLINE# #INDENT# for ( book , book_label ) in zip ( record , book_labels ) : #NEWLINE# #INDENT# for ( page , page_file ) in book : #NEWLINE# #INDENT# sents = sentence_tokenize ( page ) #NEWLINE# for sent in sents : #NEWLINE# #INDENT# w = word_tokenize ( sent ) #NEWLINE# words . extend ( w ) #NEWLINE# sent_break += len ( w ) #NEWLINE# sent_tokens . append ( ( sent_break , sent_n , page_n , book_label , record_label , page_file ) ) #NEWLINE# sent_n += 1 #NEWLINE# #UNINDENT# page_tokens . append ( ( sent_break , page_n , book_label , record_label , page_file ) ) #NEWLINE# page_n += 1 #NEWLINE# #UNINDENT# if ( verbose == 1 ) : #NEWLINE# #INDENT# pbar . update ( book_n ) #NEWLINE# #UNINDENT# book_tokens . append ( ( sent_break , book_label , record_label ) ) #NEWLINE# book_n += 1 #NEWLINE# #UNINDENT# record_tokens . append ( ( sent_break , record_label ) ) #NEWLINE# record_n += 1 #NEWLINE# #UNINDENT# idx_dt = ( 'idx' , np . int32 ) #NEWLINE# record_label_dt = ( 'record_label' , np . array ( record_names ) . dtype ) #NEWLINE# book_label_dt = ( 'book_label' , np . array ( book_names ) . dtype ) #NEWLINE# page_label_dt = ( 'page_label' , np . array ( page_n , np . str_ ) . dtype ) #NEWLINE# sent_label_dt = ( 'sentence_label' , np . array ( sent_n , np . str_ ) . dtype ) #NEWLINE# files = [ f for ( a , b , c , f ) in page_tokens ] #NEWLINE# file_dt = ( 'file' , np . array ( files , np . str_ ) . dtype ) #NEWLINE# corpus_data = dict ( ) #NEWLINE# dtype = [ idx_dt , record_label_dt ] #NEWLINE# corpus_data [ 'record' ] = np . array ( record_tokens , dtype = dtype ) #NEWLINE# dtype = [ idx_dt , book_label_dt , record_label_dt ] #NEWLINE# corpus_data [ 'book' ] = np . array ( book_tokens , dtype = dtype ) #NEWLINE# dtype = [ idx_dt , page_label_dt , book_label_dt , record_label_dt , file_dt ] #NEWLINE# corpus_data [ 'page' ] = np . array ( page_tokens , dtype = dtype ) #NEWLINE# dtype = [ idx_dt , sent_label_dt , page_label_dt , book_label_dt , record_label_dt , file_dt ] #NEWLINE# corpus_data [ 'sentence' ] = np . array ( sent_tokens , dtype = dtype ) #NEWLINE# if ( verbose == 1 ) : #NEWLINE# #INDENT# pbar . finish ( ) #NEWLINE# #UNINDENT# return ( words , corpus_data )