#NEWLINE# def load_data( path = 'imdb.pkl' , n_words = 100000 , valid_portion = 0.1 , maxlen = None , sort_by_len = True ) : #NEWLINE# #INDENT# 'Loads the dataset\n    :type path: String\n    :param path: The path to the dataset (here IMDB)\n    :type n_words: int\n    :param n_words: The number of word to keep in the vocabulary.\n        All extra words are set to unknow (1).\n    :type valid_portion: float\n    :param valid_portion: The proportion of the full train set used for\n        the validation set.\n    :type maxlen: None or positive int\n    :param maxlen: the max sequence length we use in the train/valid set.\n    :type sort_by_len: bool\n    :name sort_by_len: Sort by the sequence lenght for the train,\n        valid and test set. This allow faster execution as it cause\n        less padding per minibatch. Another mechanism must be used to\n        shuffle the train set at each epoch.\n    ' #NEWLINE# path = get_dataset_file ( path , 'imdb.pkl' , 'http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl' ) #NEWLINE# if path . endswith ( '.gz' ) : #NEWLINE# #INDENT# f = gzip . open ( path , 'rb' ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# f = open ( path , 'rb' ) #NEWLINE# #UNINDENT# train_set = pickle . load ( f ) #NEWLINE# test_set = pickle . load ( f ) #NEWLINE# f . close ( ) #NEWLINE# if maxlen : #NEWLINE# #INDENT# new_train_set_x = [ ] #NEWLINE# new_train_set_y = [ ] #NEWLINE# for ( x , y ) in zip ( train_set [ 0 ] , train_set [ 1 ] ) : #NEWLINE# #INDENT# if ( len ( x ) < maxlen ) : #NEWLINE# #INDENT# new_train_set_x . append ( x ) #NEWLINE# new_train_set_y . append ( y ) #NEWLINE# #UNINDENT# #UNINDENT# train_set = ( new_train_set_x , new_train_set_y ) #NEWLINE# del new_train_set_x , new_train_set_y #NEWLINE# #UNINDENT# ( train_set_x , train_set_y ) = train_set #NEWLINE# n_samples = len ( train_set_x ) #NEWLINE# sidx = numpy . random . permutation ( n_samples ) #NEWLINE# n_train = int ( numpy . round ( ( n_samples * ( 1.0 - valid_portion ) ) ) ) #NEWLINE# valid_set_x = [ train_set_x [ s ] for s in sidx [ n_train : ] ] #NEWLINE# valid_set_y = [ train_set_y [ s ] for s in sidx [ n_train : ] ] #NEWLINE# train_set_x = [ train_set_x [ s ] for s in sidx [ : n_train ] ] #NEWLINE# train_set_y = [ train_set_y [ s ] for s in sidx [ : n_train ] ] #NEWLINE# train_set = ( train_set_x , train_set_y ) #NEWLINE# valid_set = ( valid_set_x , valid_set_y ) #NEWLINE# def remove_unk( x ) : #NEWLINE# #INDENT# return [ [ ( 1 if ( w >= n_words ) else w ) for w in sen ] for sen in x ] #NEWLINE# #UNINDENT# ( test_set_x , test_set_y ) = test_set #NEWLINE# ( valid_set_x , valid_set_y ) = valid_set #NEWLINE# ( train_set_x , train_set_y ) = train_set #NEWLINE# train_set_x = remove_unk ( train_set_x ) #NEWLINE# valid_set_x = remove_unk ( valid_set_x ) #NEWLINE# test_set_x = remove_unk ( test_set_x ) #NEWLINE# def len_argsort( seq ) : #NEWLINE# #INDENT# return sorted ( range ( len ( seq ) ) , key = ( lambda x : len ( seq [ x ] ) ) ) #NEWLINE# #UNINDENT# if sort_by_len : #NEWLINE# #INDENT# sorted_index = len_argsort ( test_set_x ) #NEWLINE# test_set_x = [ test_set_x [ i ] for i in sorted_index ] #NEWLINE# test_set_y = [ test_set_y [ i ] for i in sorted_index ] #NEWLINE# sorted_index = len_argsort ( valid_set_x ) #NEWLINE# valid_set_x = [ valid_set_x [ i ] for i in sorted_index ] #NEWLINE# valid_set_y = [ valid_set_y [ i ] for i in sorted_index ] #NEWLINE# sorted_index = len_argsort ( train_set_x ) #NEWLINE# train_set_x = [ train_set_x [ i ] for i in sorted_index ] #NEWLINE# train_set_y = [ train_set_y [ i ] for i in sorted_index ] #NEWLINE# #UNINDENT# train = ( train_set_x , train_set_y ) #NEWLINE# valid = ( valid_set_x , valid_set_y ) #NEWLINE# test = ( test_set_x , test_set_y ) #NEWLINE# return ( train , valid , test )