Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<import_modules>
print('keras version:', __version__ )
!pip install bert-tensorflow
print(tf.version.VERSION )<import_modules>
df_train=pd.read_csv(".. /input/train.csv")
 df_test=pd.read_csv(".. /input/test.csv")
 print([df_train.shape,df_test.shape])
 print(df_train.iloc[265,1:].values.reshape(28,28)[:,10] )
print(transformers.__version__ )<set_options>
images=["%s%s"%("pixel",pixel_no)for pixel_no in range(0,784)]
train_images=np.array(df_train[images])
 train_images=(train_images.astype('float32')/255)**k
train_labels=df_train['label']
train_labels_cat=to_categorical(train_labels,num_classes=10)
 test_images=np.array(df_test[images])
 test_images=(test_images.astype('float32')/255)**k
[train_images.shape,train_labels_cat.shape,test_images.shape]
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
<define_variables>
X_train,X_test,y_train,y_test=\
train_test_split(train_images,train_labels_cat,
test_size=.2,random_state=32)
 n=int(len(X_test)/2)
 X_valid,y_valid=X_test[:n],y_test[:n]
X_test,y_test=X_test[n:],y_test[n:]
y_train_num=np.array([np.argmax(x)for x in y_train])
 y_test_num=np.array([np.argmax(x)for x in y_test])
 y_valid_num=np.array([np.argmax(x)for x in y_valid])
 [X_train.shape,X_test.shape,X_valid.shape,\
y_train.shape,y_test.shape,y_valid.shape]
SEQUENCE_LENGTH = 128
DATA_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 BERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')
 BERT_PATH_SAVEDMODEL = BERT_PATH + "/bert_multi_from_tfhub"
WEIGHTS_PATH = '.. /input/jigsaw-weights'
OUTPUT_PATH = "/kaggle/working"<load_from_csv>
def cnn_model() :
model_input=Input(shape=(28,28,1))
x=BatchNormalization()(model_input)
 x=Conv2D(28,(5,5),padding='same' )(x)
 x=LeakyReLU(alpha=.02 )(x)
 x=MaxPooling2D(strides=(2,2))(x)
 x=Dropout (.25 )(x)
 x=Conv2D(96,(5,5))(x)
 x=LeakyReLU(alpha=.02 )(x)
 x=MaxPooling2D(strides=(2,2))(x)
 x=Dropout (.25 )(x)
 x=GlobalMaxPooling2D()(x)
 x=Dense(512 )(x)
 x=LeakyReLU(alpha=.02 )(x)
 x=Dropout (.5 )(x)
 y=Dense(10,activation='softmax' )(x)
 model=Model(input=model_input,output=y)
 model.compile(loss='categorical_crossentropy',optimizer='nadam',
metrics=[categorical_accuracy,top_3_categorical_accuracy])
 return model
cnn_model=cnn_model()
cnn_model.summary()
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub2 = pd.read_csv('.. /input/ensemble/submission.csv' )<choose_model_class>
checkpointer=ModelCheckpoint(filepath=wcnn,verbose=2,save_best_only=True)
 lr_reduction=ReduceLROnPlateau(monitor='val_loss',patience=5,verbose=2,factor=.8)
 estopping=EarlyStopping(monitor='val_loss',patience=20,verbose=2)
 history=cnn_model.fit(X_train.reshape(-1,28,28,1),y_train,
validation_data=(X_valid.reshape(-1,28,28,1),y_valid),
epochs=100,batch_size=128,verbose=2,
callbacks=[checkpointer,lr_reduction,estopping] )
def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):
bert_layer = tf.saved_model.load(bert_path)
 bert_layer = hub.KerasLayer(bert_layer, trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
cased = bert_layer.resolved_object.do_lower_case.numpy()
tf.gfile = tf.io.gfile
tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)
 return tokenizer
tokenizer = get_tokenizer()<categorify>
steps,epochs=1000,10
data_generator=\
ImageDataGenerator(featurewise_std_normalization=True,
zoom_range=.2,shear_range=.2,rotation_range=30,
height_shift_range=.2,width_shift_range=.2)
 history=cnn_model.\
fit_generator(data_generator.flow(X_train.reshape(-1,28,28,1),
y_train,batch_size=128),
steps_per_epoch=steps,epochs=epochs,verbose=2,
validation_data=(X_valid.reshape(-1,28,28,1),y_valid),
callbacks=[checkpointer,lr_reduction,estopping] )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<define_variables>
cnn_model.load_weights(wcnn)
 scores=cnn_model.evaluate(X_test.reshape(-1,28,28,1),y_test,verbose=0)
 print("CNN Scores: ",(scores))
print("CNN Error: %.2f%%"%(100-scores[1]*100))
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192<load_pretrained>
Model
def mlp_model() :
model=Sequential()
model.add(Dense(784,activation='relu',input_shape=(784,)))
 model.add(Dropout (.25))
model.add(Dense(392,activation='relu'))
model.add(Dropout (.25))
model.add(Dense(196,activation='relu'))
model.add(Dropout (.25))
model.add(Dense(10,activation='softmax'))
model.compile(optimizer='nadam',loss='categorical_crossentropy',
metrics=[categorical_accuracy,top_3_categorical_accuracy])
 return model
mlp_model=mlp_model()
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer<categorify>
checkpointer=ModelCheckpoint(filepath=wcnn,verbose=2,save_best_only=True)
 lr_reduction=ReduceLROnPlateau(monitor='val_loss',patience=5,verbose=2,factor=.75)
 estopping=EarlyStopping(monitor='val_loss',patience=20,verbose=2)
 history=mlp_model.fit(X_train,y_train,validation_data=(X_valid,y_valid),
epochs=100,batch_size=128,verbose=2,
callbacks=[checkpointer,lr_reduction,estopping] )
x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN )<prepare_x_and_y>
hplot(history)
 scores=mlp_model.evaluate(X_test,y_test)
 print("
MLP Scores: ",(scores))
print("MLP Error: %.2f%%"%(100-scores[1]*100))
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values<split>
def rnn_model() :
model=Sequential()
model.add(LSTM(196,return_sequences=True,input_shape=(1,784)))
 model.add(LSTM(196,return_sequences=True))
model.add(LSTM(196))
model.add(Dense(10,activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='nadam',
metrics=[categorical_accuracy,top_3_categorical_accuracy])
 return model
rnn_model=rnn_model()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<choose_model_class>
checkpointer=ModelCheckpoint(filepath=wcnn,verbose=2,save_best_only=True)
 lr_reduction=ReduceLROnPlateau(monitor='val_loss',patience=5,verbose=2,factor=.75)
 estopping=EarlyStopping(monitor='val_loss',patience=20,verbose=2)
 history=rnn_model.fit(X_train.reshape(-1,1,X_train.shape[1]),y_train,
epochs=100,batch_size=128,verbose=2,
validation_data=(X_valid.reshape(-1,1,X_valid.shape[1]),y_valid),
callbacks=[checkpointer,lr_reduction,estopping] )
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = tf.keras.layers.Dense(300, activation='relu' )(cls_token)
 out = tf.keras.layers.Dense(128, activation='relu' )(out)
 out = tf.keras.layers.Dense(128, activation='relu' )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
n_total=7000; X2=np.copy(X_train[:n_total])
 y2=np.copy(y_train_num[:n_total] ).astype('int64')
 clf=MLPClassifier(hidden_layer_sizes=(784,),max_iter=100,alpha=1e-4,
solver='lbfgs',verbose=1,tol=1e-6,random_state=1,
learning_rate_init=7e-4,batch_size=128)
 clf.fit(X2,y2)
 print("MNIST.MLPClassifier.Train score: %f"%(clf.score(X2,y2)*100),'%')
 print("MNIST.MLPClassifier.Test score: %f"%(clf.score(X_test,y_test_num)*100),'%' )
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFDistilBertModel 
.from_pretrained('distilbert-base-multilingual-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN )<load_pretrained>
n_total=5000; n_labeled=4000
X2=np.copy(X_train[:n_total])
 y2=np.copy(y_train_num[:n_total] ).astype('int64')
 y2[n_labeled:]=-1
lp_model=label_propagation\ 
.LabelSpreading(kernel='knn',n_neighbors=10,max_iter=20)
 lp_model.fit(X2,y2)
 predicted_labels=lp_model.transduction_[n_labeled:n_total]
true_labels=y_train_num[n_labeled:n_total]
print('Label Spreading: %d labeled & %d unlabeled points(%d total)'% 
(n_labeled,n_total-n_labeled,n_total))
print(classification_report(true_labels,predicted_labels))
print('Confusion matrix')
 print(confusion_matrix(true_labels,predicted_labels,
labels=lp_model.classes_))
predict_entropies=\
stats.distributions.entropy(lp_model.label_distributions_.T)
 uncertainty_index=np.argsort(predict_entropies)[-10:]
model.load_weights(WEIGHTS_PATH+"/weights.h5" )<train_model>
predict_labels=cnn_model.predict(test_images.reshape(-1,28,28,1))
predict_labels=predict_labels.argmax(axis=-1 )
<train_model><EOS> 
submission=pd.DataFrame({"ImageId":range(1,len(predict_labels)+1),
"Label":predict_labels})
 print(submission[0:10])
 submission.to_csv('kaggle_digits_cnn.csv',index=False )