Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<load_from_csv>
print(os.listdir(".. /input"))
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_data.head()<drop_column>
raw_train_data_labeled = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 raw_train_data_unlabeled = pd.read_csv(".. /input/word2vec-nlp-tutorial/unlabeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 raw_test_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv", header=0, delimiter="\t", quoting=3 )
train_labels = train_data['label']
train_data = train_data.drop('label', axis=1 )<categorify>
imdb_data = pd.read_csv('.. /input/imdb-review-dataset/imdb_master.csv',encoding="latin-1")
 imdb_data.head()
encoder = LabelBinarizer()
train_labels = encoder.fit_transform(train_labels )<data_type_conversions>
imdb_data = imdb_data.drop(["Unnamed: 0","type","file"],axis=1 )
train_data = train_data.astype('float32')/ 255
test_data = test_data.astype('float32')/ 255<prepare_x_and_y>
imdb_data['sentiment'] = imdb_data['label'].map({"neg":0, "pos":1} )
train_data = train_data[:, :, :, np.newaxis]
test_data = test_data[:, :, :, np.newaxis]<split>
imdb_data = imdb_data.drop(["label"],axis=1 )
X_train, X_val, y_train, y_val = train_test_split(train_data, train_labels, test_size=0.1, random_state=157, stratify=train_labels )<train_model>
imdb_data = imdb_data.dropna()
data_generator = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
horizontal_flip=False,
vertical_flip=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 data_generator.fit(X_train )<choose_model_class>
imdb_data = imdb_data[['sentiment','review']]
init_relu = he_uniform(seed=157)
 init_tanh = glorot_uniform(seed=157)
 model = Sequential()
model.add(Conv2D(name='Conv_1', input_shape=(28, 28, 1), filters=64, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Conv2D(name='Conv_2', filters=64, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(MaxPooling2D(name='Pooling_1', pool_size=(2, 2), strides=2))
model.add(Dropout(0.20))
model.add(Conv2D(name='Conv_4', filters=32, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Conv2D(name='Conv_5', filters=32, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(MaxPooling2D(name='Pooling_2', pool_size=(2, 2), strides=2))
model.add(Dropout(0.20))
model.add(Flatten())
 model.add(Dense(name='FC_1', units=512, kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Dropout(0.20))
model.add(Dense(name='FC_2', units=128, kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Dense(name='output', units=10, activation='softmax', kernel_initializer=init_tanh, kernel_constraint=maxnorm(3)) )<feature_engineering>
X = raw_train_data_labeled['review']
y = raw_train_data_labeled['sentiment']
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ['PYTHONHASHSEED'] = str(157)
 random.seed(157)
 np.random.seed(157)
 tf.compat.v1.set_random_seed(157)
 session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 k.set_session(sess )<choose_model_class>
review_data = X.append(raw_test_data['review'] )
lr = 0.001
opt = Adam(learning_rate=lr, amsgrad=True)
 model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
 checkpoint = ModelCheckpoint('neural_network_checkpoint_training.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 lr_decay = LearningRateScheduler(lambda x: lr * 0.95 ** x)
 tensorboard = TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_images=True)
 model.fit_generator(data_generator.flow(X_train, y_train, batch_size=64),
epochs=50,
validation_data=(X_val, y_val),
shuffle=True,
callbacks=[tensorboard, checkpoint, lr_decay],
verbose=1)
 save_model(checkpoint.model, 'neural_network_latest_saved.h5' )<save_to_csv>
review_data = review_data.append(imdb_data['review'] )
clf = load_model('neural_network_checkpoint_training.h5')
 prediction = clf.predict(test_data)
 prediction = pd.DataFrame(np.argmax(prediction, axis=-1), columns=['Label'])
 img_idx = pd.DataFrame(np.arange(1, len(prediction)+ 1), columns=['ImageId'])
 prediction = pd.concat([img_idx, prediction], axis=1)
 prediction.to_csv('submission_cnn.csv', index=False )<load_from_url>
ntrain = X.shape[0]
df = raw_train_data_labeled.append(raw_test_data, sort=False)
 df = df.drop(['sentiment'], axis=1)
 df_review = df['review']
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py<install_modules>

 review_lines = list()
lines = review_data.values.tolist()
for line in lines:

 tokens = word_tokenize(line)
 tokens = [w.lower() for w in tokens]
table = str.maketrans('','', string.punctuation)
 stripped = [w.translate(table)for w in tokens]
words = [w for w in stripped if w.isalpha() ]
stop_words = set(stopwords.words('english'))
words = [w for w in words if w not in stop_words]
review_lines.append(words )
!python pytorch-xla-env-setup.py<install_modules>
avg_length = sum(length)/len(review_lines)
 print(avg_length )
!pip install pytorch_lightning --quiet<find_best_params>
max_len = math.ceil(avg_length )
torch_xla._XLAC._xla_get_devices()<set_options>
vocab_size = len(tokenizer.word_index)+1
embedding_matrix = np.zeros(( vocab_size, embedding_vector_size))
for word, i in tokenizer.word_index.items() :
embedding_vector = model_1.wv[word]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
torch.manual_seed(100)
 np.random.seed(100 )<load_from_csv>
X = train_test_data[:ntrain,:]
X = np.append(X,train_test_data[ntrain+25000: ,:])
 X = X.reshape(-1,123)
 y1 = y.append(imdb_data['sentiment'] )
df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df.iloc[:3,:10]<split>
X_train, X_test, y_train, y_test = train_test_split(X , y1, test_size=0.2, random_state=42, shuffle=True )
df_train, df_val = train_test_split(df, test_size=.1, stratify=df.label )<prepare_x_and_y>
model = Sequential()
model.add(Embedding(input_dim = vocab_size, output_dim = embedding_vector_size,
input_length = max_len, weights = [embedding_matrix]))
model.add(Bidirectional(LSTM(64, dropout=0.25, recurrent_dropout=0.1)))
 model.add(Dense(10))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
def toX(df):
return df.values.astype(np.float32 ).reshape(-1,1,28,28)/ 127.5 - 1.
 class MnistDataLoader(object):
def __init__(self, df, bs):
self.X = toX(df.iloc[:, 1:])
 self.y = df.values[:, 0]
self.bs = bs
self.n_batches = int(np.ceil(df.shape[0] / bs))
def __len__(self):
return self.n_batches
def __iter__(self):
m = self.y.shape[0]
for i in range(self.n_batches):
idx = np.random.randint(0, m, self.bs)
 X = torch.tensor(self.X[idx])
 y = torch.tensor(self.y[idx])
 yield X, y<define_search_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
class ResnetBlock(nn.Module):
def __init__(self, channels):
super(ResnetBlock, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(channels, channels, 3, 1, 1, bias=False),
nn.BatchNorm2d(channels),
nn.ReLU() ,
nn.Conv2d(channels, channels, 3, 1, 1, bias=False),
nn.BatchNorm2d(channels))
def forward(self, x):
x = F.relu(self.conv(x)+ x)
 x = F.max_pool2d(x, 2)
 return F.dropout2d(x,.2, self.training)
 class MiniResnet(nn.Module):
def __init__(self, channels=32):
super(MiniResnet, self ).__init__()
self.backbone = nn.Sequential(
 nn.Conv2d(1, channels, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(channels),
nn.ReLU() ,
ResnetBlock(channels),
ResnetBlock(channels)
)
 self.head = nn.Sequential(
 nn.Linear(channels*49, 256),
nn.ReLU() ,
nn.Dropout (.2),
nn.Linear(256,10))
def forward(self, x):
x = self.backbone(x)
 dims = np.prod(x.shape[1:])
 logits = self.head(x.view(-1, dims))
return logits<train_on_grid>
model.compile(optimizer='RMSprop', loss='binary_crossentropy', metrics=['acc'])
 print(model.summary())
 history = model.fit(X_train, y_train, epochs = 30, batch_size = 700, validation_data=(X_test, y_test),callbacks = [learning_rate_reduction] )
class MnistModule(pl.LightningModule):
def __init__(self):
super().__init__()
self.model = MiniResnet()
self.loss_fn = nn.CrossEntropyLoss()
def forward(self, X):
return self.model(X)
 def training_step(self, batch, batch_i):
X, y = batch
h = self.model(X)
 loss = self.loss_fn(h, y)
 self.log('train_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)
 return loss
def validation_step(self, batch, batch_idx):
x, y = batch
h = self.model(x)
 loss = self.loss_fn(h, y)
 accuracy =(h.argmax(1)== y ).float().mean()
self.log('valid_loss', loss, on_step=False, on_epoch=True, logger=True)
 self.log('accuracy', accuracy, on_step=False, on_epoch=True, logger=True)
 def configure_optimizers(self):
opt = torch.optim.Adam(self.parameters())
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=2)
 return [opt], [{ 'scheduler': scheduler, 'monitor': 'valid_loss'}]
def predict(self, X):
with torch.no_grad() :
self.eval()
logits = self.model(X)
 return F.softmax(logits, 1)
 batch_size = 512
train_dl = MnistDataLoader(df_train, batch_size)
 val_dl = MnistDataLoader(df_val, batch_size)
 module = MnistModule()
trainer = pl.Trainer(max_epochs=30, tpu_cores=1,logger=SimpleLogger())
 trainer.fit(module, train_dl, val_dl )<predict_on_test>
y_test_pred = model.predict(X_test)
x = torch.tensor(val_dl.X, device=module.device)
 h = module.predict(x ).cpu().numpy()<prepare_output>
roc_auc_score(y_test, y_test_pred, average = 'weighted' )
y_hat = h.argmax(1)
(y_hat == val_dl.y ).mean()<load_from_csv>
y_pred = model.predict(train_test_data[ntrain:ntrain+25000 , :] )
df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 df_sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )<predict_on_test>
predictions = [1 if(x>0.5)else 0 for x in y_pred ]
X = torch.tensor(toX(df_test), device = module.device)
 %time preds = module.predict(X )<save_to_csv>
ids = raw_test_data['id'].str.replace('"', '' )
df_sub['Label'] = preds.argmax(1 ).cpu().numpy()
df_sub.to_csv('submission.csv', index=False )<import_modules>
submission = pd.DataFrame({'id': ids, 'sentiment':predictions})
 submission.to_csv('word2vec.csv',index=False )