Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<predict_on_test>
%matplotlib inline
image_paths = []
targets = []
predictions = []
for i, batch in tqdm(enumerate(dm.val_dataloader())) :
with torch.no_grad() :
targets.append(batch['target'].detach().cpu().numpy())
 image_paths.append(batch['image_path'])
 pred = lit_model.model(batch['image'], batch['target'])[0]
predictions.append(pred.detach().cpu().numpy() )<prepare_output>
train_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv", delimiter='\t')
 train_data
preds_df = pd.DataFrame({'target': np.concatenate(targets),
'prediction': np.concatenate(predictions ).argmax(1),
'logits': np.concatenate(predictions ).max(1),
'image_paths': [i for j in image_paths for i in j]})
 preds_df.head()<compute_test_metric>
test_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv", delimiter='\t' )
print(metrics.classification_report(preds_df['target'], preds_df['prediction']))<compute_test_metric>
import re
import pandas
import numpy
import json
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.preprocessing.text import Tokenizer
metrics.confusion_matrix(preds_df['target'], preds_df['prediction'] )<load_from_csv>
def preprocess(review,remove_stopwords = False):
review_text = BeautifulSoup(review,"html5lib" ).get_text()
review_text = re.sub("[^a-zA-Z]"," ",review_text)
 words = review_text.lower().split()
if remove_stopwords:
stop_words = set(stopwords.words('english'))
words = [w for w in words if not w in stop_words]
clean_review = ' '.join(words)
 return clean_review
sub = pd.read_csv(f'{path}/sample_submission.csv')
 sub.head()<create_dataframe>
clean_train_reviews = []
clean_test_reviews = []
for review in train_data['review']:
clean_train_reviews.append(preprocess(review,remove_stopwords = True))
for review in test_data['review']:
clean_test_reviews.append(preprocess(review, remove_stopwords = True))
test_dataset = ImageClassificationDataset(image_names=sub['image_id'].values,
transforms=valid_augs,
labels=sub['label'].values,
img_path=f'{path}test_images/',
mode='test',
labels_to_ohe=False,
n_classes=5)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=4,
num_workers=4,
shuffle=False, 
)
<predict_on_test>
MAX_SEQUENCE_LENGTH = 174
tokenizer = Tokenizer()
tokenizer.fit_on_texts(clean_train_reviews + clean_train_reviews)
 text_sequences = tokenizer.texts_to_sequences(clean_train_reviews)
 train_inputs = pad_sequences(text_sequences,maxlen=MAX_SEQUENCE_LENGTH, padding='post')
 text_sequences = tokenizer.texts_to_sequences(clean_test_reviews)
 test_inputs = pad_sequences(text_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post' )
predictions = []
for batch in test_loader:
image = batch['image'].to('cuda')
 target = batch['target'].to('cuda')
 with torch.no_grad() :
outputs = lit_model.model(image, target)[0]
preds = outputs.argmax(1 ).detach().cpu().numpy()
predictions.append(preds )<concatenate>
print(tf.__version__ )
sub['label'] = np.concatenate(predictions )<save_to_csv>
len(tokenizer.word_index )
sub.to_csv('submission.csv', index=False )<save_to_csv>
vocab_size = len(tokenizer.word_index)+ 1
model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
 model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))
model.summary()
sub.to_csv('submission.csv', index=False )<define_variables>
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'] )
package_path = '.. /input/vision-transformer-pytorch/VisionTransformer-Pytorch'
sys.path.append(package_path)
<define_variables>
x_val = train_inputs[:5000]
partial_x_train = train_inputs[5000:]
y_val = np.array(train_data['sentiment'][:5000])
 partial_y_train = np.array(train_data['sentiment'][5000:] )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
sys.path.append(package_path )<import_modules>
history = model.fit(partial_x_train,
partial_y_train,
epochs=30,
batch_size=512,
verbose=1 )
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
pred_val = np.rint(model.predict(x_val)).astype('int32' ).squeeze()
for i in range(50):
print(y_val[i], pred_val[i] )
CFG = {
'fold_num': 12,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 384,
'epochs': 150,
'train_bs': 30,
'valid_bs': 40,
'lr': 1e-2,
'num_workers': 10,
'accum_iter': 1,
'verbose_step': 2,
'device': 'cuda:0',
'tta': 10,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
pred = np.rint(model.predict(test_inputs)).astype('int32' ).squeeze()
print(pred.shape )
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
commit_df = pd.DataFrame({'id': test_data['id'], 'sentiment':pred} )
train.label.value_counts()<load_from_csv>
commit_df.to_csv('commit7.csv', index = False )
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<categorify>
commit_test = pd.read_csv("./commit7.csv")
 commit_test
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
sample = pd.read_csv(".. /input/word2vec-nlp-tutorial/sampleSubmission.csv")
 sample