Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<feature_engineering>
print(torch.__version__)
mean = ['{0}, {1:.0f}'.format(pclass,np.nanmean(X.where(X.Pclass == pclass,inplace = False ).Fare)) for pclass in X.Pclass.unique() ]
print(mean )<drop_column>
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
 ef_models = ".. /input/efficientnet-pytorch/"
sys.path.insert(0, ef_models)
X_temp = X_temp.drop(['Ticket'],axis = 1 )<train_model>
import efficientnet_pytorch
from efficientnet_pytorch import EfficientNet
X_train, X_valid, Y_train, Y_valid = train_test_split(X_temp,Y,test_size = 0.25,random_state = 1)
 logistic = LogisticRegression(C=1, penalty="l1", solver='liblinear', random_state=7 ).fit(X_train,Y_train)
 model = SelectFromModel(logistic, prefit=True)
 X_new = model.transform(X_train)
 X_new<create_dataframe>
import glob
import torch
import albumentations
import pandas as pd
import numpy as np
from tqdm import tqdm
from PIL import Image
import joblib
import torch.nn as nn
from torch.nn import functional as F
selected_features = pd.DataFrame(model.inverse_transform(X_new),
index=X_train.index,
columns=X_train.columns)
 selected_columns = selected_features.columns[selected_features.var() != 0]
selected_columns<compute_train_metric>
MODEL_MEAN =(0.485, 0.456, 0.406)
 MODEL_STD =(0.229, 0.224, 0.225)
 IMG_HEIGHT = 137
IMG_WIDTH = 236
DEVICE="cuda"
cm2 = confusion_matrix(Y_valid,logistic.predict(X_valid))
log_acc = accuracy_score(Y_valid,logistic.predict(X_valid))
print(log_acc,'
',cm2 )<normalization>
class Se_resnext50_32x4d(nn.Module):
def __init__(self, pretrained):
super(Se_resnext50_32x4d, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["se_resnext50_32x4d"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["se_resnext50_32x4d"](pretrained=None)
 self.l0 = nn.Linear(2048, 168)
 self.l1 = nn.Linear(2048, 11)
 self.l2 = nn.Linear(2048, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
sc = MinMaxScaler()
scaled_X_temp = pd.DataFrame(sc.fit_transform(X_temp),columns = X_temp.columns)
 scaled_X_temp<find_best_model_class>
class Senet50(nn.Module):
def __init__(self, pretrained):
super(Senet50, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["se_resnet50"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["se_resnet50"](pretrained=None)
 self.l0 = nn.Linear(2048, 168)
 self.l1 = nn.Linear(2048, 11)
 self.l2 = nn.Linear(2048, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
X_train, X_valid, Y_train, Y_valid = train_test_split(scaled_X_temp,Y,test_size = 0.25,random_state = 1)
 knn_Classifier = KNeighborsClassifier(n_neighbors = 7, metric = 'minkowski', p=2)
 knn_Classifier.fit(X_train,Y_train)
 Y_pred_knn = knn_Classifier.predict(X_valid)
 cm2 = confusion_matrix(Y_valid,Y_pred_knn)
 knn_acc = accuracy_score(Y_valid,Y_pred_knn)
 print(knn_acc)
 print(cm2 )<compute_train_metric>
class EffNetB4(nn.Module):
def __init__(self, pretrained):
super(EffNetB4, self ).__init__()
if pretrained is True:
self.model = EfficientNet.from_pretrained('efficientnet-b4', in_channels=3)
 else:
self.model = EfficientNet.from_name('efficientnet-b4')
 self.l0 = nn.Linear(1792, 168)
 self.l1 = nn.Linear(1792, 11)
 self.l2 = nn.Linear(1792, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
cv_knn_score = cross_val_score(knn_Classifier,scaled_X_temp,Y,cv = 10,scoring='accuracy' ).mean()
print(cv_knn_score )<compute_train_metric>
class BengaliDatasetTest:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"image_id": img_id
}
svm_Classifier = SVC(kernel = 'rbf', random_state = 0)
 cv_svm_score = cross_val_score(svm_Classifier,scaled_X_temp,Y,cv = 10,scoring='accuracy' ).mean()
print(cv_svm_score )<compute_train_metric>
def model_predict() :
g_pred, v_pred, c_pred = [], [], []
img_ids_list = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTest(df=df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean=MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(
 dataset=dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=False,
num_workers=4 
)
 for bi, d in enumerate(data_loader):
image = d["image"]
img_id = d["image_id"]
image = image.to(DEVICE, dtype=torch.float)
 g, v, c = model(image)
 for ii, imid in enumerate(img_id):
g_pred.append(g[ii].cpu().detach().numpy())
 v_pred.append(v[ii].cpu().detach().numpy())
 c_pred.append(c[ii].cpu().detach().numpy())
 img_ids_list.append(imid)
 return g_pred, v_pred, c_pred, img_ids_list
nb_Classifier = GaussianNB()
cv_nb_score = cross_val_score(nb_Classifier,scaled_X_temp,Y,cv = 10,scoring='accuracy' ).mean()
print(cv_nb_score )<compute_train_metric>
models = []
model1 = EffNetB4(pretrained=False)
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold4.pth"])
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold5.pth"])
 models.append([model1, f".. /input/bengaliweights/effnetb4_afold6.pth"])
 TEST_BATCH_SIZE = 32
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i,v in enumerate(models):
model, path = v
model.to(DEVICE)
 model.load_state_dict(torch.load(path))
model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == 0:
final_img_ids.extend(img_ids_list)
dt_Classifier = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)
 cv_dt_score = cross_val_score(dt_Classifier,scaled_X_temp,Y,cv = 10,scoring='accuracy' ).mean()
print(cv_dt_score )<compute_train_metric>
a = np.array(final_g_pred[0])
 b = np.array(final_g_pred[1] )
rf_Classifier = RandomForestClassifier(n_estimators=10, max_depth=None, random_state=0)
 cv_rf_score = cross_val_score(rf_Classifier,scaled_X_temp, Y,cv = 8,scoring='accuracy' ).mean()
print(cv_rf_score )<compute_train_metric>
0.5*a+ 0.5*b
XGB_Classifier = XGBClassifier(n_estimators = 1000,learning_rate = 0.01)
 cv_XGB_score = cross_val_score(XGB_Classifier,scaled_X_temp, Y,cv = 18,scoring='accuracy' ).mean()
print(cv_XGB_score )<compute_train_metric>
final_g = np.argmax(np.mean(np.array(final_g_pred),axis = 0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred),axis = 0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred),axis = 0), axis=1 )
eclf = VotingClassifier(
 estimators=[('xgb',XGB_Classifier),('lr', logistic),('rf', rf_Classifier),('dt', dt_Classifier),('svm',svm_Classifier)],
voting='hard' 
)
 for clf, label in zip([XGB_Classifier, logistic, rf_Classifier, dt_Classifier,svm_Classifier, eclf],
['XGBooster Classifier', 'Logistic Regression', 'Random Forest', 'Decision Tree','SVM Classifier', 'Ensemble']):
scores = cross_val_score(clf, scaled_X_temp, Y, scoring='accuracy', cv=10)
 print("Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , label)) 
<train_on_grid>
predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))
params = {'lr__C': [1.0, 100.0], 'rf__n_estimators': [20, 200],
'xgb__n_estimators': [500,2000],'xgb__learning_rate': [0.01,0.1]}
rands = RandomizedSearchCV(estimator=eclf, param_distributions=params, cv=5)
 rands = rands.fit(scaled_X_temp, Y)
 Y_pred_rands = rands.predict(X_valid)
 rands_cm = confusion_matrix(Y_valid,Y_pred_rands)
 rands_acc = accuracy_score(Y_valid,Y_pred_rands)
 print(rands_acc)
 print(rands_cm )<load_from_csv>
sub = pd.DataFrame(predictions, columns=["row_id", "target"] )
<categorify><EOS> 
sub.to_csv("submission.csv", index=False )