Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
!pip install --no-deps.. /input/pretrined-models/timm-0.3.3-py3-none-any.whl
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 all_data['DT_time'] = all_data['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 all_data['count'] = 1
all_data['diff_days_from_first_transaction'] = all_data['D1'].fillna(0 ).apply(lambda x:(datetime.timedelta(days = x)))
 all_data['client_firstdate'] = all_data['DT_time']- all_data['diff_days_from_first_transaction']
all_data['client_firstdate_days'] =(all_data['DT_time']- all_data['diff_days_from_first_transaction'] ).apply(lambda x:str(x.date()))<define_variables>
import os
import pandas as pd
import timm
from PIL import Image, ImageDraw, ImageChops
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from tqdm import tqdm
anaysis_fea = [ 'TransactionID',
'isFraud',
'TransactionDT',
'TransactionAmt',
'ProductCD',
'device_hash','card_hash', 'V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo',
'card1','card2','card3','card4','card5','card6','client_firstdate_days','dist1','dist2','P_emaildomain','addr1','addr2','train_or_test','count','DT_time','diff_days_from_first_transaction','client_firstdate']
drop_fea = [col for col in all_data.columns if col not in anaysis_fea]<drop_column>
df = pd.read_csv(path + "/train.csv" )
all_data = all_data.drop(drop_fea,axis=1 )<set_options>
df["path"] = df["image_id"].map(lambda x: path + "/train_images/" + x)
 df = df.drop(columns=["image_id"])
 df = df.sample(frac=1 ).reset_index(drop=True )
gc.collect()<data_type_conversions>
train_df, valid_df = model_selection.train_test_split(
 df, test_size=0.2, random_state=42, stratify=df.label.values
)
all_data['card1'] = all_data['card1'].fillna(0)
 all_data['card2'] = all_data['card2'].fillna(0)
 all_data['card3'] = all_data['card3'].fillna(0)
 all_data['card5'] = all_data['card5'].fillna(0)
 all_data['card4'] = all_data['card4'].fillna('nan')
 all_data['card6'] = all_data['card6'].fillna('nan' )<feature_engineering>
train_df = train_df.reset_index().drop(columns=["index"])
 train_df.head()
all_data['card_hash'] = all_data.apply(lambda x: card_info_hash(x), axis=1 )<filter>
valid_df = valid_df.reset_index().drop(columns=["index"])
 valid_df.head()
def get_data_by_card_hash(data, card_hash):
mask = data['card_hash']==card_hash
return data.loc[mask,:].copy()
def get_data_by_device_hash(data, device_hash):
mask = data['device_hash']==device_hash
return data.loc[mask,:].copy()
def get_data_by_card_and_startdate(data, card_hash, device_hash):
mask =(data['client_firstdate_days']==card_hash)&(data['card_hash']==device_hash)
 return data.loc[mask,:].copy()<groupby>
im = Image.open(train_df["path"][0] )
all_data['count']=1
grp = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['count'].agg('sum')
 display_group = get_data_by_card_and_startdate(all_data,grp[grp>10].index[0][0],grp[grp>10].index[0][1])
 display_group[['DT_time','TransactionAmt','V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo','dist1','dist2','P_emaildomain']]<groupby>
import torch
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.dataset import Subset
from sklearn.model_selection import KFold
import matplotlib.image as img
s = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['isFraud'].agg(['mean', 'count'] )<concatenate>
class CassavaDataset(Dataset):
def __init__(self, dataframe, transform=None):
super().__init__()
self.df = dataframe
self.transform = transform
def __len__(self):
return len(self.df["path"])
 def __getitem__(self, index):
path = self.df["path"][index]
label = self.df["label"][index]
with open(path, "rb")as f:
image = Image.open(f)
 image = image.convert("RGB")
 if self.transform is not None:
image = self.transform(image)
 return image, label
Test_ID=[]
for ind in tqdm(s[(s['mean']==1)].index):
very_strange_thing = get_data_by_card_and_startdate(all_data, ind[0],ind[1])
 Test_ID.extend(very_strange_thing[very_strange_thing['isFraud'].isna() ]['TransactionID'].tolist() )<load_from_csv>
import random
submit = pd.read_csv('.. /input/rank-blend/easy_blend4.csv' )<feature_engineering>
image_size = 512
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
train_transform = transforms.Compose(
 [
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomVerticalFlip(p=0.5),
transforms.RandomResizedCrop(image_size),
make_mask_image(p=0.5, mask_size=50),
transforms.ToTensor() ,
transforms.Normalize(mean=mean, std=std),
] 
)
 valid_transform = transforms.Compose(
 [
transforms.Resize(( image_size, image_size)) ,
transforms.ToTensor() ,
transforms.Normalize(mean=mean, std=std),
]
)
mask = submit['TransactionID'].isin(Test_ID)
 submit.loc[mask,'isFraud'] =1<filter>
dataset = CassavaDataset(train_df, train_transform )
submit.loc[mask,:]<save_to_csv>
path = '.. /input/cassava-leaf-disease-classification/label_num_to_disease_map.json'
with open(path, mode = 'r')as f:
label_to_name = json.load(f )
submit.to_csv('submit_try.csv',index = False )<set_options>
class Unnormalize(object):
def __init__(self, mean, std):
self.mean = mean
self.std = std
def __call__(self, tensor):
for t, m, s in zip(tensor, self.mean, self.std):
t.mul_(s ).add_(m)
 return tensor
warnings.filterwarnings('ignore' )<load_from_csv>
unnorm = Unnormalize(mean, std )
sub_1 = pd.read_csv('.. /input/ieee-simple-lgbm/submission.csv')
 sub_2 = pd.read_csv('.. /input/ieee-cv-options/submission.csv')
 sub_3 = pd.read_csv('.. /input/ieee-lgbm-with-groupkfold-cv/submission.csv')
 sub_4 = pd.read_csv('.. /input/ieee-catboost-baseline-with-groupkfold-cv/submission.csv')
 sub_5 = pd.read_csv('.. /input/mysub18/simple_ensemble30.csv', index_col='TransactionID')
 sub_5.to_csv('submission_old.csv')
 sub_6 = pd.read_csv('.. /input/mysub18/IEEE_add_917_9547.csv', index_col='TransactionID')
 sub_7 = pd.read_csv('.. /input/mysub18/IEEE_add_918_9548.csv', index_col='TransactionID')
 sub_7 = pd.read_csv('.. /input/mysub18/IEEE_version1_version2_kernel_fix.csv', index_col='TransactionID')
 sub_7['isFraud'] += sub_5['isFraud'] * 1.5
print(sub_7)
 sub_7.to_csv('submission.csv')
 sf1 = pd.read_csv('.. /input/mysub18/fe_test2.csv', index_col='TransactionID')
 sub_7 = sub_7.merge(sf1,on='TransactionID',how='left')
 sub_7['isFraud'] = sub_7.groupby('ukey')['isFraud'].transform('mean')
 sub_7[['isFraud']].to_csv('submission_proc.csv' )<load_from_csv>
loader = DataLoader(dataset, 16, shuffle = True)
 display_batch(next(iter(loader)) )
train_transaction = pd.read_csv('.. /input//ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 train_f5 = pd.read_csv('.. /input/mysub18/fi_train4.csv', index_col='TransactionID')
 train_transaction = train_transaction.merge(train_f5, how='left', left_index=True, right_index=True)
 debug = False
if not debug:
test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 test_f5 = pd.read_csv('.. /input/mysub18/fi_test4.csv', index_col='TransactionID')
 test_transaction = test_transaction.merge(test_f5, how='left', left_index=True, right_index=True)
 test_transaction['isFruad'] = 0.0
sub = pd.read_csv('./submission.csv', index_col='TransactionID')
 train_transaction['pred'] = 0.0
test_transaction['pred'] = sub['isFraud']
key = 'card123456_add1_D15_series'
key2 = 'card123456_add1_D2_series'
key3 = 'card123456_add1_D11_series'
if debug:
cache = train_transaction[[key,key2,key3,'isFraud','pred']].values
cache2 = train_transaction['pred'].values
else:
cache = train_transaction.append(test_transaction)[[key, key2,key3, 'isFraud','pred']].values
cache2 = train_transaction.append(test_transaction)['pred'].values 
<define_variables>
import torch
import torch.nn as nn
import torch.nn.functional as F
count00 = 0
count01 = 0
count10 = 0
count11 = 0
ukey_dict = {}
ukey2_dict = {}
ukey3_dict = {}
if debug:
train_len = train_transaction.shape[0] * 4//5
else:
train_len = train_transaction.shape[0]
pred_np = []
for i in range(cache.shape[0]):
ukey = cache[i,0]
ukey2 = cache[i, 1]
ukey3 = cache[i, 2]
isFraud = cache[i,3]
pred = cache[i,4]
ukey_dict[ukey] = ukey_dict.get(ukey,[])
 ukey2_dict[ukey2] = ukey2_dict.get(ukey2, [])
 ukey3_dict[ukey3] = ukey3_dict.get(ukey3, [])
 res = cache2[i]
if i >= train_len and i < train_len + 800000:
pred = 0.5
offset = i - train_len
if int(np.sum(ukey_dict[ukey])> 0.5 * len(ukey_dict[ukey])) + int(np.sum(ukey2_dict[ukey2])> 0.5 * len(ukey2_dict[ukey2])) + int(np.sum(ukey3_dict[ukey3])> 0.5 * len(ukey3_dict[ukey3])) >= 2:
count01 +=1
res += 0.25
elif int(np.sum(ukey_dict[ukey])> 0.5 * len(ukey_dict[ukey])) + int(np.sum(ukey2_dict[ukey2])> 0.5 * len(ukey2_dict[ukey2])) + int(np.sum(ukey3_dict[ukey3])> 0.5 * len(ukey3_dict[ukey3])) >= 1:
pred = 1
count10 += 1
res += 0.10
elif int(np.sum(ukey_dict[ukey])< max(0,0.2 * len(ukey_dict[ukey])))+ int(np.sum(ukey2_dict[ukey2])< 0.2 * max(0,len(ukey2_dict[ukey2])))+ int(np.sum(ukey3_dict[ukey3])< 0.2 * max(0,len(ukey3_dict[ukey3])))> 2:
pred = 0
count11 += 1
res *= 0.2
if debug:
if pred == 0 and isFraud == 0:
count00 +=1
elif pred == 0 and isFraud == 1:
count01 +=1
elif pred == 1 and isFraud == 0:
count10 +=1
elif pred == 1 and isFraud == 1:
count11 +=1
if debug:
pred_np.append(res)
 elif i >= train_len:
pred_np.append(res)
 if i >= train_len:
continue
ukey_dict[ukey].append(isFraud)
 ukey2_dict[ukey2].append(isFraud)
 ukey3_dict[ukey3].append(isFraud)
 print(count00,count01,count10,count11)
 if debug:
train_transaction['pred'] = np.array(pred_np)
 split_pos = train_transaction.shape[0]*4//5
df = train_transaction.iloc[split_pos:,:].copy()
y_test = df['isFraud']
y_pred = df['pred']
print(roc_auc_score(y_test,y_pred))
else:
print(sub['isFraud'])
 print(np.array(pred_np))
sub['isFraud'] = np.array(pred_np)
 sub.to_csv('./sub_group_d2d15d11.csv')
<load_from_csv>
epoch = 3
batch_size = 16
num_classes = 5
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
m15 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m15.csv')
 m17 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m17.csv')
 m18 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m18.csv')
 m19 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m19.csv')
 m20 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m20.csv')
 m16 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m16.csv')
 m0 = pd.read_csv('/kaggle/input/ieee-top-models-blend/m0.csv')
 submission = pd.read_csv("/kaggle/input/ieee-fraud-detection/sample_submission.csv")
 submission.head()<train_on_grid>
resNet = timm.create_model("resnet50", pretrained=False)
 resNet.load_state_dict(torch.load(".. /input/pretrined-models/models/models/pretrained_resNet.pth"))
resNet.fc = nn.Linear(resNet.fc.in_features, num_classes)
 resNet = resNet.to(device )
submission['isFraud'] =(0.20*m15.isFraud)+ \ 
(0.20*m17.isFraud)+ \ 
(0.20*m18.isFraud)+ \ 
(0.20*m19.isFraud)+ \ 
(0.10)*m20.isFraud + \ 
(0.10*m16.isFraud)+ \ 
(0.0*m0.isFraud)
 submission.to_csv('my_blend_5.csv', index=False )<set_options>
ef_model = timm.create_model("tf_efficientnet_b2_ns", pretrained=False)
 ef_model.load_state_dict(torch.load(".. /input/pretrined-models/models/models/pretrained_ef_model.pth"))
ef_model.classifier = nn.Linear(ef_model.classifier.in_features, num_classes)
 ef_model = ef_model.to(device )
pd.options.mode.chained_assignment = None
warnings.filterwarnings("ignore", category = DeprecationWarning)
 %matplotlib inline 
<load_from_csv>
ef_optimizer = torch.optim.AdamW(ef_model.parameters() , lr=1e-4, weight_decay=0.0001)
 ef_scheduler = torch.optim.lr_scheduler.StepLR(ef_optimizer, step_size=2, gamma=0.1)
 resNet_optimizer = torch.optim.AdamW(resNet.parameters() , lr=1e-4, weight_decay=0.0001)
 resNet_scheduler = torch.optim.lr_scheduler.StepLR(resNet_optimizer, step_size=2, gamma=0.1)
 criterion=nn.CrossEntropyLoss()
dailyData = pd.read_csv(".. /input/bike-sharing-demand/train.csv" )<feature_engineering>
def calc_correction(model, df):
model.eval()
path = df["path"]
label = df["label"]
count = 0
pred_list = [0, 0, 0, 0, 0]
for i in tqdm(range(len(path))):
image_path = path[i]
image_label = label[i]
image = Image.open(image_path)
 image = valid_transform(image)
 image = image.unsqueeze(0 ).to(device)
 model = model.to(device)
 pred = model(image ).argmax(1 ).item()
pred_list[pred] += 1
if pred == image_label:
count += 1
percent = count / len(path)
 return percent, pred_list
dailyData.datetime.apply(lambda x:x.split() [0] )<feature_engineering>
def train_model(model, dataset, batch_size, optimizer, criterion, scheduler, epoch, model_title):
best_model = None
best_loss = float("inf")
 train_losses, valid_losses = [], []
kf = KFold(n_splits = 5)
 for fold,(train_index, valid_index)in enumerate(kf.split(dataset)) :
print("fold: ", fold)
 train_dataset = Subset(dataset, train_index)
 train_loader = DataLoader(train_dataset, batch_size, shuffle = True, num_workers = 4)
 valid_dataset = Subset(dataset, valid_index)
 valid_loader= DataLoader(valid_dataset, batch_size, shuffle = False)
 for epoch in range(1, epoch + 1):
epoch_start_time = time.time()
acc = []
train_loss = 0
valid_loss = 0
model.train()
for data, target in train_loader:
data = data.to(device)
 target = target.to(device)
 optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
optimizer.step()
train_loss += loss.item() *len(data)
 train_loss = train_loss/len(train_loader.sampler)
 train_losses.append(train_loss)
 model.eval()
for data, target in valid_loader:
data = data.to(device)
 target = target.to(device)
 with torch.no_grad() :
output = model(data)
 pred =(output.argmax(1)== target)
 acc.append(sum(pred)/len(pred))
loss = criterion(output, target)
 valid_loss += loss.item() *len(data)
 if valid_loss < best_loss:
best_loss = valid_loss
best_model = model
scheduler.step()
collection = sum(acc)/len(acc)
 valid_loss = valid_loss/len(valid_loader.sampler)
 valid_losses.append(valid_loss)
 print('Time: {:.3f}\t Epoch: {} \tTraining Loss: {:.3f} \tValidation Loss: {:.3f} \t Acc: {:.2f}' 
.format(time.time() - epoch_start_time, epoch, train_loss, valid_loss, collection))
num_collection = []
torch.save(best_model.state_dict() , model_title)
 return model, train_losses, valid_losses
dailyData["date"] = dailyData.datetime.apply(lambda x:x.split() [0])
 dailyData["hour"] = dailyData.datetime.apply(lambda x: x.split() [1].split(":")[0])
 dailyData["weekday"] = dailyData.date.apply(lambda dateString: calendar.day_name[datetime.strptime(dateString, "%Y-%m-%d" ).weekday() ])
 dailyData["month"] = dailyData.date.apply(lambda dateString: calendar.month_name[datetime.strptime(dateString, "%Y-%m-%d" ).month])
 dailyData["season"] = dailyData.season.map({1:"Spring", 2:"Summer", 3:"Fall", 4:"winter"})
 dailyData["weather"] = dailyData.weather.map({1: " Clear + Few clouds + Partly cloudy + Partly cloudy",\
2 : " Mist + Cloudy, Mist + Broken clouds, Mist + Few clouds, Mist ", \
3 : " Light Snow, Light Rain + Thunderstorm + Scattered clouds, Light Rain + Scattered clouds", \
4 :" Heavy Rain + Ice Pallets + Thunderstorm + Mist, Snow + Fog " } )<data_type_conversions>
train_models(resNet, ef_model )
categoryvariables = ["hour", "weekday", "month", "season", "weather", "holiday", "workingday"]
for var in categoryvariables:
dailyData[var] = dailyData[var].astype("category" )<drop_column>
ef_model.load_state_dict(torch.load(".. /input/models/ef_model.pth", map_location = device))
resNet.load_state_dict(torch.load(".. /input/models/res_model.pth", map_location = device))
dailyData = dailyData.drop(["datetime"], axis =1 )<create_dataframe>
class CassaveClassifier(nn.Module):
def __init__(self, model, ef_model):
super().__init__()
self.model = model
self.ef_model = ef_model
def forward(self, x):
x1 = self.model(x)
 x2 = self.ef_model(x)
 return(0.5 * x1 + 0.5 * x2)
 def test(self, x, rate):
x1 = self.model(x)
 x2 = self.ef_model(x)
 p = rate * x1 +(1 - rate)* x2
return p
typesCountSerie = dailyData.dtypes.value_counts()
typeNamesColumn = list(map(lambda t: t.name , typesCountSerie.index.values)) ;
typeCountColumn = typesCountSerie.values
intialDataTypeDf = pd.DataFrame({
"variableType": typeNamesColumn,
"count": typeCountColumn
})
 groupedDataTypeDf = intialDataTypeDf.groupby(['variableType'] ).sum() [['count']].reset_index()
fig,ax = plt.subplots()
fig.set_size_inches(12, 5)
 sn.barplot(data=groupedDataTypeDf, x="variableType", y="count", ax=ax)
 ax.set(xlabel = 'variableType', ylabel = 'Count', title = "Count of the different Datatypes" )<filter>
classifier = CassaveClassifier(resNet, ef_model)
 classifier = classifier.to(device )
np.sum(np.abs(dailyData["count"]-dailyData["count"].mean())<=(3*dailyData["count"].std())) 
<filter>
def test_rate() :
for rate in range(1, 10):
classifier.eval()
path = valid_df["path"]
label = valid_df["label"]
count = 0
pred_list = [0, 0, 0, 0, 0]
for i in tqdm(range(len(path))):
image_path = path[i]
image_label = label[i]
image = Image.open(image_path)
 image = valid_transform(image)
 image = image.unsqueeze(0 ).to(device)
 pred = classifier.test(image, rate/10 ).argmax(1 ).item()
pred_list[pred] += 1
if pred == image_label:
count += 1
percent = count / len(path)
 print("rate: ", rate/10)
 print("percent: ", percent )
dailyDataWithoutOutliers = dailyData[np.abs(dailyData["count"]-dailyData["count"].mean())<=(3*dailyData["count"].std())]<load_from_csv>
path = ".. /input/cassava-leaf-disease-classification/test_images/"
dataTrain = pd.read_csv(".. /input/bike-sharing-demand/train.csv")
 dataTest = pd.read_csv(".. /input/bike-sharing-demand/test.csv" )<concatenate>
image_path = []
image_id = []
for i in os.listdir(path):
image_id.append(str(i))
image_path.append(path + str(i))
data = dataTrain.append(dataTest)
 data.reset_index(inplace = True)
 data.drop('index', inplace = True, axis = 1 )<feature_engineering>
pred = []
for path in image_path:
image = Image.open(path)
 image = valid_transform(image)
 image = image.unsqueeze(0 ).to(device)
 predict = resNet(image ).argmax(1 ).item()
pred.append(predict )
data["date"] = data.datetime.apply(lambda x : x.split() [0])
 data["hour"] = data.datetime.apply(lambda x : x.split() [1].split(":")[0] ).astype("int")
 data["year"] = data.datetime.apply(lambda x : x.split() [0].split("-")[0])
 data["weekday"] = data.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).weekday())
 data["month"] = data.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).month)
 dataWindspeedOriginal = data["windspeed"]<feature_engineering>
sub = pd.DataFrame({"image_id": image_id, "label": pred} )
<define_variables><EOS> 
sub.to_csv("submission.csv", index=False )