Kaggle competition: COVID19 Global Forecasting (Week 4)
feature_importance_df = pd.DataFrame()
features = [f for f in data.columns if f != 'sold']
for filename in os.listdir('/kaggle/working/'):
if 'model' in filename:
model = joblib.load(filename)
 store_importance_df = pd.DataFrame()
store_importance_df["feature"] = features
store_importance_df["importance"] = model.feature_importances_
store_importance_df["store"] = filename[5:9]
feature_importance_df = pd.concat([feature_importance_df, store_importance_df], axis=0)
 def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:20].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(averaged over store predictions)')
 plt.tight_layout()
display_importances(feature_importance_df )<feature_engineering>
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission_csv = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
actual = False
if actual == False:
validation = sales[['id']+['d_' + str(i)for i in range(1914,1942)]]
validation['id']=pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv' ).id
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
else:
valid['sold'] = valid_preds
validation = valid[['id','d','sold']]
validation = pd.pivot(validation, index='id', columns='d', values='sold' ).reset_index()
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
validation.id = validation.id.map(d_id ).str.replace('evaluation','validation')
 test['sold'] = eval_preds
evaluation = test[['id','d','sold']]
evaluation = pd.pivot(evaluation, index='id', columns='d', values='sold' ).reset_index()
evaluation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation.id = evaluation.id.map(d_id)
 submit = pd.concat([validation,evaluation] ).reset_index(drop=True)
 submit.to_csv('submission.csv',index=False )<import_modules>
convert_dict = {'Province_State': str,'Country_Region':str,'ConfirmedCases':int,'Fatalities':int}
convert_dict_test = {'Province_State': str,'Country_Region':str}
train_data = train_data.astype(convert_dict)
 test_data = test_data.astype(convert_dict_test )
import pandas as pd<load_from_csv>
train_data['Date'] = pd.to_datetime(train_data['Date'], infer_datetime_format=True)
 test_data['Date'] = pd.to_datetime(test_data['Date'], infer_datetime_format=True )
submission = pd.read_csv('.. /input/m5-three-shades-of-dark-darker-magic/submission_v1.csv' )<feature_engineering>
train_data.loc[:, 'Date'] = train_data.Date.dt.strftime('%m%d')
 train_data.loc[:, 'Date'] = train_data['Date'].astype(int)
 test_data.loc[:, 'Date'] = test_data.Date.dt.strftime('%m%d')
 test_data.loc[:, 'Date'] = test_data['Date'].astype(int )
teams_now = 3215
teams_before = 3186
submission['F1'] *= teams_now/teams_before
submission['F1'].sum()<feature_engineering>
train_data['Country_Region'] = np.where(train_data['Province_State'] == 'nan',train_data['Country_Region'],train_data['Province_State']+' '+train_data['Country_Region'])
 test_data['Country_Region'] = np.where(test_data['Province_State'] == 'nan',test_data['Country_Region'],test_data['Province_State']+' '+test_data['Country_Region'])
sex_constant = 1.0073
submission['F4'] /= sex_constant
submission['F4'].sum()<feature_engineering>
train_data = train_data.drop(columns=['Province_State'])
 test_data = test_data.drop(columns=['Province_State'] )
color_of_the_night = 995874
color_of_the_night /= 1000000
submission['F5'] *= color_of_the_night
submission['F5'].sum()<feature_engineering>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index )
injury = 1.000376
submission['F6'] *= injury
submission['F6'].sum()<feature_engineering>
from sklearn.preprocessing import LabelEncoder
np.random.seed(198505)
 correction = np.random.randint(1000000)/1000000
submission['F7'] *= correction
submission['F7'].sum()<feature_engineering>
label_encoder1 = LabelEncoder()
label_encoder2 = LabelEncoder()
train_data['Country_Region'] = label_encoder2.fit_transform(train_data['Country_Region'])
 test_data['Country_Region'] = label_encoder2.transform(test_data['Country_Region'])
submission['F8'] *=(100-1)/100 +(100-12)/10000
submission['F8'].sum()<prepare_output>
Test_id = test_data.ForecastId
submission['F11'] = submission['F11']
submission['F11'].sum()<find_best_params>
train_data.drop(['Id'], axis=1, inplace=True)
 test_data.drop('ForecastId', axis=1, inplace=True )
for i in range(9,20):
if i!=11:
submission['F'+str(i)] *= 1.01
print(submission['F'+str(i)].sum() )<find_best_params>
missing_val_count_by_column =(train_data.isnull().sum())
 print(missing_val_count_by_column[missing_val_count_by_column>0] )
for i in range(20,29):
submission['F'+str(i)] *= 1.02
print(submission['F'+str(i)].sum() )<save_to_csv>
from xgboost import XGBRegressor
submission.to_csv('submission.csv', index=False )<import_modules>
X_train = train_data[['Country_Region','Date']]
y_train = train_data[['ConfirmedCases', 'Fatalities']]
import cv2
import matplotlib.pyplot as plt
from os.path import isfile
import torch.nn.init as init
import torch
import torch.nn as nn
from PIL import Image, ImageFilter
from sklearn.model_selection import train_test_split, StratifiedKFold
from torch.utils.data import Dataset
from torchvision import transforms
from torch.optim import Adam, SGD, RMSprop
import time
from torch.autograd import Variable
import torch.functional as F
from tqdm import tqdm
from sklearn import metrics
import urllib
import pickle
import cv2
import torch.nn.functional as F
from torchvision import models
import seaborn as sns
import random
import sys<set_options>
x_train = X_train.iloc[:,:].values
x_test = test_data.iloc[:,:].values
package_path = '.. /input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'
sys.path.append(package_path)
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(1234)
 TTA = 5
num_classes = 1
IMG_SIZE = 256
test = '.. /input/aptos2019-blindness-detection/test_images/'
def expand_path(p):
p = str(p)
 if isfile(test + p + ".png"):
return test +(p + ".png")
 return p
def p_show(imgs, label_name=None, per_row=3):
n = len(imgs)
 rows =(n + per_row - 1)//per_row
cols = min(per_row, n)
 fig, axes = plt.subplots(rows,cols, figsize=(15,15))
for ax in axes.flatten() : ax.axis('off')
 for i,(p, ax)in enumerate(zip(imgs, axes.flatten())) :
img = Image.open(expand_path(p))
ax.imshow(img)
 ax.set_title(train_df[train_df.id_code == p].diagnosis.values)
 def crop_image1(img,tol=7):
mask = img>tol
return img[np.ix_(mask.any(1),mask.any(0)) ]
def crop_image_from_gray(img,tol=7):
if img.ndim ==2:
mask = img>tol
return img[np.ix_(mask.any(1),mask.any(0)) ]
elif img.ndim==3:
gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 mask = gray_img>tol
check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ].shape[0]
if(check_shape == 0):
return img
else:
img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ]
img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0)) ]
img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0)) ]
img = np.stack([img1,img2,img3],axis=-1)
 return img
class MyDataset(Dataset):
def __init__(self, dataframe, transform=None):
self.df = dataframe
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
label = self.df.diagnosis.values[idx]
label = np.expand_dims(label, -1)
 p = self.df.id_code.values[idx]
p_path = expand_path(p)
 image = cv2.imread(p_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = crop_image_from_gray(image)
 image = cv2.resize(image,(IMG_SIZE, IMG_SIZE))
image = cv2.addWeighted(image,4, cv2.GaussianBlur(image ,(0,0), 30),-4 ,128)
 image = transforms.ToPILImage()(image)
 if self.transform:
image = self.transform(image)
 return image, label
test_transform = transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(( -120, 120)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 testset = MyDataset(pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv'),
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(testset, batch_size=16, shuffle=False)
 model = EfficientNet.from_name('efficientnet-b0')
 in_features = model._fc.in_features
model._fc = nn.Linear(in_features, num_classes)
 model.load_state_dict(torch.load('.. /input/enet-test/weight_best(3 ).pt'))
model.cuda()
for param in model.parameters() :
param.requires_grad = False
sample = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 test_pred = np.zeros(( len(sample), 1))
model.eval()
for _ in range(TTA):
with torch.no_grad() :
for i, data in tqdm(enumerate(test_loader)) :
images, _ = data
images = images.cuda()
pred = model(images)
 test_pred[i * 16:(i + 1)* 16] += pred.detach().cpu().squeeze().numpy().reshape(-1, 1)
 output = test_pred / TTA
preds1 = output.copy()
coef = [0.57, 1.37, 2.57, 3.57]
for i, pred in enumerate(output):
if pred < coef[0]:
output[i] = 0
elif pred >= coef[0] and pred < coef[1]:
output[i] = 1
elif pred >= coef[1] and pred < coef[2]:
output[i] = 2
elif pred >= coef[2] and pred < coef[3]:
output[i] = 3
else:
output[i] = 4
submission1 = pd.DataFrame({'id_code':pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' ).id_code.values,
'diagnosis':np.squeeze(output ).astype(int)} )<set_options>
package_dir = ".. /input/pretrained-models/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, package_dir)
 device = torch.device("cuda:0")
 ImageFile.LOAD_TRUNCATED_IMAGES = True<load_from_csv>
error_list = []
def return_error(estimator, x_train,x_test,y_train):
model = MultiOutputRegressor(XGBRegressor(n_estimators=estimator, random_state=0, max_depth=20))
model.fit(x_train, y_train)
 predict = MultiOutputRegressor(model.predict(x_test))
return predict
class RetinopathyDatasetTest(Dataset):
def __init__(self, csv_file, transform):
self.data = pd.read_csv(csv_file)
 self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join('.. /input/aptos2019-blindness-detection/test_images', self.data.loc[idx, 'id_code'] + '.png')
 image = Image.open(img_name)
 image = self.transform(image)
 return {'image': image}
model = pretrainedmodels.__dict__['resnet101'](pretrained=None)
 model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.last_linear = nn.Sequential(
 nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.Dropout(p=0.25),
nn.Linear(in_features=2048, out_features=2048, bias=True),
nn.ReLU() ,
nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.Dropout(p=0.5),
nn.Linear(in_features=2048, out_features=1, bias=True), 
)
 model.load_state_dict(torch.load(".. /input/mmmodel/model.bin"))
model = model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
test_transform = transforms.Compose([
transforms.Resize(( 224, 224)) ,
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 test_dataset = RetinopathyDatasetTest(csv_file='.. /input/aptos2019-blindness-detection/sample_submission.csv',
transform=test_transform)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds1 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds1[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds2 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds2[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds3 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds3[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds4 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds4[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds5 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds5[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds6 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds6[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds7 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds7[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds8 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds8[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds9 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds9[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)
 test_preds10 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds10[i * 32:(i + 1)* 32] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_preds =(test_preds1 + test_preds2 + test_preds3 + test_preds4 + test_preds5
+ test_preds6 + test_preds7 + test_preds8 + test_preds9 + test_preds10)/ 10.0
coef = [0.5, 1.5, 2.5, 3.5]
preds2 = test_preds.copy()
for i, pred in enumerate(test_preds):
if pred < coef[0]:
test_preds[i] = 0
elif pred >= coef[0] and pred < coef[1]:
test_preds[i] = 1
elif pred >= coef[1] and pred < coef[2]:
test_preds[i] = 2
elif pred >= coef[2] and pred < coef[3]:
test_preds[i] = 3
else:
test_preds[i] = 4
submission2 = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 submission2.diagnosis = test_preds.astype(int )<import_modules>
predict = return_error(1500,x_train,x_test,y_train )
<load_from_csv><EOS> 
df_sub = pd.DataFrame()
df_sub['ForecastId'] = Test_id
df_sub['ConfirmedCases'] =(predict.estimator[:,0] ).astype(int)
 df_sub['Fatalities'] =(predict.estimator[:,1] ).astype(int)
 df_sub.to_csv('submission.csv', index=False )