Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
pd.options.display.max_rows=200
pd.set_option('mode.chained_assignment', None)
 simplefilter("ignore", category=ConvergenceWarning)
 simplefilter("ignore", category=RuntimeWarning)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print('Train data dimension: ', train_df.shape)
 display(train_df.head())
 print('Test data dimension: ', test_df.shape)
 display(test_df.head() )<set_options>
train = pd.read_csv('/kaggle/input/titanic/train.csv', index_col='PassengerId')
 test = pd.read_csv('/kaggle/input/titanic/test.csv', index_col='PassengerId' )
def seed_torch(seed=1234):
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<compute_test_metric>
y_train = train.Survived.copy()
train = train.drop('Survived', axis=1)
 X_test = test
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<compute_test_metric>
def name_labeling(df):
for i in ['Mr.', 'Mrs.', 'Miss', 'Master', 'Army', 'Revered/Important', 'rare', 'Doctor']:
if i == 'Army':
df.Name[df.Name.str.contains(pat='(Major.|Col.|Capt.) ', regex=True)] = 'Army'
elif i == 'Revered/Important':
df.Name[df.Name.str.contains(pat='(Rev.|Countess.|Jonkheer.|Sir.|Lady.) ', regex=True)] = 'Revered/Important'
elif i == 'rare':
df.Name[df.Name.str.contains(pat='(Mme.|Ms.|Mlle.|Don.|Dona.) ', regex=True)] = 'rare'
elif i == 'Doctor':
df.Name[df.Name.str.contains(pat='Dr.')] = 'doctor'
else:
df.Name[df.Name.str.contains(pat=f'{i}', regex=False)] = i
return df
def sigmoid(x):
return 1 /(1 + np.exp(-x))<define_variables>
def ticket_labeling(df):
for label, pattern in [('ca', 'C[.]?A[.]?'),('soton', 'SOTON'),('ston', 'STON'),('wc', 'W[.]?[/]?C'), 
('sc', 'S[.]?C[.]?'),('a', 'A[.]?'),('soc', 'S[.]?O[.]?[/]?C'),('pp', 'PP'), 
('fc', '(F.C.|F.C.C.) '),('rest_char', '[A-Z]'),('small_serial_number', '^\d{3,5}$'), 
('large_serial_number', '^\d{6,7}$')]:
df.Ticket[df.Ticket.str.contains(pattern)] = label
return df
embed_size = 300
max_features = 75000
maxlen = 50<define_variables>
def cabin_labeling(df):
for i in ['A', 'B', 'C', 'D', 'E', 'F', 'G']:
df.Cabin[df.Cabin.str.contains(i, na=False)] = i
df.Cabin.fillna('missing', inplace=True)
 return df
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<feature_engineering>
temp = cabin_labeling(train.copy() )
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
x_train = train_df["question_text"].fillna("_
x_test = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 y_train = train_df['target'].values<statistical_test>
temp = temp.groupby(['Pclass', 'Cabin'])[['Name']].count().rename(columns={'Name':'Passengers'} )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
temp = temp.reset_index()
temp_no_missing_value = temp[temp.Cabin != 'missing']
temp_missing_value = temp[temp.Cabin == 'missing']
glove_embeddings = load_glove(tokenizer.word_index)
 paragram_embeddings = load_para(tokenizer.word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 np.shape(embedding_matrix )<split>
def combined_labeling(df):
return cabin_labeling(ticket_labeling(name_labeling(df)) )
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10 ).split(x_train, y_train))<normalization>
X_train = combined_labeling(train.copy())
 X_test = combined_labeling(test.copy() )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<define_variables>
def proportions(df):

 df = df.groupby(['Pclass', 'Cabin'])['Name'].count().reset_index().rename(columns={'Name':'Passengers'})
 total_passengers_in_cabins = df.Passengers[df.Cabin != 'missing'].sum()
cabin_proportions = df['Passengers'][df.Cabin != 'missing'] / total_passengers_in_cabins
return cabin_proportions
def no_cabins(df):

 df = df.groupby(['Pclass', 'Cabin'])['Name'].count().reset_index().rename(columns={'Name':'Passengers'})
 return df.Passengers[df.Cabin == 'missing']
def cabin_imputer(df, missing_vals, cabin_proportions):
if all(df.Pclass == 1):
imputation_ndarray = np.random.choice(['A', 'B', 'C', 'D', 'E', 'T'], size=missing_vals[0], p=cabin_proportions[1])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
elif all(df.Pclass == 2):
imputation_ndarray = np.random.choice(['D', 'E', 'F'], size=missing_vals[1], p=cabin_proportions[2])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
elif all(df.Pclass == 3):
imputation_ndarray = np.random.choice(['E', 'F', 'G'], size=missing_vals[2], p=cabin_proportions[3])
 missing_values_index = df[df.Cabin == 'missing'].index
imputation_series = pd.Series(imputation_ndarray, index=missing_values_index)
 return imputation_series
def imputer(df):
df.Age.fillna(df.groupby(['Pclass', 'Sex'] ).Age.transform('mean'), inplace=True)
 cabin_proportions = X_train.groupby(['Pclass'] ).apply(proportions)
 missing_vals = list(df.groupby(['Pclass'] ).apply(no_cabins))
missing_val_imputed_series = df.groupby(['Pclass'] ).apply(partial(cabin_imputer,
missing_vals=missing_vals,
cabin_proportions=cabin_proportions))
missing_val_imputed_series = missing_val_imputed_series.reset_index().set_index('PassengerId')
 missing_val_imputed_series = missing_val_imputed_series.drop('Pclass', axis=1 ).sort_index().iloc[:, 0]
for index, val in zip(missing_val_imputed_series.index, missing_val_imputed_series):
df.Cabin[index] = val
for feature in df.columns:
df[feature].fillna(df[feature].mode() [0], inplace=True)
 df.Cabin[df.Cabin == 'T'] = 'E'
df.reset_index(drop=True, inplace=True)
 return df
128*4<define_variables>
X_train_imputed = imputer(X_train.copy())
 X_test_imputed = imputer(X_test.copy() )
batch_size = 512
n_epochs = 5<choose_model_class>
class FeatureEngineering(BaseEstimator, TransformerMixin):
def __init__(self, drop_Cabin=False, drop_Name=False, Embarked_target=False, SibSp_Parch_simplify=True,
drop_Ticket=False, scaler='MinMaxScaler', smoothing=10, test=False):
self.drop_Cabin = drop_Cabin
self.drop_Name = drop_Name
self.drop_Ticket = drop_Ticket
self.Embarked_target = Embarked_target
self.SibSp_Parch_simplify = SibSp_Parch_simplify
self.scaler = scaler
self.smoothing = smoothing
self.target_encoder = None
self.test = test
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Pclass'] = X.Pclass.astype(str)
 dummies = pd.get_dummies(X.loc[:, ['Pclass', 'Sex']])
 X = pd.concat([X, dummies], axis=1 ).drop(['Pclass', 'Sex'], axis=1)
 if not self.test and len(X)> 400:
self.cols = ['Ticket', 'Name', 'Cabin'] if not self.Embarked_target else ['Cabin', 'Name', 'Ticket', 'Embarked']
self.target_encoder = TargetEncoder(cols=self.cols, smoothing=self.smoothing)
 self.target_encoder.fit(X.loc[:, self.cols], y_train.iloc[X.index])
 X.loc[:, self.cols] = self.target_encoder.transform(X.loc[:, self.cols])
 else:
X.loc[:, self.cols] = self.target_encoder.transform(X.loc[:, self.cols])
 if self.drop_Cabin:
X.drop('Cabin', axis=1, inplace=True)
 if self.drop_Name:
X.drop('Name', axis=1, inplace=True)
 if self.drop_Ticket:
X.drop('Ticket', axis=1, inplace=True)
 if self.SibSp_Parch_simplify:
X['SibSp'].replace({0:'No', 1:'Yes', 2:'Yes', 3:'Yes', 4:'Yes',
5:'Yes', 6:'Yes', 7:'Yes', 8:'Yes', 9:'Yes'}, inplace=True)
 X['Parch'].replace({0:'No', 1:'Yes', 2:'Yes', 3:'Yes', 4:'Yes',
5:'Yes', 6:'Yes', 7:'Yes', 8:'Yes', 9:'Yes'}, inplace=True)
 dummies = pd.get_dummies(X.loc[:, ['SibSp', 'Parch']])
 X = pd.concat([X, dummies], axis=1 ).drop(['SibSp', 'Parch'], axis=1)
 X = pd.get_dummies(X)
 features_to_scale = [feature for feature in X.columns if X[feature].max() != 1 and X[feature].min() != 1]
if not self.test and len(X)> 400:
if self.scaler == 'StandardScaler':
self.scale_transformer = StandardScaler()
elif self.scaler == 'MinMaxScaler':
self.scale_transformer = MinMaxScaler()
elif self.scaler == 'RobustScaler':
self.scale_transformer = RobustScaler()
scaled_features = self.scale_transformer.fit_transform(X.loc[:, features_to_scale])
 X.loc[:, features_to_scale] = pd.DataFrame(data=scaled_features, columns=features_to_scale, index=X.index)
 else:
scaled_features = self.scale_transformer.fit_transform(X.loc[:, features_to_scale])
 X.loc[:, features_to_scale] = pd.DataFrame(data=scaled_features, columns=features_to_scale, index=X.index)
 return X
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(1024, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<define_variables>
def results(cv_results_, n):
df = pd.DataFrame(cv_results_)[['params', 'mean_test_score']].nlargest(n, columns='mean_test_score')
 for i in range(len(df)) :
print(f'{df.iloc[i, 0]} : {df.iloc[i, 1]}' )
train_preds = np.zeros(( len(train_df)))
 test_preds = np.zeros(( len(test_df)))
 seed_torch()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_df)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
param_grid = {'feature_engineering__drop_Cabin':[True, False],
'feature_engineering__drop_Ticket':[True, False],
'feature_engineering__drop_Name':[True, False],
'feature_engineering__Embarked_target':[True, False],
'feature_engineering__SibSp_Parch_simplify':[False],
'feature_engineering__scaler':['StandardScaler'],
'feature_engineering__smoothing':[5]}
feature_engineering_pipeline = Pipeline([('feature_engineering', FeatureEngineering()), 
('model', LogisticRegression())])
 grid = GridSearchCV(feature_engineering_pipeline, param_grid,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
search_result = threshold_search(y_train, train_preds)
 search_result<save_to_csv>
grid.fit(X_train_imputed.copy() , y_train )
submission = test_df[['qid']].copy()
submission['prediction'] = test_preds > search_result['threshold']
submission.to_csv('submission.csv', index=False )<feature_engineering>
fe = FeatureEngineering(drop_Cabin=True, drop_Name=False, drop_Ticket=False, Embarked_target=False, SibSp_Parch_simplify=False,
scaler='StandardScaler', smoothing=5 )
train=pd.read_json('.. /input/train.json')
 test=pd.read_json('.. /input/test.json')
 train['ingredients'] = [", ".join(ingredients)for ingredients in train['ingredients']]
test['ingredients']=[", ".join(ingredients)for ingredients in test['ingredients']]
def multiclass_logloss(actual, predicted, eps=1e-15):

 if len(actual.shape)== 1:
actual2 = np.zeros(( actual.shape[0], predicted.shape[1]))
for i, val in enumerate(actual):
actual2[i, val] = 1
actual = actual2
clip = np.clip(predicted, eps, 1 - eps)
 rows = actual.shape[0]
vsota = np.sum(actual * np.log(clip))
return -1.0 / rows * vsota
lblencdr=preprocessing.LabelEncoder()
y_cuisine=lblencdr.fit_transform(train['cuisine'].values)
 X_train, X_val, y_train, y_val = train_test_split(train['ingredients'].values, y_cuisine,
stratify=y_cuisine,
test_size=0.1,
shuffle=True,
random_state=0)
 vect=TfidfVectorizer().fit(list(X_train)+list(X_val))
X_train_vect=vect.transform(X_train)
 x_valid_vect=vect.transform(X_val)
 ftwo_scorer=metrics.make_scorer(fbeta_score, average='micro', beta=0.5)
 param_grid={
'kernel': ['rbf'],
'C': [0.1, 1.0, 100],
'gamma': [0,1,10]
}
classifier=SVC()
model=GridSearchCV(estimator=classifier, param_grid=param_grid, scoring=ftwo_scorer,
verbose=10, n_jobs=4, cv=5)
 model.fit(X_train_vect, y_train)
 print("Best score: %0.3f" % model.best_score_)
 print("Best Parameters set:")
 best_parameters=model.best_estimator_.get_params()
for param_name in sorted(param_grid.keys()):
print("\t%s: %r" %(param_name, best_parameters[param_name]))
predictions=model.predict(vect.transform(X_val))
print(f1_score(predictions, y_val, average='micro'))
y_predict=model.predict(vect.transform(test['ingredients']))
test['cuisine']=lblencdr.inverse_transform(y_predict)
 test = test.sort_values('id' , ascending=True)
 test[['id' , 'cuisine' ]].to_csv("submission.csv", index=False )<load_from_csv>
X_train_fe = fe.fit_transform(X_train_imputed.copy() )
warnings.filterwarnings('ignore')
 sub0 = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv', parse_dates=['Date'])
 train.shape, test.shape<merge>
fe.test = True
know = test[test.Date <= train.Date.max() ]
not_know = test[test.Date > train.Date.max() ]
know = know.merge(train, on=['Date','Country_Region','Province_State'], how='left')
 know.head()<feature_engineering>
X_test_fe = fe.transform(X_test_imputed.copy() )
train['days'] =(train['Date'] - train.Date.min() ).dt.days
train['location'] = train['Country_Region'] + ' ' + train['Province_State'].fillna('')
 train['location'] = train['location'].str.strip()
not_know['days'] =(not_know['Date'] - train.Date.min() ).dt.days
not_know['location'] = not_know['Country_Region'] + ' ' + not_know['Province_State'].fillna('')
 not_know['location']= not_know['location'].str.strip()<feature_engineering>
def parameter_plot(model, X, y, n_estimators=[100, 200, 300, 400, 600, 900, 1300, 1700, 2000, 2500], hyper_param=None, **kwargs):
param_name, param_vals = hyper_param
param_grid = {'n_estimators':n_estimators,
f'{param_name}':param_vals}
grid = GridSearchCV(model(**kwargs), param_grid,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', n_jobs=-1, verbose=2)
 grid.fit(X, y)
 results = pd.DataFrame(grid.cv_results_)['mean_test_score'].values
results = results.reshape(len(param_vals), len(n_estimators))
plt.figure(figsize=(15, 9))
for i in range(1, len(param_vals)+ 1):
plt.plot(n_estimators, results[i-1], label=f'{param_name} - {param_vals[i-1]}')
 plt.legend()
plt.show()
def get_param(loc):
_ = train[train.location == loc]
_['diff'] = _.ConfirmedCases.diff()
_['pct'] = _.ConfirmedCases.pct_change()
initial_speed = _.loc[_.ConfirmedCases.diff().argmax() ,'pct']
initial_mid = _.loc[_.ConfirmedCases.diff().argmax() , 'days']
initial_max = _.ConfirmedCases.max() * 2.1
return initial_speed, initial_mid, initial_max
get_param('Finland' )<train_on_grid>
def learning_curve_plotter(Model, X, y, params_1, params_2, step=50):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
 plt.figure(figsize=(16, 7))
for i,(name, params)in enumerate([params_1, params_2]):
train_score = []
val_score = []
plt.subplot(1, 2, i+1)
 for j in range(100, len(X_train), step):
model = Model(**params ).fit(X_train[:j], y_train[:j])
 train_score.append(model.score(X_train[:j], y_train[:j]))
val_score.append(model.score(X_test, y_test))
plt.plot(train_score, 'r-', label='Training accuracy')
 plt.plot(val_score, 'b-', label='Validation accuracy')
 plt.title(f'{name}')
 plt.xlabel('Training set size')
 plt.ylabel('Accuracy')
 plt.legend()
plt.show()
loc_list = train.location.unique()
all_param = pd.DataFrame(index=loc_list, columns=['k','x_0','y_max'])
 for loc in loc_list:
_ = train[train.location == loc]
nn = not_know[not_know.location == loc]
initial_max = _.ConfirmedCases.max() *2
x = _.days
y1 = _.ConfirmedCases
try:
popt, pcov = opt.curve_fit(log_curve, x, y1, p0 = get_param(loc))
popt[2] = max(popt[2],get_param(loc)[2]/2)
 except RuntimeError:
popt = get_param(loc)
 y1_hat = log_curve(nn.days, *popt)
 not_know.loc[y1_hat.index, 'ConfirmedCases'] = y1_hat
all_param.loc[loc, 'k'] = popt[0]
all_param.loc[loc, 'x_0'] = popt[1]
all_param.loc[loc, 'y_max'] = popt[2]
print('Done' )<filter>
param_grid_logreg = {'penalty':['elasticnet'],
'C':[0.03],
'l1_ratio':[0.0],
'solver':['saga']}
all_param.loc['Finland', 'y_max']<merge>
grid_logreg = GridSearchCV(LogisticRegression() , param_grid_logreg,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
latest = train[train.Date == train.Date.max() ]
latest['DeathRate'] = latest['Fatalities'] / latest['ConfirmedCases']
not_know2 = not_know.merge(latest[['location','DeathRate']], on='location')
 not_know2['Fatalities'] = not_know2['ConfirmedCases'] * not_know2['DeathRate']*1.1<feature_engineering>
grid_logreg.fit(X_train_fe, y_train )
not_know2['ConfirmedCases'] = not_know2['ConfirmedCases'].round()
not_know2['Fatalities'] = not_know2['Fatalities'].round()<save_to_csv>
param_grid_knn = {'n_neighbors':[20],
'weights':['uniform'],
'algorithm':['ball_tree']}
sub1 = pd.concat([know[['ForecastId', 'ConfirmedCases','Fatalities']],
not_know2[['ForecastId', 'ConfirmedCases','Fatalities']]])
 sub1=sub1.sort_values('ForecastId' ).reset_index(drop=True)
 sub1.to_csv('submission.csv', index=False )<load_from_csv>
grid_knn = GridSearchCV(KNeighborsClassifier() , param_grid_knn,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
PATH_WEEK4='/kaggle/input/covid19-global-forecasting-week-4'
df_train = pd.read_csv(f'{PATH_WEEK4}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK4}/test.csv' )<load_from_csv>
grid_knn.fit(X_train_fe, y_train )
mitigation = pd.read_csv('/kaggle/input/mitigation-day/mitigations.csv')
 mitigation.loc[mitigation['Name']=="United States",'Name']='US'
mitigation['start'] = pd.to_datetime(mitigation['start'], infer_datetime_format=True )<data_type_conversions>
param_grid_svc = {'C':[0.5],
'kernel':['rbf'],
'gamma':[0.1]}
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )<feature_engineering>
grid_svc = GridSearchCV(SVC() , param_grid_svc, cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
NULL_VAL = "NULL_VAL"
def fillState(state, country):
if state == NULL_VAL:
return country
return state +':' + country
def fillState2(state, country):
if type(state)==str:
return country
return state +':' + country
X_Train = df_train.loc[:, ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
X_Train['State'].fillna(NULL_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 firstDay = np.min(X_Train['Date'])
 X_Train.loc[:, 'Date'] =(X_Train['Date']-np.min(X_Train['Date'])).values / 86400000000000
X_Train["Date"] = X_Train["Date"].astype(int)
 X_Test = df_test.loc[:, ['State', 'Country', 'Date', 'ForecastId']]
X_Test['State'].fillna(NULL_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] =(X_Test['Date']-firstDay ).values / 86400000000000
X_Test["Date"] = X_Test["Date"].astype(int )<data_type_conversions>
grid_svc.fit(X_train_fe, y_train )
mitigation['start_day'] =(( mitigation['start'] - firstDay ).values / 86400000000000 ).astype(int )<categorify>
param_grid_random = {'n_estimators':[200, 500],
'max_depth':[5, 9],
'max_samples':[0.5, 0.7],
'max_features':[0.5, 0.7],
'min_samples_split':[2, 5, 8]}
le = LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
condition_train =(X_Train.Country == country)&(X_Train.State == state)
 X_Train_CS = X_Train.loc[condition_train, ['Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['Date']]
condition_test =(X_Test.Country == country)&(X_Test.State == state)
 X_Test_CS = X_Test.loc[condition_test, ['Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['Date']]
row = mitigation[mitigation['State'] == state]
if row.shape[0]>0:
day = row['start_day'].values[0]
print("use mitigation", state, day)
 X_Train_CS['Mitigation'] =(X_Train_CS['Date'] - day)> 0
X_Train_CS['Mitigation_day'] = X_Train_CS['Date'] - day
X_Train_CS.loc[X_Train_CS['Mitigation_day']<0, 'Mitigation_day'] = 0
X_Test_CS['Mitigation'] =(X_Test_CS['Date'] - day)> 0
X_Test_CS['Mitigation_day'] = X_Test_CS['Date'] - day
X_Test_CS.loc[X_Test_CS['Mitigation_day']<0, 'Mitigation_day'] = 0
else:
X_Train_CS['Mitigation'] = False
X_Train_CS['Mitigation_day'] = 0
X_Test_CS['Mitigation'] = False
X_Test_CS['Mitigation_day'] = 0
model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
 df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()<save_to_csv>
grid_random = GridSearchCV(RandomForestClassifier() , param_grid_random,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=4 )
df_out.to_csv('submission.csv', index=False )<load_from_csv>
grid_random.fit(X_train_fe, y_train )
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 train.rename(columns={'Country_Region':'Country'}, inplace=True)
 train.rename(columns={'Province_State':'State'}, inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 train['Date'] = train.Date.dt.strftime("%m%d")
 train['Date'] = train['Date'].astype(int)
 train["State"].fillna("",inplace=True)
 train["CountryState"] = train["Country"] + train["State"]
print("train done")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 test.rename(columns={'Country_Region':'Country'}, inplace=True)
 test.rename(columns={'Province_State':'State'}, inplace=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
 test['Date'] = test.Date.dt.strftime("%m%d")
 test['Date'] = test['Date'].astype(int)
 test["State"].fillna("",inplace=True)
 test["CountryState"] = test["Country"] + test["State"]
print("test done")
<categorify>
param_grid_gradient = {'max_depth':[3],
'n_estimators':[300, 400, 500],
'learning_rate':[0.035, 0.055],
'subsample':[0.4, 0.6],
'max_features':[0.4, 0.6],
'min_samples_split':[2, 5, 8, 12]
}
%%time
filterwarnings('ignore')
 le = LabelEncoder()
finaloutput = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 CountryState = train.CountryState.unique()
for CS in CountryState:
trainIndia = train[train["CountryState"] == CS]
testIndia = test[test["CountryState"] == CS]
trainIndia.CountryState = le.fit_transform(trainIndia.CountryState)
 X = trainIndia[['CountryState', 'Date']]
Y = trainIndia[['ConfirmedCases']]
eval_set = [(X,Y)]
model1 = boostmodel(learning_rate=0.3,silent=0, n_estimators=1000)
 model1.fit(X, Y,eval_set=eval_set,early_stopping_rounds=100)
 testX = testIndia[['CountryState', 'Date']]
testX.CountryState = le.fit_transform(testX.CountryState)
 ConfirmedCases_Pred = model1.predict(testX)
 X = trainIndia[['CountryState', 'Date']]
Y = trainIndia[['Fatalities']]
eval_set = [(X,Y)]
model2 = boostmodel(learning_rate=0.3,silent=0, n_estimators=1000)
 model2.fit(X, Y,eval_set=eval_set,early_stopping_rounds=100)
 testX = testIndia[['CountryState', 'Date']]
testX.CountryState = le.fit_transform(testX.CountryState)
 Fatalities_Pred = model2.predict(testX)
 XForecastId = testIndia.loc[:, 'ForecastId']
output = pd.DataFrame({'ForecastId': XForecastId, 'ConfirmedCases': ConfirmedCases_Pred, 'Fatalities': Fatalities_Pred})
 finaloutput = pd.concat([finaloutput, output], axis=0)
 print("Program completed" )<feature_engineering>
grid_gradient = GridSearchCV(GradientBoostingClassifier() , param_grid_gradient,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
finaloutput.ConfirmedCases.apply(math.floor )<save_to_csv>
grid_gradient.fit(X_train_fe, y_train )
finaloutput.ForecastId = finaloutput.ForecastId.astype('int')
 finaloutput.ConfirmedCases = round(finaloutput.ConfirmedCases,1)
 finaloutput.Fatalities = round(finaloutput.Fatalities,1)
 finaloutput = finaloutput[['ForecastId','ConfirmedCases','Fatalities']]
finaloutput.to_csv("submission.csv",index=False)
 print("done" )<import_modules>
param_grid_xgb = {'n_estimators':[300, 450],
'learning_rate':[0.02, 0.03],
'max_depth':[6],
'subsample':[0.5, 0.7],
'colsample_bylevel':[0.5, 0.7],
'reg_lambda':[1, 5, 15, ]
}
import numpy as np
import pandas as pd<load_from_csv>
grid_xgb = GridSearchCV(XGBClassifier() , param_grid_xgb,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
X_train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 X_test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 X_submission = pd.read_csv('.. /input/covid19-global-forecasting-week-4/submission.csv' )<data_type_conversions>
grid_xgb.fit(X_train_fe, y_train )
X_train['Date'] = pd.to_datetime(X_train['Date'])
 X_test['Date'] = pd.to_datetime(X_test['Date'])
 X_test['Date']<count_unique_values>
logreg = LogisticRegression(**{'C': 0.03, 'l1_ratio': 0, 'penalty': 'elasticnet', 'solver': 'saga'})
 svc = SVC(**{'C': 0.5, 'gamma': 0.1, 'kernel': 'rbf'})
 knn = KNeighborsClassifier(**{'algorithm': 'ball_tree', 'n_neighbors': 20, 'weights': 'uniform'})
 rfc = RandomForestClassifier(**{'max_depth': 5, 'max_features': 0.5, 'max_samples': 0.7,
'min_samples_split': 5, 'n_estimators': 500})
 gradient = GradientBoostingClassifier(**{'learning_rate': 0.055, 'max_depth': 3, 'max_features': 0.4,
'min_samples_split': 2, 'n_estimators': 400, 'subsample': 0.4})
 xgb = XGBClassifier(**{'colsample_bylevel': 0.5, 'learning_rate': 0.03, 'max_depth': 6,
'n_estimators': 300, 'reg_lambda': 5, 'subsample': 0.7})
 estimators = [('logreg', logreg),('knn', knn),('svc', svc),('rfc', rfc),('gradient', gradient), 
('xgb', xgb)]
stack = StackingClassifier(estimators=estimators,
cv=10, n_jobs=-1 )
print(X_train.Country_Region.nunique() )<define_variables>
stack.fit(X_train_fe, y_train )
countries_no_province = [i for i in countries if i not in countries_with_provinces]
len(countries_no_province )<data_type_conversions>
y_preds = stack.predict(X_test_fe )
X_train['Province_State'] = X_train['Province_State'].fillna('unknown')
 X_test['Province_State'] = X_test['Province_State'].fillna('unknown' )<groupby>
submission = pd.DataFrame({'PassengerId':test.index,
'Survived':y_preds} )
X_train[X_train['Country_Region'].isin(countries_with_provinces)].groupby(['Country_Region'] ).agg({'Province_State':'nunique'} )<data_type_conversions>
submission.to_csv('submission.csv', index=False )
X_train['Date'] = X_train['Date'].dt.strftime("%m%d")
 X_train['Date'] = X_train['Date'].astype(int)
 X_test['Date'] = X_test['Date'].dt.strftime("%m%d")
 X_test['Date'] = X_test['Date'].astype(int )<data_type_conversions>
submission.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
pd.read_csv('submission.csv' )