Kaggle competition: StumbleUpon Evergreen Classification Challenge
<SOS> metric: AUC Kaggle data source: stumbleupon-evergreen-classification-challenge<merge>
%matplotlib inline
def calculate_days_since_event(df, feature_name, casualties, casualties_amount, groupby=["country"]):
cases_df = df.loc[df[casualties] > casualties_amount][groupby + ["Date"]].groupby(groupby, as_index=False ).min()
cases_df = cases_df.rename({"Date": "relevant_date"}, axis=1)
 df = pd.merge(df, cases_df, on=groupby, how="left")
 df[feature_name] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["relevant_date"])).dt.days
df.loc[df[feature_name] < 0, feature_name] = 0
df = df.drop("relevant_date", axis = 1)
 return df<feature_engineering>
df_train=pd.read_csv('/kaggle/input/stumbleupon/train.tsv',sep='\t')
 df_test=pd.read_csv('/kaggle/input/stumbleupon/test.tsv',sep='\t',usecols=['urlid','boilerplate'])
def generate_time_features(df):
df = calculate_days_since_event(df, "days_from_first_death", "Fatalities", 0, ["country"])
 df = calculate_days_since_event(df, "days_from_first_case", "ConfirmedCases", 0, ["country"])
 df = calculate_days_since_event(df, "days_from_first_case_province", "ConfirmedCases", 0, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_first_death_province", "Fatalities", 0, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_case", "ConfirmedCases", 99, ["country"])
 df = calculate_days_since_event(df, "days_from_centenary_case_province", "Fatalities", 99, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_daily_cases_province", "ConfirmedCases_daily", 99, ["country", "state"])
 df = calculate_days_since_event(df, "days_from_centenary_daily_cases", "ConfirmedCases_daily", 99, ["country"])
 df["days_from_first_ever_case"] =(pd.to_datetime(df["Date"])- pd.to_datetime("2019-12-01")).dt.days
df.loc[df["days_from_first_ever_case"] < 0, "days_from_first_ever_case"] = 0
df["days_from_quarantine"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["quarantine"])).dt.days
df["days_from_quarantine"].fillna(0)
 df.loc[df["days_from_quarantine"] < 30, "days_from_quarantine"] = 0
df.loc[df["days_from_quarantine"] >= 30, "days_from_quarantine"] = 1
df["days_from_school"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["schools"])).dt.days
df["days_from_school"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_school"] < 30, "days_from_school"] = 0
df.loc[df["days_from_school"] >= 30, "days_from_school"] = 1
df["days_from_publicplace"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["publicplace"])).dt.days
df["days_from_publicplace"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_publicplace"] < 30, "days_from_publicplace"] = 0
df.loc[df["days_from_publicplace"] >= 30, "days_from_publicplace"] = 1
df["days_from_gathering"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["gathering"])).dt.days
df["days_from_gathering"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_gathering"] < 30, "days_from_gathering"] = 0
df.loc[df["days_from_gathering"] >= 30, "days_from_gathering"] = 1
df["days_from_nonessential"] =(pd.to_datetime(df["Date"])- pd.to_datetime(df["nonessential"])).dt.days
df["days_from_nonessential"].fillna(df["days_from_quarantine"])
 df.loc[df["days_from_nonessential"] < 30, "days_from_nonessential"] = 0
df.loc[df["days_from_nonessential"] >= 30, "days_from_nonessential"] = 1
return df<concatenate>
df_train['alchemy_category'].value_counts()
def generate_ar_features(df, group_by_cols, value_cols):
diff_df = df.groupby(group_by_cols)[value_cols].diff().fillna(0)
 diff_df.columns = [col + "_daily" for col in value_cols]
value_cols += [col + "_daily" for col in value_cols]
df = pd.concat([df, diff_df], axis=1)
 pct_df = df.groupby(group_by_cols)[value_cols].pct_change().fillna(0)
 pct_df.columns = [col + "_pct_change" for col in value_cols]
value_cols += [col + "_pct_change" for col in value_cols]
df = pd.concat([df, pct_df], axis=1)
 yesterday_df = df.groupby(group_by_cols)[value_cols].shift()
value_cols = [col + "_yesterday" for col in value_cols]
yesterday_df.columns = value_cols
df = pd.concat([df, yesterday_df], axis=1)
 three_days_avg = df.groupby(group_by_cols)[value_cols].rolling(3 ).mean()
three_days_avg = three_days_avg.reset_index() [value_cols]
three_days_avg.columns = [col + "_3_day_avg" for col in three_days_avg.columns]
df = pd.concat([df, three_days_avg], axis=1)
 seven_days_avg = df.groupby(group_by_cols)[value_cols].rolling(7 ).mean()
seven_days_avg = seven_days_avg.reset_index() [value_cols]
seven_days_avg.columns = [col + "_7_day_avg" for col in seven_days_avg.columns]
df = pd.concat([df, seven_days_avg], axis=1)
 df = df.replace([np.inf, -np.inf], 0)
 return df<feature_engineering>
df_train['boilerplate'].replace(to_replace=r'"title":', value="",inplace=True,regex=True)
 df_train['boilerplate'].replace(to_replace=r'"url":',value="",inplace=True,regex=True)
 df_train['boilerplate'].replace(to_replace=r'{|}',value="",inplace=True,regex=True)
 df_train['boilerplate']=df_train['boilerplate'].str.lower()
df_test['boilerplate'].replace(to_replace=r'"title":', value="",inplace=True,regex=True)
 df_test['boilerplate'].replace(to_replace=r'"url":',value="",inplace=True,regex=True)
 df_test['boilerplate'].replace(to_replace=r'{|}',value="",inplace=True,regex=True)
 df_test['boilerplate']=df_test['boilerplate'].str.lower()
def generate_features(df):
group_by_cols = ["state","country"]
value_cols = ["ConfirmedCases", "Fatalities", "country_cases", "country_fatalities"]
df = aggregate_label(df)
 df = append_external_data(df)
 df = generate_ar_features(df, group_by_cols, value_cols)
 df = generate_time_features(df)
 df["dow"] = pd.to_datetime(df["Date"] ).dt.dayofweek
df.loc[df["ConfirmedCases_yesterday"]<0, "ConfirmedCases_yesterday"] = 0
df.loc[df["Fatalities_yesterday"]<0, "Fatalities_yesterday"] = 0
return df<feature_engineering>
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased' )
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 train = train.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 train.loc[train["state"].isna() , "state"] = "Unknown"
train = generate_features(train)
 print(train["Date"].min() , "-", train["Date"].max())
 train.loc[train["country"] == "Italy"].tail()<filter>
SEQ_length=512
Xids=np.zeros(( df_train.shape[0],SEQ_length))
Xmask=np.zeros(( df_train.shape[0],SEQ_length))
y=np.zeros(( df_train.shape[0],1))
Xids_test=np.zeros(( df_test.shape[0],SEQ_length))
Xmask_test=np.zeros(( df_test.shape[0],SEQ_length))
Xids
train.loc[train["state"] == "Hubei"]<load_from_csv>
for i,sequence in enumerate(df_train['boilerplate']):
tokens=tokenizer.encode_plus(sequence,max_length=SEQ_length,padding='max_length',add_special_tokens=True,
truncation=True,return_token_type_ids=False,return_attention_mask=True,
return_tensors='tf')
 Xids[i,:],Xmask[i,:],y[i,0]=tokens['input_ids'],tokens['attention_mask'],df_train.loc[i,'label']
for i,sequence in enumerate(df_test['boilerplate']):
tokens=tokenizer.encode_plus(sequence,max_length=SEQ_length,padding='max_length',add_special_tokens=True,
truncation=True,return_token_type_ids=False,return_attention_mask=True,
return_tensors='tf')
 Xids_test[i,:],Xmask_test[i,:]=tokens['input_ids'],tokens['attention_mask']
test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 test = test.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 test.loc[test["state"].isna() , "state"] = "Unknown"
print(test["Date"].min() , "-", test["Date"].max())
 test.head()<feature_engineering>
tf.config.get_visible_devices()
train.loc[train["Date"]<"2020-03-22", "split"] = "train"
train.loc[train["Date"]>="2020-03-22", "split"] = "test"<import_modules>
dataset=tf.data.Dataset.from_tensor_slices(( Xids,Xmask,y))
def map_func(input_ids,mask,labels):
return {'input_ids':input_ids,'attention_mask':mask},labels
dataset=dataset.map(map_func)
 dataset=dataset.shuffle(100000 ).batch(32 ).prefetch(1000)
 DS_size=len(list(dataset))
train=dataset.take(round(DS_size*0.85))
val=dataset.skip(round(DS_size*0.85))
import lightgbm as lgb
from sklearn.metrics import mean_squared_error<train_model>
dataset_test=tf.data.Dataset.from_tensor_slices(( Xids_test,Xmask_test))
def map_func(input_ids,mask):
return {'input_ids':input_ids,'attention_mask':mask}
dataset_test=dataset_test.map(map_func)
 dataset_test=dataset_test.batch(32 ).prefetch(1000 )
def train_model(df, label, base_label, features=features, **kwargs):
X_train = df.loc[df["split"] == "train"][features]
y_train = np.log(df.loc[df["split"] == "train"][label] + 1)
 b_train = np.log(df.loc[df["split"] == "train"][base_label] + 1)
 X_test = df.loc[df["split"] == "test"][features]
y_test = np.log(df.loc[df["split"] == "test", label] + 1)
 b_test = np.log(df.loc[df["split"] == "test", base_label] + 1)
 print(kwargs)
 model = lgb.LGBMRegressor(**kwargs)
 model.fit(X_train, y_train, init_score = b_train)
 y_pred = model.predict(X_test)
 print(np.sqrt(mean_squared_error(y_test, y_pred + b_test)))
 return model<train_model>
distil_bert = 'distilbert-base-uncased'
config = DistilBertConfig(dropout=0.2, attention_dropout=0.2)
 config.output_hidden_states = False
transformer_model = TFDistilBertModel.from_pretrained(distil_bert, config = config)
 input_ids_in = tf.keras.layers.Input(shape=(SEQ_length,), name='input_ids', dtype='int32')
 input_masks_in = tf.keras.layers.Input(shape=(SEQ_length,), name='attention_mask', dtype='int32')
 embedding_layer = transformer_model(input_ids_in, attention_mask=input_masks_in)[0]
X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(embedding_layer)
 X = tf.keras.layers.GlobalMaxPool1D()(X)
 X = tf.keras.layers.Dense(50, activation='relu' )(X)
 X = tf.keras.layers.Dropout(0.2 )(X)
 X = tf.keras.layers.Dense(1, activation='sigmoid' )(X)
 model = tf.keras.Model(inputs=[input_ids_in, input_masks_in], outputs = X)
 for layer in model.layers[:3]:
layer.trainable = False
lgb_model_cases = train_model(train, "ConfirmedCases", "ConfirmedCases_yesterday",
max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8 )<train_model>
model.compile(loss=tf.keras.losses.BinaryCrossentropy() ,
optimizer='adam',metrics=[tf.keras.metrics.AUC() ,tf.keras.metrics.Precision() ,tf.keras.metrics.Recall()
] )
lgb_model_fatalities = train_model(train, "Fatalities", "Fatalities_yesterday", features,
max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8
)<prepare_x_and_y>
history=model.fit(train,validation_data=val,epochs=3 )
X_train = train[features]
y_train = np.log(train["ConfirmedCases"] + 1)
 b_train = np.log(train["ConfirmedCases_yesterday"] + 1)
 cases_model = lgb.LGBMRegressor(max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8 
)
 cases_model.fit(X_train, y_train, init_score = b_train)
 X_train = train[features]
y_train = np.log(train["Fatalities"] + 1)
 b_train = np.log(train["Fatalities_yesterday"] + 1)
 fatalities_model = lgb.LGBMRegressor(max_depth=5,
colsample_bytree=0.8,
learning_rate=0.1,
n_estimators=500,
subsample=0.8)
 fatalities_model.fit(X_train, y_train, init_score = b_train )<load_from_csv>
predictions=model.predict(dataset_test)
 df_test['label']=predictions
df_test.to_csv('submission.csv',columns=['urlid','label'],index=False )
base_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 base_df = base_df.rename({"Province_State": "state", "Country_Region": "country"}, axis=1)
 base_df.loc[base_df["state"].isna() , "state"] = "Unknown"
scoring_dates = test["Date"].unique()<import_modules>
input_x=tf.data.Dataset.from_tensor_slices(( Xids,Xmask,y))
def map_func(input_ids,mask,labels):
return {'input_ids':input_ids,'attention_mask':mask}
input_x=input_x.map(map_func)
 input_x=input_x.shuffle(100000 ).batch(32 ).prefetch(1000)
 y_true = y
from datetime import datetime as dt, timedelta<feature_engineering>
y_pred=model.predict(dataset)
 y_pred
pred_df = pd.DataFrame(columns=base_df.columns)
 for date in scoring_dates.tolist() :
print(date)
 new_df = base_df.loc[base_df["Date"] < date].copy()
curr_date_df = test.loc[test["Date"] == date].copy()
curr_date_df["ConfirmedCases"] = 0
curr_date_df["Fatalities"] = 0
new_df = new_df.append(curr_date_df ).reset_index(drop=True)
 new_df = generate_features(new_df)
 new_df[features] = new_df[features]
predictions = cases_model.predict(new_df[features])+ np.log(new_df["ConfirmedCases_yesterday"] + 1)
 new_df["predicted_cases"] = round(np.maximum(np.exp(predictions)- 1, new_df["ConfirmedCases_yesterday"]))
predictions = fatalities_model.predict(new_df[features])+ np.log(new_df["Fatalities_yesterday"] + 1)
 new_df["predicted_fatalities"] = np.maximum(np.exp(predictions)- 1, new_df["Fatalities_yesterday"])
 new_df["predicted_fatalities"] = round(np.minimum(new_df["predicted_fatalities"], new_df["predicted_cases"]*0.2))
new_df.loc[new_df["Date"] == date, "ConfirmedCases"] = new_df.loc[new_df["Date"] == date, "predicted_cases"]
new_df.loc[new_df["Date"] == date, "Fatalities"] = new_df.loc[new_df["Date"] == date, "predicted_fatalities"]
pred_df = pred_df.append(new_df.loc[new_df["Date"] == date][pred_df.columns.tolist() ])
 if date not in base_df["Date"].unique() :
base_df = base_df.append(new_df.loc[new_df["Date"] == date][base_df.columns.tolist() ] )<filter>
y_pred = np.round(y_pred)
 y_pred
<filter><EOS> 
print(metrics.classification_report(y_true, y_pred))