Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<load_from_csv>
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
from xgboost import XGBClassifier
from xgboost import plot_importance
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix,classification_report, mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, cross_val_score, GridSearchCV
train = pd.read_csv(".. /input/otto-group-product-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/otto-group-product-classification-challenge/test.csv")
 sample = pd.read_csv(".. /input/otto-group-product-classification-challenge/sampleSubmission.csv")
 tage = train.target.values
X = train.drop("id",axis = 1)
 X = X.drop("target",axis = 1)
 y = test.drop("id",axis = 1)
 lbl_enc = preprocessing.LabelEncoder()
label_encoded_y = lbl_enc.fit_transform(tage)
 X_train, X_test, y_train, y_test = train_test_split(X, label_encoded_y,test_size=0.20, random_state=29)
 train_data = lgb.Dataset(X_train, label=y_train)
 eval_data = lgb.Dataset(X_test, label=y_test, reference= train_data)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
'num_class': 9,
'verbose': 2,
}
model = lgb.train(
 params,
train_data,
valid_sets=eval_data,
num_boost_round=100,
verbose_eval=10,
)<predict_on_test>
def load_csv(dataset, datadir):

 df = pd.read_csv(
 f"{datadir}/{dataset}.csv",
dtype={"Country/Region": "category"},
parse_dates=["Date"], 
)
 df["DayOfYear"] = df["Date"].dt.dayofyear
df["Province/State"].fillna(df["Country/Region"], inplace=True)
 return df
def sigmoid(x, x0, L, k):
return L /(1 + np.exp(-k *(x - x0)))
 def fit(df: pd.DataFrame, index: str, channel: str):
x = df[index].values
y = df[channel].values
try:
p, pc = curve_fit(
 sigmoid,
x,
y,
p0=(np.median(x), y[-1], 0.1),
maxfev=100000,
bounds=([0, y[-1], 0], [np.inf, np.inf, 2]),
ftol=1e-5, 
)
 except RuntimeError:
return [-1] * 9
return list(p)+ pc[np.triu_indices_from(pc)].tolist()
def apply_fit(df: pd.DataFrame, index: str, channel: str)-> pd.DataFrame:

 fits = df.groupby(["Country/Region", "Province/State"])[
[index, channel]
].apply(lambda x: fit(x, index=index, channel=channel))
return pd.DataFrame(
 fits.tolist() ,
columns=["x0", "L", "k", "sx0", "sx0L", "sx0k", "sL", "sLk", "sk"],
index=fits.index,
).add_suffix(f"_{channel}")
 def prep_pop(filename: str)-> pd.DataFrame:

 df = pd.read_csv(
 filename,
engine="python",
skiprows=901,
index_col=0,
names=(
 "idx",
"country",
"Year",
"Series",
"Value",
"Footnotes",
"Source", 
),
usecols=["country", "Year", "Series", "Value"], 
)
 pivot =(
 df[df["Year"] == 2019] 
.pivot(columns="Series", values="Value")
.dropna() 
.rename(
 columns={
"Population aged 0 to 14 years old(percentage)": "young%",
"Population aged 60+ years old(percentage)": "old%",
"Population density": "density",
"Population mid-year estimates(millions)": "population",
"Population mid-year estimates for females(millions)": "fem",
"Population mid-year estimates for males(millions)": "male",
"Sex ratio(males per 100 females)": "ratio",
} 
)
)
 pivot["fem%"] = pivot["fem"] * 100.0 / pivot["population"]
return pivot.drop(columns=["fem", "male", "ratio"])
 def prep_hh(filename: str)-> pd.DataFrame:

 return(
 pd.read_csv(
 filename,
usecols=[0, 3, 4, 5, 6, 7, 8, 18],
names=[
"country",
"date",
"avg_hh",
"hh%1",
"hh%2-3",
"hh%4-5",
"hh%6+",
"hh65+",
],
na_values=".. ",
header=0,
parse_dates=["date"], 
)
.sort_values(by=["country", "date"])
.groupby("country")
.tail(1)
.drop(columns=["date"])
.set_index("country")
)
 def predict(df):
df["y_ConfirmedCases"] =(
(
 df["L_ConfirmedCases"]
/(
 1
+ np.exp(
 -df["k_ConfirmedCases"]
*(df["DayOfYear"] - df["x0_ConfirmedCases"])
)
)
)
.round() 
.astype(int)
)
 df["y_Fatalities"] =(
(
 df["L_Fatalities"]
/(
 1
+ np.exp(
 -df["k_Fatalities"]
*(df["DayOfYear"] - df["x0_Fatalities"])
)
)
)
.round() 
.astype(int)
)
 return df
def merge_support_main(main_df, supp_df):
return pd.merge(
 main_df,
supp_df,
left_on="Country/Region",
right_on="country",
how="left", 
)
 def augment_dataset(DATADIR="data/"):
train = load_csv("train", DATADIR)
 test = load_csv("test", DATADIR)
 fits_cc = apply_fit(train, index="DayOfYear", channel="ConfirmedCases")
 fits_f = apply_fit(train, index="DayOfYear", channel="Fatalities")
 fits = fits_cc.join(fits_f)
 train = pd.merge(
 train, fits.reset_index() , on=["Country/Region", "Province/State"] 
)
 test = pd.merge(
 test, fits.reset_index() , on=["Country/Region", "Province/State"] 
)
 sf = pd.read_csv(f"{DATADIR}/supporting_features.csv", index_col=0 ).drop(
 columns=["avg_HH"] 
)
 sf.index.name = "country"
train = merge_support_main(train, sf)
 test = merge_support_main(test, sf)
 hh = prep_hh(
 f"{DATADIR}/population_division_UN_Houseshold_Size_and_Composition_2019.csv" 
)
 train = pd.merge(
 train, hh, left_on="Country/Region", right_on="country", how="left" 
)
 test = pd.merge(
 test, hh, left_on="Country/Region", right_on="country", how="left" 
)
 pop = prep_pop(
 f"{DATADIR}/SYB62_1_201907_Population, Surface Area and Density.csv" 
)
 train = pd.merge(
 train, pop, left_on="Country/Region", right_on="country", how="left" 
)
 test = pd.merge(
 test, pop, left_on="Country/Region", right_on="country", how="left" 
)
 train = predict(train)
 test = predict(test)
 return train, test
y_pred_prob = model.predict(y, num_iteration=model.best_iteration)
<save_to_csv>
train = pd.read_csv("/kaggle/input/covid19aug/augmented/train_aug.csv")
 test = pd.read_csv("/kaggle/input/covid19aug/augmented/test_aug.csv" )
y_pred_prob = pd.DataFrame(y_pred_prob, index=sample.id.values, columns=sample.columns[1:])
 y.to_csv("tes.csv",index=False)
 y_pred_prob.to_csv("out1.csv", index_label='id' )<import_modules>
def prepare_data(df: pd.DataFrame):
df = df[df['ConfirmedCases'] > 100].reset_index()
df['days_since_first100cases'] = df.groupby(['Province/State', 'Country/Region'] 
)['ConfirmedCases'].transform(lambda x: range(len(x)))
 return df.drop(['x0_ConfirmedCases', 'L_ConfirmedCases', 'k_ConfirmedCases', 'sx0_ConfirmedCases',
'sx0L_ConfirmedCases', 'sx0k_ConfirmedCases', 'sL_ConfirmedCases',
'sLk_ConfirmedCases', 'sk_ConfirmedCases', 'x0_Fatalities',
'L_Fatalities', 'k_Fatalities', 'sx0_Fatalities', 'sx0L_Fatalities',
'sx0k_Fatalities', 'sL_Fatalities', 'sLk_Fatalities', 'sk_Fatalities',
'DayOfYear', 'index', 'Id', 'Date'], axis=1)
 def impute_values(df: pd.DataFrame):
features = ['life_expectancy_years', 'literacy_rates', 'veg_supply_person_kg_year',
'respiratory_infections_death', 'deaths_from_smoking%', 'young%',
'old%', 'density', 'population', 'fem%', 'number_doc_per1000',
'surgeons_act_working', 'obstetricians_act_working',
'anaesthesiologists_act_working', 'avg_hh', 'hh%1', 'hh%2-3', 'hh%4-5',
'hh%6+', 'hh65+']
df_per_country = df.groupby(
 ['Province/State', 'Country/Region'], as_index=False ).min()
df_per_country = df_per_country[[
'Province/State', 'Country/Region']+features].replace(0, np.nan)
 column_null = df_per_country.isnull().mean()
column_null_drop = list(column_null[column_null > 0.5].index)
 df_per_country_clean = df_per_country.drop(
 column_null_drop, axis=1 ).fillna(df_per_country.median())
 return df.drop(features, axis=1 ).merge(df_per_country_clean,
on=['Province/State', 'Country/Region'], how='left' )
from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import lightgbm as lgb_org
import optuna.integration.lightgbm as lgb<define_variables>
def model(df, ylab):
features = df.drop(['Country/Region','Province/State',
'ConfirmedCases', 'Fatalities', 'Long', 'y_Fatalities', 'y_ConfirmedCases'
], axis=1)
 label = df[ylab]
train_df, test_df, train_label, test_label = train_test_split(
 features, label, test_size=0.2)
 model = XGBClassifier(objective='reg:linear')
 model.fit(train_df, train_label)
 test_pred = model.predict(test_df)
 return model, np.sqrt(mean_squared_error(test_label, test_pred))
base_dir = Path(
 "/kaggle/input/otto-group-product-classification-challenge/"
)<load_from_csv>
df = impute_values(prepare_data(train))
df.head(3 )
def read_csv(path):
df = pd.read_csv(path)
 for col in df.columns:
if col.startswith("feat_"):
df[col] = df[col].astype("int32")
 return df<load_from_csv>
xgb_cases, rmse_cases= model(df,'y_ConfirmedCases' )
df = read_csv(str(base_dir / "train.csv"))<feature_engineering>
xgb_fatalities, rmse_fatalities = model(df,'y_Fatalities' )
class_to_order = dict()
order_to_class = dict()
for idx, col in enumerate(df.target.unique()):
order_to_class[idx] = col
class_to_order[col] = idx
df["target_ord"] = df["target"].map(class_to_order )<define_variables>
prediction = predict(test)[["ForecastId", "y_ConfirmedCases", "y_Fatalities"]]
prediction.columns = ["ForecastId","ConfirmedCases","Fatalities"]
prediction.head()
prediction.to_csv('submission.csv', index=False )