Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_model>
train_df=pd.read_csv(".. /input/train.csv")
 test_df=pd.read_csv(".. /input/test.csv")
 train_df.head()
model.save_model('model_{}.cbm'.format(MODEL_VER))<define_variables>
combined_df=pd.concat([train_df,test_df] )
%%time
spred = get_df(is_train=False, backward_lags=BACKWARD_LAGS)
 for pred_day in tqdm(range(1, 28 + 28 + 1)) :
pred_date = datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=pred_day)
 pred_date_back = pred_date - timedelta(days=BACKWARD_LAGS + 1)
 print('-' * 70)
 print('forecast day forward:', pred_day, '| forecast date:', pred_date)
 spred_data = spred[(spred['date'] >= pred_date_back)&(spred['date'] <= pred_date)].copy()
spred_data = make_features(spred_data)
 spred_data = spred_data.loc[spred['date'] == pred_date, train_cols]
spred_data[cat_cols] = spred_data[cat_cols].fillna(0)
 spred.loc[spred['date'] == pred_date, 'sales'] = model.predict(spred_data)
 del spred_data
gc.collect()<feature_engineering>
combined_df.loc[combined_df["Age"].isnull() ].groupby(["Pclass","Sex"])["Sex"].agg(["count"] )
spred_subm = spred.loc[spred['date'] > END_DATE, ['id', 'd', 'sales']].copy()
last_d = int(spred.loc[spred['date'] == END_DATE, 'd'].unique() [0].replace('d_', ''))
print('last d num:', last_d)
 spred_subm['d'] = spred_subm['d'].apply(lambda x: 'F{}'.format(int(x.replace('d_', '')) - last_d))
spred_subm.loc[spred_subm['sales'] < 0, 'sales'] = 0<save_to_csv>
combined_df.groupby(["Sex","SibSp","Parch"])["Age"].agg(["median"] )
f_cols_val = ['F{}'.format(x)for x in range(1, 28 + 1)]
f_cols_eval = ['F{}'.format(x)for x in range(28 + 1, 28 + 28 + 1)]
spred_subm_eval = spred_subm.copy()
spred_subm.drop(columns=f_cols_eval, inplace=True)
 spred_subm_eval.drop(columns=f_cols_val, inplace=True)
 spred_subm_eval.columns = spred_subm.columns
spred_subm_eval['id'] = spred_subm_eval['id'].str.replace('validation', 'evaluation')
 spred_subm = pd.concat([spred_subm, spred_subm_eval], axis=0, sort=False)
 spred_subm.reset_index(drop=True, inplace=True)
 spred_subm.to_csv('submission.csv', index=False)
 print('submission saved:', spred_subm.shape )<import_modules>
combined_df.loc[combined_df["SibSp"]==8]
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb<define_variables>
combined_df["Name_key"]=""
combined_df["Name_key"]=combined_df["Name"].str.split(',',expand=True)[1].str.split(' ',expand=True)[1]
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }<set_options>
combined_df.groupby(["Sex","Name_key","Parch"])["Age"].agg(["median"] )
pd.options.display.max_columns = 50<define_variables>
combined_df["Fare_Group"]=pd.cut(combined_df["Fare"],range(0,350,50),right=False)
 combined_df["Fare_Group"]=combined_df["Fare_Group"].astype("object")
 combined_df["Fare_Group"].fillna("[300,600)",inplace=True )
h = 28
max_lags = 57
tr_last = 1913
fday = datetime(2016,4, 25)
 fday<load_from_csv>
Age_values=combined_df.groupby(["Sex","Pclass","Name_key","Parch"])["Age"].agg(["median"] ).reset_index()
Age_values.loc[Age_values["median"].isnull() ]
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
null_age=combined_df.loc[combined_df["Age"].isnull() ][['PassengerId','Sex','Name_key','Pclass','Parch']]
Age_values_f=pd.merge(Age_values,null_age,how='inner',on=['Sex','Name_key','Pclass','Parch'])
 Age_values_f=Age_values_f.rename(columns={'median':"Age"})
 Age_values_f.loc[Age_values_f["Age"].isnull() ]
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
combined_df.loc[combined_df["PassengerId"].isin([1257,980,1234])]
FIRST_DAY = 350<correct_missing_values>
combined_df.groupby(["Sex"])["Age"].agg(["median"] )
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
Age_values_f.loc[(Age_values_f["Age"].isnull())&(Age_values_f["Sex"]=='female'),"Age"]=27.0
Age_values_f.loc[(Age_values_f["Age"].isnull())&(Age_values_f["Sex"]=='male'),"Age"]=28.0
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
train_df.set_index('PassengerId',inplace=True)
 Age_values_f.set_index('PassengerId',inplace=True)
 train_df.update(Age_values_f)
 train_df.reset_index(inplace=True)
 test_df.set_index('PassengerId',inplace=True)
 test_df.update(Age_values_f)
 test_df.reset_index(inplace=True)
<create_dataframe>
train_df.groupby(["Embarked"])["Embarked"].agg(["count"] )
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=cat_feats,
free_raw_data=False )<prepare_x_and_y>
train_df.loc[train_df["PassengerId"]==62.0,"Embarked"]='S'
train_df.loc[train_df["PassengerId"]==830,"Embarked"]='S'
del df, X_train, y_train, fake_valid_inds,train_inds ; gc.collect()<init_hyperparams>
train_df["Cabin_prefix"]=train_df["Cabin"].str[0:1]
train_df.groupby(["Cabin_prefix","Pclass"])["Pclass"].agg(["count"] ).reset_index()
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
train_df.drop(columns={"Cabin","Cabin_prefix"},inplace=True )
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )<save_model>
test_df.loc[test_df["Fare"].isnull() ]
m_lgb.save_model("model.lgb" )<count_unique_values>
train_df.loc[train_df["Pclass"]==3].loc[train_df["Age"]>50].groupby(["Age","Sex"])["Fare"].agg(["mean"] )
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()<import_modules>
train_df.loc[train_df["Pclass"]==3].loc[train_df["Age"]>50].groupby(["Sex"])["Fare"].agg(["mean"] )
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb<define_variables>
test_df.loc[test_df["Fare"].isnull() ,"Fare"]=7.518522
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }<set_options>
related_people=combined_df[["PassengerId","Name","SibSp","Parch","Ticket","Embarked"]].copy()
related_people["Last_Name"]=""
related_people["Last_Name"]=related_people["Name"].str.split(",",expand=True)[0]
related_people["total_related"]=related_people["SibSp"]+related_people["Parch"]+1
X=related_people.loc[(related_people["SibSp"]>0)|(related_people["Parch"]>0)].groupby(["Last_Name","total_related"])["PassengerId"].agg(["count"])
 X.reset_index(inplace=True)
 Y=pd.DataFrame(X.loc[X["count"]==X["total_related"]]["Last_Name"])
 Y["RGroup"]=Y["Last_Name"]
Y.drop_duplicates(inplace=True)
 Y.set_index("Last_Name",inplace=True)
 related_people["RGroup"]=""
related_people.set_index("Last_Name",inplace=True)
 related_people.update(Y)
 related_people.reset_index(inplace=True)
 related_people.loc[related_people["Ticket"]=='113760',"RGroup"]="Carter_1"
related_people.loc[related_people["Ticket"]=='224252',"RGroup"]="Carter_2"
pd.options.display.max_columns = 50<define_variables>
Y=pd.DataFrame(related_people.loc[(related_people["SibSp"]==0)&(related_people["Parch"]==0)][["Last_Name","Ticket"]])
 Y["RGroup"]=Y["Last_Name"]+'_'+Y["Ticket"]
Y.drop_duplicates(inplace=True)
 Y.set_index(["Last_Name","Ticket"],inplace=True)
 related_people.set_index(["Last_Name","Ticket"],inplace=True)
 related_people.update(Y)
 related_people.reset_index(inplace=True )
h = 28
max_lags = 70
tr_last = 1913
fday = datetime(2016,4, 25)
 fday<load_from_csv>
X=related_people.loc[related_people["RGroup"]==""].groupby(["Ticket","total_related"])["PassengerId"].agg(["count"])
 X.reset_index(inplace=True)
 Y=pd.DataFrame(X.loc[X["count"]==X["total_related"]]["Ticket"])
 Y["RGroup"]=Y["Ticket"]+'_R'
Y.drop_duplicates(inplace=True)
 Y.set_index("Ticket",inplace=True)
 related_people.set_index("Ticket",inplace=True)
 related_people.update(Y)
 related_people.reset_index(inplace=True)
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
related_people.loc[related_people["PassengerId"]==249,"RGroup"]="Beckwith_M"
related_people.loc[related_people["PassengerId"]==872,"RGroup"]="Beckwith_M"
related_people.loc[related_people["PassengerId"]==137,"RGroup"]="Beckwith_M"
related_people.loc[related_people["PassengerId"]==572,"RGroup"]="Lamson_M"
related_people.loc[related_people["PassengerId"]==1248,"RGroup"]="Lamson_M"
related_people.loc[related_people["PassengerId"]==969,"RGroup"]="Lamson_M"
related_people.loc[related_people["PassengerId"]==1200,"RGroup"]="Hays_M"
related_people.loc[related_people["PassengerId"]==821,"RGroup"]="Hays_M"
related_people.loc[related_people["PassengerId"]==588,"RGroup"]="Frolicher_M"
related_people.loc[related_people["PassengerId"]==1289,"RGroup"]="Frolicher_M"
related_people.loc[related_people["PassengerId"]==540,"RGroup"]="Frolicher_M"
related_people.loc[related_people["PassengerId"]==581,"RGroup"]="Jacobsohn_M"
related_people.loc[related_people["PassengerId"]==1133,"RGroup"]="Jacobsohn_M"
related_people.loc[related_people["PassengerId"]==218,"RGroup"]="Jacobsohn_M"
related_people.loc[related_people["PassengerId"]==601,"RGroup"]="Jacobsohn_M"
related_people.loc[related_people["PassengerId"]==756,"RGroup"]="Hamalainen_M"
related_people.loc[related_people["PassengerId"]==248,"RGroup"]="Hamalainen_M"
related_people.loc[related_people["PassengerId"]==1130,"RGroup"]="Hamalainen_M"
related_people.loc[related_people["PassengerId"]==313,"RGroup"]="Lahtinen_M"
related_people.loc[related_people["PassengerId"]==1041,"RGroup"]="Lahtinen_M"
related_people.loc[related_people["PassengerId"]==418,"RGroup"]="Lahtinen_M"
related_people.loc[related_people["PassengerId"]==530,"RGroup"]="Hocking_M"
related_people.loc[related_people["PassengerId"]==944,"RGroup"]="Hocking_M"
related_people.loc[related_people["PassengerId"]==775,"RGroup"]="Hocking_M"
related_people.loc[related_people["PassengerId"]==832,"RGroup"]="Richards_M"
related_people.loc[related_people["PassengerId"]==408,"RGroup"]="Richards_M"
related_people.loc[related_people["PassengerId"]==438,"RGroup"]="Richards_M"
related_people.loc[related_people["PassengerId"]==105,"RGroup"]="Gustafsson_M"
related_people.loc[related_people["PassengerId"]==393,"RGroup"]="Gustafsson_M"
related_people.loc[related_people["PassengerId"]==207,"RGroup"]="Gustafsson_M"
related_people.loc[related_people["PassengerId"]==86,"RGroup"]="Gustafsson_M"
related_people.loc[related_people["PassengerId"]==69,"RGroup"]="3101281"
related_people.loc[related_people["PassengerId"]==480,"RGroup"]="Hirvonen_M"
related_people.loc[related_people["PassengerId"]==896,"RGroup"]="Hirvonen_M"
related_people.loc[related_people["PassengerId"]==477,"RGroup"]="Renouf_M"
related_people.loc[related_people["PassengerId"]==727,"RGroup"]="Renouf_M"
related_people.loc[related_people["PassengerId"]==70,"RGroup"]="Kink_M"
related_people.loc[related_people["PassengerId"]==1268,"RGroup"]="Kink_M"
related_people.loc[related_people["PassengerId"]==185,"RGroup"]="Kink_M"
related_people.loc[related_people["PassengerId"]==1286,"RGroup"]="Kink_M"
related_people.loc[related_people["PassengerId"]==1057,"RGroup"]="Kink_M"
related_people.loc[related_people["PassengerId"]==1037,"RGroup"]="Vander Planke_M"
related_people.loc[related_people["PassengerId"]==19,"RGroup"]="Vander Planke_M"
related_people.loc[related_people["PassengerId"]==39,"RGroup"]="Vander Planke_M"
related_people.loc[related_people["PassengerId"]==334,"RGroup"]="Vander Planke_M"
related_people.loc[related_people["PassengerId"]==206,"RGroup"]="Strom_M"
related_people.loc[related_people["PassengerId"]==252,"RGroup"]="Strom_M"
related_people.loc[related_people["PassengerId"]==443,"RGroup"]="347076"
related_people.loc[related_people["PassengerId"]==268,"RGroup"]="Strom_M"
related_people.loc[related_people["PassengerId"]==1106,"RGroup"]="347091"
related_people.loc[related_people["PassengerId"]==193,"RGroup"]="350046"
related_people.loc[related_people["PassengerId"]==722,"RGroup"]="350048"
related_people.loc[related_people["PassengerId"]==893,"RGroup"]="Hocking_M"
related_people.loc[related_people["PassengerId"]==41,"RGroup"]="7546"
related_people.loc[related_people["PassengerId"]==566,"RGroup"]="Davies_M"
related_people.loc[related_people["PassengerId"]==901,"RGroup"]="Davies_M"
related_people.loc[related_people["PassengerId"]==1079,"RGroup"]="Davies_M"
related_people.loc[related_people["PassengerId"]==923,"RGroup"]="Renouf_M"
related_people.loc[related_people["PassengerId"]==1211,"RGroup"]="Renouf_M"
related_people.loc[related_people["PassengerId"]==672,"RGroup"]="Hays_M"
related_people.loc[related_people["PassengerId"]==984,"RGroup"]="Hays_M"
related_people.loc[related_people["PassengerId"]==274,"RGroup"]="PC 17596"
related_people.loc[related_people["PassengerId"]==665,"RGroup"]="Hirvonen_M"
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
related_people.drop(columns={"Ticket","Last_Name","Name","SibSp","Parch","total_related"})
 related_people.set_index("PassengerId",inplace=True)
 train_df["RGroup"]=""
train_df.set_index("PassengerId",inplace=True)
 train_df.update(related_people)
 train_df.reset_index(inplace=True)
 related_people.reset_index(inplace=True)
FIRST_DAY = 800 
<correct_missing_values>
related_people.set_index("PassengerId",inplace=True)
 test_df["RGroup"]=""
test_df.set_index("PassengerId",inplace=True)
 test_df.update(related_people)
 test_df.reset_index(inplace=True)
 related_people.reset_index(inplace=True )
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
train_df["Sex"]=train_df["Sex"].astype("category")
 train_df["Pclass"]=train_df["Pclass"].astype("category")
 train_df["Embarked"]=train_df["Embarked"].astype("category")
 train_df["RGroup"]=train_df["RGroup"].astype("category")
 train_df.drop(columns={"Name","Ticket","PassengerId"},inplace=True)
 train_df=pd.get_dummies(train_df,columns=["Pclass"])
 train_df=pd.get_dummies(train_df,columns=["Sex"])
 train_df=pd.get_dummies(train_df,columns=["Embarked"])
 train_df.rename(columns={"Pclass_1.0":"Pclass_1","Pclass_2.0":"Pclass_2","Pclass_3.0":"Pclass_3"},inplace=True)
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
test_df["Sex"]=test_df["Sex"].astype("category")
 test_df["Pclass"]=test_df["Pclass"].astype("category")
 test_df["Embarked"]=test_df["Embarked"].astype("category")
 test_df["RGroup"]=test_df["RGroup"].astype("category")
 test_passengerId=test_df["PassengerId"].copy()
test_df.drop(columns={"Name","Cabin","Ticket","PassengerId"},inplace=True)
 test_df=pd.get_dummies(test_df,columns=["Pclass"])
 test_df=pd.get_dummies(test_df,columns=["Sex"])
 test_df=pd.get_dummies(test_df,columns=["Embarked"])
train_data = lgb.Dataset(X_train, label = y_train, categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_inds = np.random.choice(len(X_train), 1000000)
 fake_valid_data = lgb.Dataset(X_train.iloc[fake_valid_inds], label = y_train.iloc[fake_valid_inds],categorical_feature=cat_feats,
free_raw_data=False )<init_hyperparams>
train_df_dec=train_df.copy()
train_df_dec = train_df_dec.sample(frac=1 ).reset_index(drop=True)
 row_count = train_df_dec.shape[0]
split_point = int(row_count*0.30)
 cv_data, train_data = train_df_dec[:split_point].copy() , train_df_dec[split_point:].copy()
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 2500,
}<train_model>
train_df_dec
R=train_data.groupby(["RGroup"])["Survived"].agg(["count","sum"])
 R.reset_index(inplace=True)
 R["percentage"]=R["sum"]/R["count"]
R.drop(columns={"count","sum"},inplace=True )
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=100 )<save_model>
R.set_index("RGroup",inplace=True)
 train_data["percentage"]=np.nan
train_data.set_index("RGroup",inplace=True)
 train_data.update(R)
 train_data.reset_index(inplace=True)
 R.reset_index(inplace=True)
m_lgb.save_model("model.lgb" )<count_unique_values>
R.set_index("RGroup",inplace=True)
 cv_data["percentage"]=np.nan
cv_data.set_index("RGroup",inplace=True)
 cv_data.update(R)
 cv_data.reset_index(inplace=True)
 R.reset_index(inplace=True)
 cv_data.loc[(cv_data["percentage"].isnull()),"percentage"]=0.5
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()<import_modules>
R.set_index("RGroup",inplace=True)
 test_df["percentage"]=np.nan
test_df.set_index("RGroup",inplace=True)
 test_df.update(R)
 test_df.reset_index(inplace=True)
 R.reset_index(inplace=True)
 test_df.loc[(test_df["percentage"].isnull()),"percentage"]=0.5
<save_to_csv>
Y_train=train_data["Survived"].copy()
X_train=train_data.drop(columns={"Survived"})
 Y_cv=cv_data["Survived"].copy()
X_cv=cv_data.drop(columns={"Survived"})
 X_train.drop(columns={"RGroup"},inplace=True)
 X_cv.drop(columns={"RGroup"},inplace=True)
data1 = pd.read_csv('.. /input/blenddata2/M5 Darkmagic.csv' ).sort_values(by = 'id' ).reset_index(drop=True)
 data2 = pd.read_csv('.. /input/blenddata/M5 ForecasteR v2.csv' ).sort_values(by = 'id' ).reset_index(drop=True)
 submission = data1.copy()
for i in range(1,29):
data1['F'+str(i)] *= 1.02
for c in submission.columns :
if c != 'id' :
submission[c] = 0.33*data1[c] + 0.67*data2[c]
submission.to_csv('submission.csv',index=False )<import_modules>
scores=[]
for x in range(1,7):
clf=tree.DecisionTreeClassifier(criterion='entropy',max_depth=x)
 clf=clf.fit(X_train,Y_train)
 Y_pred=clf.predict(X_cv)
 scores.append(metrics.accuracy_score(Y_cv,Y_pred))
print(scores )
import numpy as np
import pandas as pd<save_to_csv>
X_train.drop(columns={"percentage"},inplace=True)
 X_cv.drop(columns={"percentage"},inplace=True)
 test_df.drop(columns={"percentage"},inplace=True)
 test_df.drop(columns={"RGroup"},inplace=True)
 scores=[]
for x in range(1,18):
clf=tree.DecisionTreeClassifier(criterion='entropy',max_depth=x)
 clf=clf.fit(X_train,Y_train)
 Y_pred=clf.predict(X_cv)
 scores.append(metrics.accuracy_score(Y_cv,Y_pred))
print(scores )
submission = pd.read_csv('.. /input/m5-forecast-attack-of-the-data-table/submission.csv')
 for i in range(1,29):
submission['F'+str(i)] *= 1.04
submission.to_csv('submission.csv', index=False )<import_modules>
train_df_dec.drop(columns={"RGroup"},inplace=True)
 Target_df=train_df_dec["Survived"]
train_df_dec.drop(columns={"Survived"},inplace=True )
import numpy as np
import pandas as pd<load_from_csv>
clf=tree.DecisionTreeClassifier(criterion='entropy',max_depth=4)
 scores_cv = cross_val_score(clf, train_df_dec,Target_df, cv=10)
 scores_cv
data1 = pd.read_csv('.. /input/m5-more-data-table-and-xgb/submission_lgbm.csv')
 data2 = pd.read_csv('.. /input/m5-dark-magic/submission.csv' )<define_variables>
clf=tree.DecisionTreeClassifier(criterion='entropy',max_depth=4)
 clf=clf.fit(X_train,Y_train)
 dot_data = tree.export_graphviz(clf, out_file=None,feature_names=X_train.columns.values.tolist())
 graph = graphviz.Source(dot_data)
 graph
<define_variables><EOS> 
Y_pred=clf.predict(test_df)
 my_submission = pd.DataFrame({'PassengerId': test_passengerId, 'Survived': Y_pred})
 my_submission.to_csv('submission.csv', index=False)