Kaggle competition: Titanic - Machine Learning from Disaster
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<define_variables>
train_df = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df = pd.read_csv("/kaggle/input/titanic/test.csv" )
debug = False<feature_engineering>
train_df.isnull().sum()
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek 
<categorify>
test_df.isnull().sum()
preprocess(train_df )<categorify>
train_df.isnull().sum()
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 train_df['building_mean'] = train_df['building_id'].map(building_mean)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 train_df['building_min'] = train_df['building_id'].map(building_min)
 train_df['building_max'] = train_df['building_id'].map(building_max)
 train_df['building_std'] = train_df['building_id'].map(building_std )<count_missing_values>
train_df.drop('Cabin', axis=1, inplace=True)
 train_df.isnull().sum()
weather_train_df.isna().sum()<groupby>
train_df.dropna(subset=['Embarked'], inplace=True)
 train_df.isnull().sum()
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
train_df['Age'].fillna(train_df['Age'].mean() , inplace=True)
 train_df.isnull().sum()
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
test_df.isnull().sum()
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<data_type_conversions>
test_df.drop('Cabin', axis=1, inplace=True)
 test_df.isnull().sum()
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<feature_engineering>
test_df['Fare'] = test_df['Fare'].fillna(train_df['Fare'].median())
 test_df.isnull().sum()
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )<categorify>
test_df['Age'].fillna(train_df['Age'].mean() , inplace=True)
 test_df.isnull().sum()
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<define_variables>
X = train_df.drop(['PassengerId', 'Survived', 'Name', 'Ticket', 'Fare'], axis=1)
 y = train_df['Survived']
n_test_df = test_df.drop(['PassengerId', 'Name', 'Ticket', 'Fare'], axis=1 )
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']<count_unique_values>
X = pd.get_dummies(X, drop_first=True )
weather_train_df['site_id'].nunique()<prepare_x_and_y>
n_test_df = pd.get_dummies(n_test_df, drop_first=True )
def create_X_y(train_df, target_meter, site_id):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 target_train_df = target_train_df[target_train_df['site_id'] == site_id]
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42 )
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
parameters = {"learning_rate": [0.1, 0.01, 0.001],
"gamma" : [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2],
"max_depth": [2, 4, 7, 10],
"colsample_bytree": [0.3, 0.6, 0.8, 1.0],
"subsample": [0.2, 0.4, 0.5, 0.6, 0.7],
"reg_alpha": [0, 0.5, 1],
"reg_lambda": [1, 1.5, 2, 3, 4.5],
"min_child_weight": [1, 3, 5, 7],
"n_estimators": [100, 250, 500, 1000]}
xgb_rscv = RandomizedSearchCV(XGBClassifier() , param_distributions = parameters, cv = 7, verbose = 3, random_state = 40)
 model_rscv = xgb_rscv.fit(X_train, y_train)
 model_rscv.best_params_
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed )<groupby>
tuned_model = XGBClassifier(booster='gbtree', subsample= 0.7,
reg_lambda= 3,
reg_alpha= 1,
n_estimators= 100,
min_child_weight= 3,
max_depth= 10,
learning_rate = 0.001,
gamma= 0.01,
colsample_bytree= 0.6)
 tuned_model.fit(X_train, y_train )
<train_model>
tuned_model.score(X_test, y_test )
<load_pretrained><EOS> 
predictions = tuned_model.predict(n_test_df)
 output = pd.DataFrame({'PassengerId': test_df.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )