Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<compute_test_metric>
warnings.filterwarnings("ignore")
RMSLE = RMSLE_Func(y_test, predictions)
 print(RMSLE)
 del x_test, y_test, predictions<predict_on_test>
train = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/train.csv")
 test = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/test.csv")
 trainlabel = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/trainlabel.csv" )
%%time
predictions = []
step = 50000
for i in range(0, len(test), step):
predictions.extend(np.expm1(reg.predict(test.iloc[i: min(i+step, len(test)) , :], num_iteration=reg.best_iteration)))
<save_to_csv>
dtrain = xgb.DMatrix(train, label = trainlabel )
%%time
Submission = pd.DataFrame(test.index,columns=['row_id'])
 Submission['meter_reading'] = predictions
Submission['meter_reading'].clip(lower=0,upper=None,inplace=True)
 Submission.to_csv("lgbm.csv",index=None)
<load_from_csv>
w0 = average *(1 - p)/(( 1 - average)* p)
 print(w0 )
train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')
 building_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')
 weather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv' )<data_type_conversions>
w1 = average / p
w2 =(1 - average)/(1 - p)
 print(w1, w2 )
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 def get_meteorological_features(data):
def calculate_rh(df):
df['relative_humidity'] = 100 *(np.exp(( 17.625 * df['dew_temperature'])/(243.04 + df['dew_temperature'])) / np.exp(( 17.625 * df['air_temperature'])/(243.04 + df['air_temperature'])))
 def calculate_fl(df):
flike_final = []
flike = []
for i in range(len(df)) :
at = df['air_temperature'][i]
rh = df['relative_humidity'][i]
ws = df['wind_speed'][i]
flike.append(feels_like(Temp(at, unit = 'C'), rh, ws))
for i in range(len(flike)) :
flike_final.append(flike[i].f)
 df['feels_like'] = flike_final
del flike_final, flike, at, rh, ws
calculate_rh(data)
 calculate_fl(data)
 return data
weather_df = get_meteorological_features(weather_df)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["dayofweek"] = df["timestamp"].dt.weekday
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04",
"2016-09-05", "2016-10-10", "2016-11-11", "2016-11-24", "2016-12-26",
"2017-01-02", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04",
"2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25",
"2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04",
"2018-09-03", "2018-10-08", "2018-11-12", "2018-11-22", "2018-12-25",
"2019-01-01"]
df['group'] = df['timestamp'].dt.month
df['group'].replace(( 1, 2, 3, 4), 1, inplace = True)
 df['group'].replace(( 5, 6, 7, 8), 2, inplace = True)
 df['group'].replace(( 9, 10, 11, 12), 3, inplace = True)
 df["is_holiday"] =(df.timestamp.isin(holidays)).astype(int)
 df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df<merge>
def weighted_log_loss(preds, dtrain):
label = dtrain.get_label()
return "weighted_logloss", -np.mean(w1 * label * np.log(preds)+ w2 *(1 - label)* np.log(1 - preds))
weather_df = fill_weather_dataset(weather_df)
 train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True)
 train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
train_df = features_engineering(train_df)
 train_df['meter_reading'] = np.log1p(train_df["meter_reading"] )<load_from_csv>
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.1
params["max_depth"] = 6
params["min_child_weight"] = 1
params["gamma"] = 0
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params['max_bin'] = 256
model1 = xgb.cv(params, dtrain, num_boost_round = 2000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 200,
verbose_eval = 50 )
bad_rows = pd.read_csv('/kaggle/input/rows-do-drop/rows_to_drop.csv')
 train_df.drop(bad_rows.loc[:, '0'], inplace = True)
 train_df.reset_index(drop = True, inplace = True )<define_variables>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_bin"] = 256
evaluation_list = []
for depth in [5, 6]:
for child_weight in [1, 2.5, 4]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 for depth in [7, 8]:
for child_weight in [4, 5, 6]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
target = 'meter_reading'
categorical = ['building_id', 'site_id', 'primary_use', 'meter', 'is_holiday', 'dayofweek']
numeric_cols = [col for col in train_df.columns if col not in categorical + [target, 'timestamp', 'group']]
features = categorical + numeric_cols<choose_model_class>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_bin"] = 256
evaluation_list = []
for depth in [5, 6, 7]:
for child_weight in [3, 3.5, 4, 4.5]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
def run_lgbm(train, cat_features = categorical, num_rounds = 20000, folds = 3):
kf = GroupKFold(n_splits = folds)
 models = []
param = {'num_leaves': 500,
'objective': 'regression',
'learning_rate': 0.05,
'boosting': 'gbdt',
'subsample': 0.4,
'feature_fraction': 0.7,
'n_jobs': -1,
'seed': 50,
'metric': 'rmse'
}
oof = np.zeros(len(train))
for tr_idx, val_idx in tqdm(kf.split(train, groups = train['group']), total = folds):
tr_x, tr_y = train[features].iloc[tr_idx], train[target].iloc[tr_idx]
vl_x, vl_y = train[features].iloc[val_idx], train[target].iloc[val_idx]
tr_data = lgb.Dataset(tr_x, label = tr_y, categorical_feature = categorical)
 vl_data = lgb.Dataset(vl_x, label = vl_y, categorical_feature = categorical)
 clf = lgb.train(param, tr_data, num_rounds, valid_sets = [tr_data, vl_data], verbose_eval = 25,
early_stopping_rounds = 50)
 models.append(clf)
 oof[val_idx] = clf.predict(vl_x)
 gc.collect()
score = np.sqrt(metrics.mean_squared_error(train[target], np.clip(oof, a_min=0, a_max=None)))
 print('Our oof cv is :', score)
 return models
models = run_lgbm(train_df )<load_from_csv>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 4
evaluation_list = []
for bin in [200, 230, 256, 280]:
params = {**fix_params, **{"max_bin": bin}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df)
 test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
weather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
 test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
test_df = features_engineering(test_df )<load_from_csv>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.08
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 3.5
evaluation_list = []
for bin in [220, 240, 270]:
params = {**fix_params, **{"max_bin": bin}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
def predictions(models, iterations = 50):
set_size = len(test_df)
 batch_size = set_size // iterations
meter_reading = []
for i in tqdm(range(iterations)) :
pos = i*batch_size
fold_preds = [np.expm1(model.predict(test_df[features].iloc[pos : pos+batch_size])) for model in models]
meter_reading.extend(np.mean(fold_preds, axis=0))
print(len(meter_reading))
assert len(meter_reading)== set_size
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = np.clip(meter_reading, a_min=0, a_max=None)
 submission.to_csv('fe2_lgbm.csv', index=False)
 print('We are done!')
 predictions(models )<set_options>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 4
fix_params["max_bin"] = 256
evaluation_list = []
for row in [0.7, 0.8, 0.9]:
for col in [0.7, 0.8, 0.9]:
params = {**fix_params, **{"subsample": row, "colsample_bytree": col}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
py.init_notebook_mode(connected=True)
<set_options>
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 4
fix_params["max_bin"] = 256
evaluation_list = []
for row in [0.75, 0.8, 0.85]:
for col in [0.85, 0.9]:
params = {**fix_params, **{"subsample": row, "colsample_bytree": col}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.06
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
model6 = xgb.cv(params, dtrain, num_boost_round = 6000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 150,
verbose_eval = 50 )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.04
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
model4 = xgb.cv(params, dtrain, num_boost_round = 6000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 150,
verbose_eval = 50 )
leak_df = pd.read_feather('.. /input/ashare-leak-data-station-2/leak2.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]<count_values>
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.02
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
model2 = xgb.cv(params, dtrain, num_boost_round = 6000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 150,
verbose_eval = 50 )
leak_df.meter.value_counts()<count_duplicates>
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.02
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
dtest = xgb.DMatrix(test)
 t = pd.read_csv(".. /input/quora-question-pairs/test.csv" )
print(leak_df.duplicated().sum() )<feature_engineering>
model = xgb.train(params, dtrain, num_boost_round = 3600)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission3600.csv', index=False )
print(len(leak_df)/ len(train_df))<set_options>
model = xgb.train(params, dtrain, num_boost_round = 3800)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission3800.csv', index=False )
del train_df
gc.collect()<load_from_csv>
model = xgb.train(params, dtrain, num_boost_round = 4100)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission4100.csv', index=False )