Kaggle competition: New York City Taxi Fare Prediction
train_new = train_new.drop(['Cabin'],axis = 1)
 test_new = test_new.drop(['Cabin'],axis = 1 )<normalization>
def incremental_learning(data, model):
start = time.time()
data["pickup_datetime"] = pd.to_datetime(data["pickup_datetime"], infer_datetime_format=True)
 print("Conversion took", time.time() - start, "s")
 add_travel_vector_features(data)
 data = pd.concat([data, timeExpansion(data["pickup_datetime"])], axis=1)
 X_tr, X_te, y_tr, y_te = train_test_split(
 data.drop(["key", "pickup_datetime", "fare_amount"], axis=1),
data['fare_amount'],
test_size=0.2)
 del data
params = {
'boosting_type': 'gbdt',
'objective': 'regression_l1',
'metric': {'l1'},
'num_leaves': 12,
'learning_rate': 0.1,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'random_state': 77,
'n_jobs': 4,
'bagging_freq': 5,
'verbose': 2
}
dset = lgbm.Dataset(X_tr, y_tr, free_raw_data=True)
 model = lgbm.train(params,
init_model = model,
train_set = dset,
keep_training_booster = True,
num_boost_round = 100)
 MAE = mean_absolute_error(y_te, model.predict(X_te))
print("Mean Absolute Error:", MAE)
 return model
sc = StandardScaler()
train3 = sc.fit_transform(train_new)
 test3 = sc.transform(test_new )<choose_model_class>
def partial_import(filename, skip, rows):
return pd.read_csv('.. input/train.csv', skiprows=skip, nrows=rows)
 def add_travel_vector_features(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
def month_translation(number):
month_list = ["jan", "feb", "mar",
"apr", "may", "jun", "jul",
"aug", "sep", "oct", "nov", "dec"]
for num, i in enumerate(month_list):
if num == number-1:
return i
def quarter_translation(number):
quarter_border = [0, 3, 6, 9, 12, 15, 18, 21, 24]
quarter_list = ["11", "12", "13", "14", "21", "22", "23", "24"]
for num, i in enumerate(range(1, len(quarter_border))):
if(number < quarter_border[i])&(number >= quarter_border[i-1]):
return quarter_list[num]
def week_translation(number):
week_border = [0, 7, 14, 21, 31]
week_list = ["1W", "2W", "3W", "4W"]
for num, i in enumerate(range(1, len(week_border))):
if(number <= week_border[i])&(number > week_border[i-1]):
return week_list[num]
def timeExpansion(timeSeries):
additional_cols = ["year", "month", "quarter", "week"]
dummy0 = pd.DataFrame(np.zeros(( len(timeSeries), len(additional_cols)) , 'int'),
columns=additional_cols,
index=timeSeries.index)
 dummy0["year"] = [i.year for i in timeSeries]
dummy0["month"] = [month_translation(i.month)for i in timeSeries]
dummy0["day"] = [week_translation(i.day)for i in timeSeries]
dummy0["quarter"] = [quarter_translation(i.hour)for i in timeSeries]
return pd.get_dummies(dummy0)
 def predict_and_submit(submit=True):
if "model.pkl" in files:
with open('.. /model.pkl', 'rb')as fin:
estimator = pickle.load(fin)
 test_df = pd.read_csv('.. /input/test.csv')
 add_travel_vector_features(test_df)
 test_df["pickup_datetime"] = pd.to_datetime(test_df["pickup_datetime"], infer_datetime_format=True)
 test_X = pd.concat([test_df, timeExpansion(test_df["pickup_datetime"])], axis=1)
 test_X.drop(["key", "pickup_datetime"], axis=1, inplace=True)
 print("Test Shape: ",test_X.shape)
 test_y_predictions = estimator.predict(test_X)
 if submit:
submission = pd.DataFrame(
 {'key': test_df.key, 'fare_amount': test_y_predictions},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )
<define_search_space><EOS> 
training_step = list(np.arange(40_000_000, 55_000_001, 1_000_000))
estimator = None
origin_cols = ['key',
'fare_amount',
'pickup_datetime',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'passenger_count']
for i in range(1, len(training_step)) :
start = time.time()
files = os.listdir(".. /")
 if "model.pkl" in files:
with open('.. /model.pkl', 'rb')as fin:
estimator = pickle.load(fin)
 print(training_step[i-1], training_step[i])
 train_df = pd.read_csv(".. /input/train.csv", skiprows=training_step[i-1], nrows=training_step[i])
 train_df.columns = origin_cols
estimator = incremental_learning(train_df, estimator)
 print("Operation", i, "took ", time.time() - start, "
")
 del train_df
with open('.. /model.pkl', 'wb')as fout:
pickle.dump(estimator, fout)
 del estimator
gc.collect()
predict_and_submit()