Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<import_modules>
grph = True
kaggle = True
if grph:
def evaluate_model(model):

 accuracy = cross_val_score(model, mytrain, target, cv=5, scoring='accuracy' ).mean()
return accuracy
models = {
'Logistic regression':LogisticRegression(random_state = 42),
'Decision tree':DecisionTreeClassifier(random_state = 42),
'Random forest':RandomForestClassifier(random_state = 42)
 }
for key, mymodel in models.items() :
accuracy = evaluate_model(model = mymodel)
 print('{:20}: {:2.2f}%'.format(key, accuracy*100))<import_modules>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_colwidth', -1)
 pd.set_option('mode.chained_assignment','raise' )
from sklearn.model_selection import GridSearchCV<choose_model_class>
def read_files(chunks = 1100, chunk_no = None):
random.seed(42)
 nrows = int(55423857 / chunks)
 def skiprows(x):
if x == 0:
return False
return chunk_no != random.randrange(chunks)
 if kaggle:
df = pd.read_csv('.. /input/train.csv', nrows = nrows, parse_dates=["pickup_datetime"])
 else:
with zipfile.ZipFile('all.zip')as zipf:
with zipf.open('train.csv')as myZip:
if chunk_no == None:
df = pd.read_csv(myZip,nrows=nrows,parse_dates=["pickup_datetime"])
 else:
df = pd.read_csv(myZip,skiprows=skiprows,parse_dates=["pickup_datetime"])
 random.seed(42)
 df = df.sample(frac=1 ).reset_index(drop=True)
 if kaggle:
df_test = pd.read_csv('.. /input/test.csv', nrows = nrows, parse_dates=["pickup_datetime"])
 else:
with zipfile.ZipFile('all.zip')as zipf:
with zipf.open('test.csv')as myZip:
df_test = pd.read_csv(myZip,parse_dates=["pickup_datetime"])
 return df, df_test
models = [
{
'name':'Logistic regression',
'estimator':LogisticRegression(random_state = 42),
'hyperparameters':{
'solver': ['newton-cg', 'lbfgs', 'liblinear']
}
},
{
'name':'Decision tree',
'estimator':DecisionTreeClassifier(random_state = 42),
'hyperparameters':{
'criterion':['entropy','gini'],
'splitter':['best','random'],
'max_depth':range(4,9),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[3,4,5]
}
}, {
'name':'RandomForestClassifier',
'estimator':RandomForestClassifier(random_state = 42),
'hyperparameters':{
'n_estimators' : [10, 15, 18, 19, 20, 21, 22],
'criterion':['entropy','gini'],
'max_depth':range(4,9),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[3,4,5]
}
}
]
def tune_models(models):
for model in models:
print(model['name'])
 print('-' * len(model['name']))
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5, scoring='accuracy')
 grid.fit(mytrain, target)
 model['best_score'] = grid.best_score_
model['best_params'] = grid.best_params_
model['best_model'] = grid.best_estimator_
print('Best accuracy: {:.2f}%'.format(model['best_score']*100))
print('Best parameters: {}
'.format(model['best_params']))
return models
models = tune_models(models )<choose_model_class>
chunks = 8
mymodel = RandomForestClassifier(criterion='entropy', max_depth=7, max_features = 'auto', min_samples_split = 4, n_estimators = 18, random_state=42 )<compute_test_metric>
df,df_test = read_files(chunks )
mymodel.fit(mytrain, target)
 myprediction = mymodel.predict(mytrain)
 print(f'Precision: {precision_score(myprediction, target)*100:.2f}%')
 print(f'Accuracy: {accuracy_score(myprediction, target)*100:.2f}%')
<save_to_csv>
def calculate_distance(data):
for df in data:
R=6371.0
pi=math.pi
df_tmp = df.eval(,inplace=False)
 df['dist'] = df_tmp['distance']
df['dist2'] = df_tmp['distance'] * df_tmp['distance']
return data
myprediction = mymodel.predict(mytest)
 submission = pd.DataFrame({'PassengerId':mytest.index,'Survived':myprediction})
 submission.to_csv('submissionRFC_nosplit.csv',index=False )<import_modules>
long_km =.1 / 11.160876733592401
lat_km =.1 / 3.077915051090274
def is_airport(data):
for df in data:
df["is_jfk_pickup"] = False
df.loc[df.eval(
 '(-73.778889 - 2*@long_km < pickup_longitude < -73.778889 + 2*@long_km)\
&\ 
(40.639722 - 1*@lat_km < pickup_latitude < 40.639722 +.5*@lat_km)')
 ,"is_jfk_pickup"] = True
df["is_ewr_pickup"] = False
df.loc[df.eval(
 '(-74.168611 - 2*@long_km < pickup_longitude < -74.168611 + 2*@long_km)\
&\ 
(40.6925 - 1*@lat_km < pickup_latitude < 40.6925 + 1*@lat_km)')
 ,"is_ewr_pickup"] = True
df["is_lga_pickup"] = False
df.loc[df.eval(
 '(-73.872611 - 1.4*@long_km < pickup_longitude < -73.872611 + 1.5*@long_km)\
&\ 
(40.77725 -.3*@lat_km < pickup_latitude < 40.77725 +.3*@lat_km)')
 ,"is_lga_pickup"] = True
df["is_jfk_dropoff"] = False
df.loc[df.eval(
 '(-73.778889 - 2*@long_km < dropoff_longitude < -73.778889 + 2*@long_km)\
&\ 
(40.639722 - 1*@lat_km < dropoff_latitude < 40.639722 +.5*@lat_km)')
 ,"is_jfk_dropoff"] = True
df["is_ewr_dropoff"] = False
df.loc[df.eval(
 '(-74.168611 - 2*@long_km < dropoff_longitude < -74.168611 + 2*@long_km)\
&\ 
(40.6925 - 1*@lat_km < dropoff_latitude < 40.6925 + 1*@lat_km)')
 ,"is_ewr_dropoff"] = True
df["is_lga_dropoff"] = False
df.loc[df.eval(
 '(-73.872611 - 1.4*@long_km < dropoff_longitude < -73.872611 + 1.5*@long_km)\
&\ 
(40.77725 -.3*@lat_km < dropoff_latitude < 40.77725 +.3*@lat_km)')
 ,"is_lga_dropoff"] = True
return data
from sklearn.model_selection import train_test_split<import_modules>
df,df_test = calculate_distance([df,df_test] )
from sklearn.model_selection import train_test_split<train_model>
df,df_test = is_airport([df,df_test] )
train_x, test_x, train_y, test_y = train_test_split(mytrain, target, test_size = 0.2, random_state=42)
 mymodel = RandomForestClassifier(criterion='entropy', max_depth=7, max_features = 'auto', min_samples_split = 4, n_estimators = 18, random_state=42)
 mymodel.fit(train_x, train_y)
 mymodel.score(test_x, test_y)
<predict_on_test>
def is_business_day(date):
return bool(len(pd.bdate_range(date, date)))
 def time_related_vars(data):
for df in data:
df['weekday'] = df['pickup_datetime'].dt.weekday
df['day'] = df['pickup_datetime'].dt.day
df['hour'] = df['pickup_datetime'].dt.hour
df['month'] = df['pickup_datetime'].dt.month
df['year'] = df['pickup_datetime'].dt.year
df['daytime'] = False
df.loc[(df.hour >= 8)&(df.hour < 20),'daytime'] = True
df['time_stamp'] = df['year'].astype('str')+ '-' + df['month'].astype('str')+ '-' + df['day'].astype('str')+ ':' + df['hour'].astype('str')
 return data
myprediction = mymodel.predict(mytest )<save_to_csv>
df,df_test = time_related_vars(( df,df_test))
submission = pd.DataFrame({'PassengerId':mytest.index,'Survived':myprediction})
 submission.to_csv('submissionRF_optim_param.csv',index=False )<set_options>
grp = df.groupby('time_stamp',as_index=False)['fare_amount'].agg(['sum','mean','std','skew','count'] ).reset_index()
df = df.merge(grp, how='left', on=['time_stamp'])
 df_test = df_test.merge(grp, how='left', on=['time_stamp'] )
rc('animation', html='jshtml')
 device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<load_pretrained>
def basic_fare() :
df['min_fare'] = 2.5 + 0.5 * 5 * df.dist / 1.6
df_test['min_fare'] = 2.5 + 0.5 * 5 * df_test.dist / 1.6
df['min_multiple'] = df['fare_amount'] / df['min_fare']
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge')
 train_path = data_path / 'training'
valid_path = data_path / 'evaluation'
test_path = data_path / 'test'
train_tasks = { task.stem: json.load(task.open())for task in train_path.iterdir() }
valid_tasks = { task.stem: json.load(task.open())for task in valid_path.iterdir() }<categorify>
if not kaggle:
df.to_pickle('df.pkl')
 df_test.to_pickle('df_test.pkl' )
class CAModel(nn.Module):
def __init__(self, num_states):
super(CAModel, self ).__init__()
self.transition = nn.Sequential(
 nn.Conv2d(num_states, 32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.Conv2d(32, num_states, kernel_size=1)
)
 def forward(self, x, steps=1):
for _ in range(steps):
x = self.transition(torch.softmax(x, dim=1))
return x<find_best_params>
if not kaggle:
df = pd.read_pickle('df.pkl')
 df_test = pd.read_pickle('df_test.pkl' )
def solve_task(task, max_steps=12):
model = CAModel(10 ).to(device)
 num_epochs = 100
num_repeat = 25
criterion = nn.CrossEntropyLoss()
losses = np.zeros(( num_repeat - 1)* num_epochs)
 for i in range(1, num_repeat):
if i > num_repeat-4:
num_steps = max_steps
else:
num_steps = random.randint(1, max_steps)
 optimizer = torch.optim.Adam(model.parameters() , lr=(0.04 / i))
for e in range(num_epochs):
optimizer.zero_grad()
loss = 0.0
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 y = torch.tensor(sample["output"] ).long().unsqueeze(0 ).to(device)
 y_pred = model(x, num_steps)
 loss += criterion(y_pred, y)
 y_in = torch.from_numpy(inp2img(sample["output"])).unsqueeze(0 ).float().to(device)
 y_pred = model(y_in, 1)
 loss += criterion(y_pred, y)
 loss.backward()
optimizer.step()
losses[(i - 1)* num_epochs + e] = loss.item()
return model, num_steps, losses
@torch.no_grad()
def predict(model, task):
predictions = []
for sample in task:
x = torch.from_numpy(inp2img(sample["input"])).unsqueeze(0 ).float().to(device)
 pred = model(x, 200 ).argmax(1 ).squeeze().cpu().numpy()
predictions.append(pred)
 return predictions
task = train_tasks["db3e9e38"]["train"]
model, num_steps, losses = solve_task(task )<predict_on_test>
df.dropna(inplace=True)
 df.drop(df.index[(df.pickup_longitude < -75)| 
(df.pickup_longitude > -72)| 
(df.pickup_latitude < 40)| 
(df.pickup_latitude > 42)],inplace=True)
 df.drop(df.index[(df.dropoff_longitude < -75)| 
(df.dropoff_longitude > -72)| 
(df.dropoff_latitude < 40)| 
(df.dropoff_latitude > 42)],inplace=True )
predictions = predict(model, task)
 for i in range(len(task)) :
plot_sample(task[i], predictions[i] )<predict_on_test>
max_fare_multiple = 20
min_fare_multiple = 0.05
test = train_tasks["db3e9e38"]["test"]
predictions = predict(model, test)
 for i in range(len(test)) :
plot_sample(test[i], predictions[i] )<predict_on_test>
drop_too_expensive = df.index[(df.fare_amount >(max_fare_multiple *(2.5 + 0.5 * 5 * df.dist / 1.6)))]
print("n_dropped={}, {}%".format(len(drop_too_expensive),len(drop_too_expensive)/len(df)))
 df.drop(drop_too_expensive,inplace=True )
def evaluate(tasks):
result = []
predictions = []
for idx, task in tqdm(tasks.items()):
if input_output_shape_is_same(task):
model, _, _ = solve_task(task["train"])
 pred = predict(model, task["test"])
 score = calk_score(task["test"], pred)
 else:
pred = [el["input"] for el in task["test"]]
score = [0] * len(task["test"])
 predictions.append(pred)
 result.append(score)
 return result, predictions<compute_test_metric>
drop_too_cheap = df.index[df.fare_amount <(min_fare_multiple *(2.5 + 0.5 * 5 * df.dist / 1.6)) ]
print("n_dropped={}, {}%".format(len(drop_too_cheap),len(drop_too_cheap)/len(df)))
 df.drop(drop_too_cheap,inplace=True )
train_result, train_predictions = evaluate(train_tasks)
 train_solved = [any(score)for score in train_result]
total = sum([len(score)for score in train_result])
 print(f"solved : {sum(train_solved)} from {total}({sum(train_solved)/total})" )<import_modules>
df['float_pickup_datetime'] = df['pickup_datetime'].values.astype('float')
 df_test['float_pickup_datetime'] = df_test['pickup_datetime'].values.astype('float' )
import pandas as pd<import_modules>
dftr = df.sample(frac = 0.8, random_state=42)
 dfval = df.drop(dftr.index )
import pandas as pd<load_from_csv>
ll = ['passenger_count','dist','bday','weekday','hour','daytime','businesstime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude','lat_diff','long_diff']
ll = ['dist']
ll = ['passenger_count','dist']
ll = ['dist','bday']
ll = ['dist','bday','weekday']
ll = ['dist','bday','weekday','hour']
ll = ['dist','bday','weekday','hour','daytime']
ll = ['dist','bday','weekday','hour','businesstime']
ll = ['dist','bday','weekday','hour','daytime','businesstime']
ll = ['dist','month']
ll = ['dist','bday','weekday','hour','daytime','month']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude','lat_diff','long_diff']
ll = ['dist','bday','weekday','hour','daytime','month','lat_avg','long_avg','lat_diff','long_diff']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']
ll = ['dist','dist2','weekday','hour','daytime','year','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude',
'is_jfk_pickup', 'is_ewr_pickup', 'is_lga_pickup', 'is_jfk_dropoff', 'is_ewr_dropoff', 'is_lga_dropoff']
ll = ['dist','dist2','float_pickup_datetime', 'weekday','hour','daytime','year','month','pickup_longitude',
'pickup_latitude','dropoff_longitude','dropoff_latitude','is_jfk_pickup', 'is_ewr_pickup',
'is_lga_pickup', 'is_jfk_dropoff', 'is_ewr_dropoff', 'is_lga_dropoff',
'sum','mean','std','skew','count']
submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='output_id')
 display(submission.head() )<categorify>
ntr = len(dftr['dist'])
 nval = len(dfval['dist'])
 nte = len(df_test['dist'])
 Xtr = pd.DataFrame(dftr[ll] ).values.reshape(ntr, len(ll))
Xval = pd.DataFrame(dfval[ll] ).values.reshape(nval, len(ll))
Xte = pd.DataFrame(df_test[ll] ).values.reshape(nte, len(ll))
Ytr = dftr['min_multiple'].values
Yval = dfval['min_multiple'].values
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<predict_on_test>
max_bin = 2*255
gbm = lgb.LGBMRegressor(n_estimators=10000, silent=False, max_bin=max_bin )
for output_id in tqdm(submission.index):
task_id = output_id.split('_')[0]
pair_id = int(output_id.split('_')[1])
 f = str(test_path / str(task_id + '.json'))
with open(f, 'r')as read_file:
task = json.load(read_file)
 if input_output_shape_is_same(task):
print(task_id)
 model, _, _ = solve_task(task["train"])
 pred = predict(model, task["train"])
 for i in range(len(task['train'])) :
plot_sample(task['train'][i], pred[i])
 pred = predict(model, task["test"])
 for i in range(len(task['test'])) :
plot_sample(task['test'][i], pred[i])
 pred = flattener(pred[pair_id].tolist())
 else:
data = task['test'][pair_id]['input']
pred_1 = flattener(data)
 data = [[5 if i==0 else i for i in j] for j in data]
pred_2 = flattener(data)
 data = [[0 for i in j] for j in data]
pred_3 = flattener(data)
 pred = pred_1 + ' ' + pred_2 + ' ' + pred_3 + ' '
submission.loc[output_id, 'output'] = pred
submission.to_csv('submission.csv' )<import_modules>
parameters = gbm.fit(Xtr, Ytr )
import json
import re
import os
import numpy as np
import pandas as pd<categorify>
if not kaggle:
pickle.dump(gbm, open('model.pkl', 'wb'))
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<define_variables>
if not kaggle:
gbm = pickle.load(open('model.pkl', 'rb'))
testDirectory = "/kaggle/input/abstraction-and-reasoning-challenge/test/"<load_from_disk>
Ytest = gbm.predict(Xte )
def readTaskFile(filename):
f = open(filename, "r")
 data = json.loads(f.read())
 data["id"] = re.sub(" (.*/)|(\.json)", "", filename)
 f.close()
return data<load_pretrained>
df_test['fare_amount'] = pd.Series(Ytest)* df_test['min_fare']
df_test[['key','fare_amount']].to_csv('submission.csv',index=False )
filename = testDirectory+"3b4c2228.json"
readTaskFile(filename )<categorify>
print("FARE MULTIPLE, LGBM")
 print(parameters)
 print(ll)
 print("df size: {}".format(len(df)))
 print("chunks: {}".format(chunks))
print("max bin: {}".format(max_bin))
print("min,max_fare_multiple {} {}".format(min_fare_multiple,max_fare_multiple))
def getNoopAnswer(filename):
data = readTaskFile(filename)
 testSection = data["test"]
ident = data["id"]
numTests = len(testSection)
 answer = {}
for i in range(numTests):
answer[ident+"_"+str(i)] = flattener(testSection[i]["input"])
 return answer<define_variables>
Ytr_hat = gbm.predict(Xtr)
 Yval_hat = gbm.predict(Xval)
filename = testDirectory+"3b4c2228.json"
getNoopAnswer(filename )<categorify>
dftr['fare_predicted'] = pd.Series(Ytr)* dftr['min_fare']
dfval['fare_predicted'] = pd.Series(Yval)* dfval['min_fare']
dftr['fare_err'] =(dftr['fare_amount'] - dftr['fare_predicted'] ).abs()
dfval['fare_err'] =(dfval['fare_amount'] - dfval['fare_predicted'] ).abs()
def getAnswers(directory, f):
answers = {}
for _, _, filenames in os.walk(directory):
for filename in filenames:
answers.update(f(directory+filename))
return answers<save_to_csv>
rms_tr =(dftr['fare_err']**2 ).mean() **.5
print("Accuracy training: ", rms_tr )
answers = getAnswers(testDirectory, getNoopAnswer)
 f = open("submission.csv", "w")
 f.write("output_id,output
")
 keys = answers.keys()
for key in keys:
value = answers[key]
f.write(key + "," + value + "
")
 f.close()<load_from_csv>
rms_val =(dfval['fare_err']**2 ).mean() **.5
print("Accuracy2: ", rms_val )
f2 = open("submission.csv", "r")
 print(f2.read())
 f2.close()<define_variables>
err100 = dfval.nlargest(100,'fare_err')
 err100.to_csv('100errval.csv' )