Kaggle competition: New York City Taxi Fare Prediction
data_path = Path('/kaggle/input/abstraction-and-reasoning-challenge/')
 training_path = data_path / 'training'
evaluation_path = data_path / 'evaluation'
test_path = data_path / 'test'
training_tasks = sorted(os.listdir(training_path))
evaluation_tasks = sorted(os.listdir(evaluation_path))
test_tasks = sorted(os.listdir(test_path))
print(len(training_tasks), len(evaluation_tasks), len(test_tasks))<load_from_disk>
% matplotlib inline
plt.style.use('seaborn-whitegrid' )
def get_data(task_filename):
with open(task_filename, 'r')as f:
task = json.load(f)
 return task
num2color = ["black", "blue", "red", "green", "yellow", "gray", "magenta", "orange", "sky", "brown"]
color2num = {c: n for n, c in enumerate(num2color)}<categorify>
df_test = pd.read_csv(".. /input/test.csv")
 df_train = pd.read_csv('.. /input/train.csv', nrows = 500_000,)
 df_train.dtypes
def flattener(pred):
str_pred = str([row for row in pred])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<load_from_csv>
df_train.isnull().sum()
sample_sub = pd.read_csv(data_path/'sample_submission.csv')
 sample_sub = sample_sub.set_index('output_id')
 sample_sub.head()<train_model>
df_train=df_train.dropna()
df_test=df_test.dropna()
print(df_train.isnull().sum())
 print(df_test.isnull().sum() )
def check(task, learn_func, pred_func):
n = len(task["train"])+ len(task["test"])
 fig, axs = plt.subplots(3, n, figsize=(4*n,12), dpi=50)
 plt.subplots_adjust(wspace=0.3, hspace=0.3)
 fig_num = 0
param_ls=[]
for i, t in enumerate(task["train"]):
t_in, t_out = np.array(t["input"]), np.array(t["output"])
 t_pred, param = learn_func(t_in,t_out)
 if param==False:
print('Not suitable')
 plt.close('all')
 return
param_ls.append(param)
 axs[0][fig_num].imshow(t_in, cmap=cmap, norm=norm)
 axs[0][fig_num].set_title(f'Train-{i} in')
 axs[0][fig_num].set_yticks(list(range(t_in.shape[0])))
 axs[0][fig_num].set_xticks(list(range(t_in.shape[1])))
 axs[1][fig_num].imshow(t_out, cmap=cmap, norm=norm)
 axs[1][fig_num].set_title(f'Train-{i} out')
 axs[1][fig_num].set_yticks(list(range(t_out.shape[0])))
 axs[1][fig_num].set_xticks(list(range(t_out.shape[1])))
 axs[2][fig_num].imshow(t_pred, cmap=cmap, norm=norm)
 axs[2][fig_num].set_title(f'Train-{i} pred')
 axs[2][fig_num].set_yticks(list(range(t_pred.shape[0])))
 axs[2][fig_num].set_xticks(list(range(t_pred.shape[1])))
 fig_num += 1
if len(set(param_ls)) ==1:
param=param_ls[0]
for i, t in enumerate(task["test"]):
t_in = np.array(t["input"])
 t_pred = pred_func(t_in,param)
 axs[0][fig_num].imshow(t_in, cmap=cmap, norm=norm)
 axs[0][fig_num].set_title(f'Test-{i} in')
 axs[0][fig_num].set_yticks(list(range(t_in.shape[0])))
 axs[0][fig_num].set_xticks(list(range(t_in.shape[1])))
 axs[1][fig_num].imshow(t_pred, cmap=cmap, norm=norm)
 axs[1][fig_num].set_title(f'Test-{i} pred')
 axs[1][fig_num].set_yticks(list(range(t_pred.shape[0])))
 axs[1][fig_num].set_xticks(list(range(t_pred.shape[1])))
 fig_num += 1
print(t_pred)
 if mode=='test':
print('mode test')
 sub_ls=flattener(t_pred.tolist())
 sample_sub.loc[f'{test_tasks[iteration][:-5]}_{i}','output'] = sample_sub.loc[f'{test_tasks[iteration][:-5]}_{i}','output'] +' '+sub_ls
plt.show()
else:
print('NOT THIS TYPE')
 return<concatenate>
df_train.query("fare_amount <= 0" ).count()
def flip_rot(matrix_pred,matrix_out):
flip_0=np.flip(matrix_pred, 0)
 matrix_out=matrix_out.tolist()
matrix_out=flattener(matrix_out)
 if matrix_out==flattener(matrix_pred.tolist()):
print('!!!match1!!!')
 return 'y'
elif matrix_out==flattener(( np.rot90(matrix_pred)).tolist()):
print('!!!match2!!!')
 return 'np.rot90(y)'
elif matrix_out==flattener(( np.rot90(matrix_pred, 2)).tolist()):
print('!!!match3!!!')
 return 'np.rot90(y, 2)'
elif matrix_out==flattener(( np.rot90(matrix_pred, 3)).tolist()):
print('!!!match4!!!')
 return 'np.rot90(y, 3)'
elif matrix_out==flattener(flip_0.tolist()):
print('!!!match5!!!')
 return 'np.flip(y, 0)'
elif matrix_out==flattener(( np.rot90(flip_0)).tolist()):
print('!!!match6!!!')
 return 'np.rot90(np.flip(y, 0)) '
elif matrix_out==flattener(( np.rot90(flip_0, 2)).tolist()):
print('!!!match7!!!')
 return 'np.rot90(np.flip(y, 0),2)'
elif matrix_out==flattener(( np.rot90(flip_0, 3)).tolist()):
print('!!!match8!!!')
 return 'np.rot90(np.flip(y, 0),3)'
else:
return 'y'<split>
def haversine_distance(lat1, long1, lat2, long2):
R = 6371
phi1 = np.radians(lat1)
 phi2 = np.radians(lat2)
 delta_phi = np.radians(lat2-lat1)
 delta_lambda = np.radians(long2-long1)
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 return d
def pattern_1_learn(x,t_out):
def split_by_gray_line(arr):
H, W = arr.shape
Y = [-1]
for y in range(H):
if(arr[y, :]==color3 ).all() :
Y.append(y)
 Y.append(H)
 X = [-1]
for x in range(W):
if(arr[:, x]==color3 ).all() :
X.append(x)
 X.append(W)
 res = [[arr[y1+1:y2, x1+1:x2] for x1, x2 in zip(X[:-1], X[1:])] for
y1, y2 in zip(Y[:-1], Y[1:])]
return res
def change_color(arr, d):
res = arr.copy()
for k, v in d.items() :
res[arr==k] = v
return res
for color2 in range(10):
for color3 in range(10):
for color5 in range(10):
for color6 in range(10):
x_split = split_by_gray_line(x)
 if len(x_split)==1 and len(x_split[0])==2:
x1, x2 = x_split[0]
elif len(x_split)==2 and len(x_split[0])==1:
x1, x2 = x_split
x1=x1[0]
x2=x2[0]
else:
continue
if x1.shape!=x2.shape:
continue
rules=['operator.or_(x1,x2)','operator.and_(x1,x2)',
'operator.xor(x1,x2)']
for rule in rules:
zero_one = lambda t: 0 if(t == color5)else 1
vfunc = np.vectorize(zero_one)
 try:
x1=vfunc(x1)
 x2=vfunc(x2)
 except ValueError as e:
print(e)
 continue
try:
y = eval(rule)
 except ValueError as e:
continue
change_color_v2 = lambda t: color6 if(t == 0)else color2
vfunc = np.vectorize(change_color_v2)
 y=vfunc(y)
 f_r=flip_rot(y,t_out)
 y=eval(f_r)
 if flattener(t_out.tolist())==flattener(y.tolist()):
param=(color2,color3,color5,color6,rule,f_r)
 return y, param
return _, False
def pattern_1_pred(x,param):
color2,color3,color5,color6,rule,f_r = param
def split_by_gray_line(arr):
H, W = arr.shape
Y = [-1]
for y in range(H):
if(arr[y, :]==color3 ).all() :
Y.append(y)
 Y.append(H)
 X = [-1]
for x in range(W):
if(arr[:, x]==color3 ).all() :
X.append(x)
 X.append(W)
 res = [[arr[y1+1:y2, x1+1:x2] for x1, x2 in zip(X[:-1], X[1:])] for
y1, y2 in zip(Y[:-1], Y[1:])]
return res
def change_color(arr, d):
res = arr.copy()
for k, v in d.items() :
res[arr==k] = v
return res
x_split = split_by_gray_line(x)
 if len(x_split)==1 and len(x_split[0])==2:
x1, x2 = x_split[0]
elif len(x_split)==2 and len(x_split[0])==1:
x1, x2 = x_split
x1=x1[0]
x2=x2[0]
else:
print('errrrrr')
 zero_one = lambda t: 0 if(t == color5)else 1
vfunc = np.vectorize(zero_one)
 x1=vfunc(x1)
 x2=vfunc(x2)
 try:
y = eval(rule)
 except ValueError as e:
print(e)
 change_color_v2 = lambda t: color6 if(t == 0)else color2
vfunc = np.vectorize(change_color_v2)
 y=vfunc(y)
 y=eval(f_r)
 return y<train_model>
df_train = df_train.reset_index()
df_test=df_test.reset_index()
mode='train'
for iteration in range(400):
task = get_data(str(training_path / training_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',training_tasks[iteration] )<train_model>
for i in range(len(df_test)) :
df_test.loc[[i], 'distance'] = haversine_distance(df_test.loc[i]['pickup_latitude'], df_test.loc[i]['pickup_longitude'], df_test.loc[i]['dropoff_latitude'], df_test.loc[i]['dropoff_longitude'] )
mode='eval'
for iteration in range(400):
task = get_data(str(evaluation_path / evaluation_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',evaluation_tasks[iteration] )<categorify>
for i in range(len(df_train)) :
df_train.loc[[i], 'distance'] = haversine_distance(df_train.loc[i]['pickup_latitude'], df_train.loc[i]['pickup_longitude'], df_train.loc[i]['dropoff_latitude'], df_train.loc[i]['dropoff_longitude'] )
mode='test'
for iteration in range(100):
task = get_data(str(test_path / test_tasks[iteration]))
acc = check(task, pattern_1_learn, pattern_1_pred)
 print(f'Task({iteration}):',test_tasks[iteration] )<save_to_csv>
df_train.query('distance == 0' ).count()
pd.options.display.max_rows = 999
display(sample_sub)
 len(sample_sub)
 sample_sub.to_csv('submission.csv' )<import_modules>
df_train['distance'] = pd.to_numeric(df_train['distance'])
 df_train = df_train.query('distance > 0.0')
 df_test['distance'] = pd.to_numeric(df_test['distance'] )
import torch
from torch.utils.data import Dataset
import numpy as np
import json
from glob import glob
import os
import random
import pickle
import torch.nn.functional as F
import matplotlib.pyplot as plt
from matplotlib import colors
import pandas as pd
import torch.nn as nn
import math<set_options>
df_train=df_train.assign(fare_per_mile = df_train.fare_amount/df_train.distance )
seed = 42
print(f'setting everything to seed {seed}')
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
cmap = colors.ListedColormap(
 ['
'
norm = colors.Normalize(vmin=0, vmax=10 )<define_variables>
df_train.query('(fare_per_mile > 13)or(distance < 0.1)' ).count()
class ARCTest(Dataset):
def __init__(self, root, imgsz=30):

 super(ARCTest, self ).__init__()
self.out_rows, self.out_cols = imgsz, imgsz
task_paths = sorted(glob(f'{root}/test/*.json'))
self.support_x_batch, self.support_y_batch, self.query_x_batch, self.task_paths = self.create_batch(task_paths)
 def pad_im(self, task, out_rows, out_cols, cval=10):
ip = []
op = []
for mode in ['train']:
num_pairs = len(task[mode])
 input_im = np.zeros(( num_pairs, 1, out_rows, out_cols))
output_im = np.zeros(
(num_pairs, 1, out_rows, out_cols), dtype=np.long)
 for task_num in range(num_pairs):
im = np.array(task[mode][task_num]['input'])
 nrows, ncols = im.shape
if(nrows > out_rows)or(ncols > out_cols):
return 0, 0, 1, 0
im = np.pad(im,(( out_rows-nrows, 0),(out_cols-ncols, 0)) , mode='constant',
constant_values=(cval, cval))
input_im[task_num, 0] = im
im = np.array(task[mode][task_num]['output'])
 nrows, ncols = im.shape
if(nrows > out_rows)or(ncols > out_cols):
return 0, 0, 1, 0
im = np.pad(im,(( out_rows-nrows, 0),(out_cols-ncols, 0)) , mode='constant',
constant_values=(cval, cval))
output_im[task_num, 0] = im
ip.extend(input_im)
 op.extend(output_im)
 test_ip = []
num_pairs = len(task['test'])
 input_im = np.zeros(( num_pairs, 1, out_rows, out_cols))
for task_num in range(num_pairs):
im = np.array(task['test'][task_num]['input'])
 nrows, ncols = im.shape
if(nrows > out_rows)or(ncols > out_cols):
return 0, 0, 1, 0
im = np.pad(im,(( out_rows-nrows, 0),(out_cols-ncols, 0)) , mode='constant',
constant_values=(cval, cval))
input_im[task_num, 0] = im
test_ip.extend(input_im)
 return np.vstack(ip), np.vstack(op), 0, np.vstack(test_ip)
 def create_batch(self, task_paths):

 x_batch, y_batch,query_x_batch = [], [], []
all_task_paths = []
for task_file in task_paths:
with open(task_file, 'r')as f:
task = json.load(f)
 input_im, output_im, not_valid, query_im = self.pad_im(task, self.out_rows,
self.out_cols)
 if not_valid:
continue
x_batch.extend(input_im[None])
 y_batch.extend(output_im[None])
 query_x_batch.extend(query_im[None])
 all_task_paths.append(task_file)
 return x_batch, y_batch, query_x_batch, all_task_paths
def __getitem__(self, index):

 support_x = torch.tensor(
 self.support_x_batch[index], dtype=torch.float32)
 support_y = torch.tensor(
 self.support_y_batch[index], dtype=torch.long)
 return support_x[:, None], support_y.reshape(-1), self.task_paths[index]
def __len__(self):
return len(self.support_x_batch)
<categorify>
data = [df_train,df_test]
for i in data:
i["pickup_datetime"] = pd.to_datetime(i["pickup_datetime"])
 i['year'] = i['pickup_datetime'].dt.year
i['mth'] = i['pickup_datetime'].dt.month
i['date'] = i['pickup_datetime'].dt.day
i['day_of_week'] = i['pickup_datetime'].dt.dayofweek
i['hour'] = i['pickup_datetime'].dt.hour
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(
 0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x)
 class TransformerModel(nn.Module):
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
super(TransformerModel, self ).__init__()
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
 encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.encoder = nn.Embedding(ntoken, ninp)
 self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
 def _generate_square_subsequent_mask(self, sz):
mask =(torch.triu(torch.ones(sz, sz)) == 1 ).transpose(0, 1)
 mask = mask.float().masked_fill(mask == 0, float(
 '-inf')).masked_fill(mask == 1, float(0.0))
return mask
def init_weights(self):
initrange = 0.1
self.encoder.weight.data.uniform_(-initrange, initrange)
 self.decoder.bias.data.zero_()
self.decoder.weight.data.uniform_(-initrange, initrange)
 def forward(self, src):
if self.src_mask is None or self.src_mask.size(0)!= len(src):
device = src.device
mask = self._generate_square_subsequent_mask(len(src)).to(device)
 self.src_mask = mask
src = self.encoder(src)* math.sqrt(self.ninp)
 src = self.pos_encoder(src)
 output = self.transformer_encoder(src, self.src_mask)
 output = self.decoder(output)
 return output 
<load_pretrained>
X = df_train.drop(columns=['fare_amount', 'pickup_datetime', 'fare_per_mile', 'key'])
 y = df_train['fare_amount'] - 2.5
gbm_param_grid = {
'colsample_bytree': [0.7],
'n_estimators': [50],
'max_depth': [5]
}
gbm = xgb.XGBRegressor()
grid_mse = GridSearchCV(estimator=gbm, param_grid=gbm_param_grid,
scoring='neg_mean_squared_error', cv=2, verbose=1)
 grid_mse.fit(X, y)
 print("Best parameters found: ", grid_mse.best_params_)
 print("Lowest RMSE found: ", np.sqrt(np.abs(grid_mse.best_score_)) )
innerstepsize = 1e-2
innerepochs = 1000
device = torch.device('cpu')
 arc_dataset = ARCTest(
 root='/kaggle/input/abstraction-and-reasoning-challenge/', imgsz=15)
 all_train_acc = []
imgsz, num_class = 15, 11
for step,(( x, y, task_path), q)in enumerate(zip(arc_dataset, arc_dataset.query_x_batch)) :
task_id = task_path.split('/')[-1]
state = torch.load('.. /input/arctransformermodel/epoch_1_step_264_acc_0.688.pth', map_location='cpu')
 model.load_state_dict(state)
 optimizer = torch.optim.AdamW(model.parameters() , lr=innerstepsize)
 x, y = x.to(device), y.to(device)
 x = x.to(device ).reshape(-1, imgsz*imgsz ).long()
train_losses = []
train_acc = []
model.train()
for _ in range(innerepochs):
optimizer.zero_grad()
outputs = model(x ).reshape(-1, num_class)
 loss = F.cross_entropy(outputs, y)
 loss.backward()
optimizer.step()
train_losses.append(loss.item())
 acc =(outputs.argmax(1)== y ).float().mean().item()
train_acc.append(acc)
 print('\ttraining loss:',
np.mean(train_losses), '\ttraining acc:', np.mean(train_acc))
all_train_acc.append(np.mean(train_acc))
model.eval()
with torch.no_grad() :
q = torch.tensor(
 q.reshape(-1, imgsz*imgsz)).to(device ).long()
outputs = F.softmax(model(q), dim=1)
 outputs = outputs.argmax(2 ).reshape(-1, imgsz, imgsz)
 plot_figure(x, y, q, outputs, im_num=task_id, img_sz=imgsz)
 for task_num, preds in enumerate(outputs.cpu().numpy()):
frow = np.nonzero(np.count_nonzero(preds-10, axis=1)) [0][0]
fcol = np.nonzero(np.count_nonzero(preds-10, axis=0)) [0][0]
preds = np.copy(preds[frow:, fcol:])
 preds[preds == 10] = 0
sample_sub.loc[f'{task_id[:-5]}_{task_num}',
'output'] = flattener(preds.astype(int ).tolist())
 print('
mean train acc:', np.mean(all_train_acc),
'stddev train acc:', np.std(all_train_acc))<save_to_csv>
test_key = df_test['key']
x_pred = df_test.drop(columns=['key', 'pickup_datetime'])
 prediction = grid_mse.predict(x_pred)+ 2.5
<define_variables><EOS> 
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('./taxi_fare_submission.csv',index=False)
 submission.head()