Kaggle competition: New York City Taxi Fare Prediction
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<find_best_params>
datatypes = {'key': 'str',
'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
train_df = pd.read_csv('.. /input/train.csv', nrows=5000000, dtype=datatypes )
%%time
best_final_score, best_score_threshold = 0, 0
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold<compute_test_metric>
def manhattan_distance(lat1, long1, lat2, long2):
diff_lat = abs(lat1 - lat2)
 diff_long = abs(long1 - long2)
 return(diff_lat + diff_long)
 def distance_between_points(df):
df['diff_lat'] = abs(df['dropoff_latitude'] - df['pickup_latitude'])
 df['diff_long'] = abs(df['dropoff_longitude'] - df['pickup_longitude'])
 df['manhattan_dist'] = df['diff_lat'] + df['diff_long']
jfk = [40.6413, -73.7781]
lga = [40.7769, -73.8740]
ewr = [40.6895, -74.1745]
df['jfk_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], jfk[0], jfk[1])
 df['jfk_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], jfk[0], jfk[1])
 df['lga_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], lga[0], lga[1])
 df['lga_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], lga[0], lga[1])
 df['ewr_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], ewr[0], ewr[1])
 df['ewr_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], ewr[0], ewr[1])
 distance_between_points(train_df )
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30 )<define_variables>
def extract_date_details(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S UTC')
 df['year'] = df['pickup_datetime'].apply(lambda date: date.year)
 df['month'] = df['pickup_datetime'].apply(lambda date: date.month)
 df['day'] = df['pickup_datetime'].apply(lambda date: date.weekday())
 df['hour'] = df['pickup_datetime'].apply(lambda date: date.hour)
 extract_date_details(train_df)
 train_df
i = 0
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>best_score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
show_result(image_id, pred_boxes, gt_boxes )<data_type_conversions>
def remove_outliers(df):
df = df.dropna()
df = df[(df['diff_lat'] < 5.0)&(df['diff_long'] < 5.0)]
df = df[(df['diff_lat'] >.001)&(df['diff_long'] >.001)]
df = df[(df['pickup_longitude'] < -72)&(df['pickup_longitude'] > -75)]
df = df[(df['pickup_latitude'] < 42)&(df['pickup_latitude'] > 39)]
df = df[(df['dropoff_longitude'] < -72)&(df['dropoff_longitude'] > -75)]
df = df[(df['dropoff_latitude'] < 42)&(df['dropoff_latitude'] > 39)]
df = df[(df['fare_amount'] > 2.50)&(df['fare_amount'] < 200)&(df['passenger_count'] <= 6)&(df['passenger_count'] > 0)]
return df
train_df = remove_outliers(train_df)
 len(train_df )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 def make_predictions(images, score_threshold=0.1):
images = torch.stack(images ).cuda().float()
predictions = []
for net in models:
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.55, skip_box_thr=0.45, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<feature_engineering>
def convert_to_one_hot(column, num_buckets, df, starting_index = 0):
df_size = df.shape[0]
one_hots = np.zeros(( df_size, num_buckets), dtype='byte')
 one_hots[np.arange(df_size), df[column].values - starting_index] = 1
return one_hots
def season_to_numeric(x):
return int(x.split('-')[0])- 1996
data["season"] = data["season"].apply(season_to_numeric )<feature_engineering>
year = convert_to_one_hot('year', 7, train_df, 2009)
 hour = convert_to_one_hot('hour', 24, train_df, 0 )
def home_or_away(x):
if "@" in x:
return -1
elif "vs." in x:
return 1
else:
return 0
data['matchup'] = data['matchup'].apply(home_or_away )<feature_engineering>
def bucketize_feature(df,column):
buckets = train_df[column].quantile([.1,.2,.3,.4,.5,.6,.7,.8,.9] ).values
bins = np.array(df[column].values)
 lower_bound = -100000
for i in range(buckets.shape[0]):
upper_bound = buckets[i]
bins[(bins >= lower_bound)&(bins < upper_bound)] = i
lower_bound = upper_bound
bins[(bins < 0)|(bins > 8)] = 9
bins = np.array(bins, dtype='byte')
 return bins
p_long = bucketize_feature(train_df, 'pickup_longitude')
 p_lat = bucketize_feature(train_df, 'pickup_latitude')
 d_long = bucketize_feature(train_df, 'dropoff_longitude')
 d_lat = bucketize_feature(train_df, 'dropoff_latitude' )
def time_from_start(x):
if x["period"] <= 4:
period_minutes = x["period"] * 12
else:
period_minutes = 48 +(x["period"] - 4)* 5
return(period_minutes - x["minutes_remaining"])* 60 - x["seconds_remaining"]
data['time_from_start'] = data.apply(time_from_start, axis = 1 )<count_values>
def feature_cross(a1, a2):
rows = a1.shape[0]
cols = 100
cross = np.zeros(( rows, cols), dtype='byte')
 cross[np.arange(rows),(a1 * 10)+ a2] = 1
return cross
p_lat_x_long = feature_cross(p_lat, p_long)
 d_lat_x_long = feature_cross(d_lat, d_long )
def categorical_to_freq(feature_name):
my_counts = data[feature_name].value_counts()
return data[feature_name].apply(lambda x: my_counts[x])
 action_type = categorical_to_freq("action_type")
 combined_shot_type = categorical_to_freq("combined_shot_type")
 shot_type = categorical_to_freq("shot_type")
 shot_zone_area = categorical_to_freq("shot_zone_area")
 shot_zone_basic = categorical_to_freq("shot_zone_basic")
 shot_zone_range = categorical_to_freq("shot_zone_range")
<categorify>
manhattan = train_df['manhattan_dist'].values.reshape(len(train_df), 1)
 jfk_p = train_df['jfk_dist_pickup'].values.reshape(len(train_df), 1)
 jfk_d = train_df['jfk_dist_dropoff'].values.reshape(len(train_df), 1)
 lga_p = train_df['lga_dist_pickup'].values.reshape(len(train_df), 1)
 lga_d = train_df['lga_dist_dropoff'].values.reshape(len(train_df), 1)
 ewr_p = train_df['ewr_dist_pickup'].values.reshape(len(train_df), 1)
 ewr_d = train_df['ewr_dist_dropoff'].values.reshape(len(train_df), 1)
 train_X = np.concatenate(( p_lat_x_long, d_lat_x_long, year, hour, manhattan, jfk_p, jfk_d, lga_p, lga_d, ewr_p, ewr_d), axis=1)
 train_y = train_df['fare_amount'].values
print(train_X.shape)
 print(train_y.shape )
mydata = data[["matchup","season", "time_from_start", "game_event_id", "game_id", "lat", "loc_x", "loc_y", "lon", "minutes_remaining", "period", "playoffs", "seconds_remaining", "shot_distance", "shot_made_flag", "shot_id"]]
mydata = pd.concat([mydata, action_type, combined_shot_type, shot_type, shot_zone_area, shot_zone_basic, shot_zone_range], axis = 1)
 train = mydata[mydata["shot_made_flag"].notnull() ]
test = mydata[mydata["shot_made_flag"].isnull() ]
target = train["shot_made_flag"]
train = train.drop(["shot_made_flag", "shot_id"], 1)
 test_shot_id = test["shot_id"]
test = test.drop(["shot_made_flag", "shot_id"], 1)
 
<train_model>
validate_df = pd.read_csv('.. /input/train.csv', skiprows=range(1,10000001), nrows=10000, dtype=datatypes )
gbrt = GradientBoostingRegressor(n_estimators=100, learning_rate=0.05, max_depth=5, random_state=0, loss='ls' ).fit(train, target)
 gbrt_res = gbrt.predict(test )<save_to_csv>
distance_between_points(validate_df)
 validate_df = remove_outliers(validate_df)
 def extract_features(df):
extract_date_details(df)
 p_lo = bucketize_feature(df, 'pickup_longitude')
 p_la = bucketize_feature(df, 'pickup_latitude')
 d_lo = bucketize_feature(df, 'dropoff_longitude')
 d_la = bucketize_feature(df, 'dropoff_latitude')
 p_la_x_lo = feature_cross(p_la, p_lo)
 d_la_x_lo = feature_cross(d_la, d_lo)
 yr = convert_to_one_hot('year', 7, df, 2009)
 hr = convert_to_one_hot('hour', 24, df, 0)
 manhattan = df['manhattan_dist'].values.reshape(len(df), 1)
 jfk_p = df['jfk_dist_pickup'].values.reshape(len(df), 1)
 jfk_d = df['jfk_dist_dropoff'].values.reshape(len(df), 1)
 lga_p = df['lga_dist_pickup'].values.reshape(len(df), 1)
 lga_d = df['lga_dist_dropoff'].values.reshape(len(df), 1)
 ewr_p = df['ewr_dist_pickup'].values.reshape(len(df), 1)
 ewr_d = df['ewr_dist_dropoff'].values.reshape(len(df), 1)
 print(p_la_x_lo.shape)
 print(d_la_x_lo.shape)
 print(yr.shape)
 print(hr.shape)
 print(manhattan.shape)
 X = np.concatenate(( p_la_x_lo, d_la_x_lo, yr, hr, manhattan, jfk_p, jfk_d, lga_p, lga_d, ewr_p, ewr_d), axis=1)
 return X
X = extract_features(validate_df)
 true_y = validate_df['fare_amount'].values
output = pd.concat([test_shot_id.reset_index(drop = True), pd.Series(data = gbrt_res, name = 'shot_made_flag')], axis = 1)
 output.to_csv("my_third_submission.csv", index = False)
 print(check_output(["cp", ".. /input/sample_submission.csv", "sample_submission.csv"] ).decode("utf8"))<save_to_csv>
model = tf.keras.Sequential()
model.add(layers.Dense(128, activation='relu', input_dim=238))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer='adam',
loss='mse',
metrics=['mae'])
 model.fit(train_X, train_y, epochs=15, batch_size=256,validation_split=0.1 )
print(check_output(["head", "my_third_submission.csv"] ).decode("utf8"))<import_modules>
result = model.predict(X ).flatten()
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import xgboost as xgb
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LinearRegression
from sklearn.metrics import log_loss, accuracy_score
from sklearn.cross_validation import cross_val_score, train_test_split<load_from_csv>
mean_y = np.mean(train_df['fare_amount'].values)
 diff = true_y - result
mse = np.sum(diff ** 2)/ len(diff)
 rmse = np.sqrt(mse)
 print(rmse )
data = pd.read_csv('.. /input/data.csv' )<feature_engineering>
distance_between_points(test_df)
 X_test = extract_features(test_df)
 pred_y_test = model.predict(X_test ).flatten()
total_actions = dict(data.action_type.value_counts())
 threshold = 100
data['type'] = data.apply(lambda row: row['action_type'] if total_actions[row['action_type']] >= threshold \
else row['combined_shot_type'], axis=1)
 data['time_remaining'] = data.apply(lambda row: row['minutes_remaining'] * 60 + row['seconds_remaining'], axis=1)
 threshold = 3
anomaly = 14
data['last_moment'] = data.apply(lambda row: row['time_remaining'] < threshold or row['time_remaining'] == anomaly, axis=1)
 data['away'] = data.matchup.str.contains('@')
 data['season'] = data.apply(lambda row: int(row['season'].split('-')[0]), axis=1)
 data['shot_distance'] = data.apply(lambda row: 28 if row['shot_distance'] > 28 else row['shot_distance'], axis=1)
 def get_acc(data, field):
ct = pd.crosstab(data.shot_made_flag, data[field] ).apply(lambda x: x / x.sum() , axis=0)
 x, y = ct.columns, ct.values[1, :]
plt.figure(figsize=(7, 5))
plt.plot(x, y)
 plt.xlabel(field)
 plt.ylabel('% shots made')
 plt.show()
def sort_encode(data, field):
ct = pd.crosstab(data.shot_made_flag, data[field] ).apply(lambda x: x / x.sum() , axis=0)
 temp = list(zip(ct.values[1, :], ct.columns))
temp.sort()
new_map = {}
for index,(acc, old_number)in enumerate(temp):
new_map[old_number] = index
new_field = field + '_sort_enumerated'
data[new_field] = data[field].map(new_map)
 get_acc(data, new_field)
 data.drop([
'action_type',
'combined_shot_type',
'game_event_id',
'game_id',
'lat',
'lon',
'minutes_remaining',
'seconds_remaining',
'time_remaining',
'team_id',
'team_name',
'matchup',
'game_date',
'shot_type',
'playoffs',
'season',
'loc_x',
'loc_y',
], axis=1, inplace=True)
 dummies = [
'period',
'type',
'shot_zone_area',
'shot_zone_basic',
'shot_zone_range',
'opponent',
]
dummie_counter = {}
for dummy in dummies:
dummie_counter[dummy] = len(data[dummy].unique())
 data = pd.get_dummies(data, columns=dummies)
 data.head()<count_unique_values>
sample_submission = pd.read_csv('.. /input/sample_submission.csv' )
<prepare_x_and_y><EOS> 
sample_submission['fare_amount'] = pd.Series(pred_y_test)
 sample_submission.to_csv('nn_submission.csv', index=False )