Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<sort_values>
%matplotlib inline
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
DATA_PATH = ".. /input"
TRAIN_FILE_PATH = os.path.join(DATA_PATH, "train.csv")
 TEST_FILE_PATH = os.path.join(DATA_PATH, "test.csv")
 SAMPLE_SUB_FILE_PATH = os.path.join(DATA_PATH, "sample_submission.csv")
 SUB_FILE_PATH = os.path.join(DATA_PATH, "submission.csv" )
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()<save_to_csv>
train = pd.read_csv(TRAIN_FILE_PATH, index_col="id", parse_dates=["pickup_datetime", "dropoff_datetime"] )
df.to_csv('submission.csv', index=False )<set_options>
train_set, val_set = train_test_split(train, test_size=0.2, random_state=42)
 print(len(train_set), len(val_set))
train = train_set
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from collections import deque
from scipy import sparse
import implicit
from IPython.display import display
import zipfile
from pathlib import Path
import time
import warnings; warnings.simplefilter('ignore' )<load_from_csv>
class ClipOutliers(BaseEstimator, TransformerMixin):
def __init__(self, attribute_name, lower_limit=None, upper_limit=None):
self.attribute_name = attribute_name
self.lower_limit = lower_limit
self.upper_limit = upper_limit
def fit(self, df, y=None):
return self
def transform(self, df, y=None):
if self.attribute_name not in df.columns:
return df
elif self.lower_limit != None:
return df[df[self.attribute_name] > self.lower_limit]
elif self.upper_limit != None:
return df[df[self.attribute_name] < self.upper_limit]
else:
return df
clip_outliers_in_trip_duration = ClipOutliers("trip_duration", lower_limit=None, upper_limit=6000)
 train_trip_duration_clipped = clip_outliers_in_trip_duration.fit_transform(train)
 clip_outliers_in_passenger_count = ClipOutliers("passenger_count", lower_limit=0, upper_limit=None)
 train_clipped = clip_outliers_in_trip_duration.fit_transform(train_trip_duration_clipped)
 plt.subplots(figsize=(15, 6))
train_clipped.boxplot()
df_coupon_list_train = pd.read_csv('.. /input/coupon-purchase-prediction-translated/coupon_list_train_translated.csv')
 df_coupon_detail_train = pd.read_csv('.. /input/coupon-purchase-prediction-translated/coupon_detail_train_translated.csv')
 df_coupon_list_test = pd.read_csv('.. /input/coupon-purchase-prediction-translated/coupon_list_test_translated.csv')
 df_user_list = pd.read_csv('.. /input/coupon-purchase-prediction-translated/user_list_translated.csv')
<merge>
taxi = train
df_purchased_coupons_train = df_coupon_detail_train.merge(df_coupon_list_train, on='COUPON_ID_hash', how='inner' )<define_variables>
pickup_mean_latitude = taxi[(taxi["pickup_latitude"] > 40.0)&(taxi["pickup_latitude"] < 42.0)]["pickup_latitude"].mean()
pickup_mean_longitude = taxi[(taxi["pickup_longitude"] > -74.2)&(taxi["pickup_longitude"] < -73.5)]["pickup_longitude"].mean()
print("pickup_mean_latitude = {}".format(pickup_mean_latitude))
print("pickup_mean_longitude = {}".format(pickup_mean_longitude))
dropoff_mean_latitude = taxi[(taxi["dropoff_latitude"] > 40.0)&(taxi["dropoff_latitude"] < 42.0)]["pickup_latitude"].mean()
dropoff_mean_longitude = taxi[(taxi["dropoff_longitude"] > -74.2)&(taxi["dropoff_longitude"] < -73.5)]["pickup_longitude"].mean()
print("dropoff_mean_latitude = {}".format(dropoff_mean_latitude))
print("dropoff_mean_longitude = {}".format(dropoff_mean_longitude))
features = ['COUPON_ID_hash', 'USER_ID_hash', 'GENRE_NAME', 'DISCOUNT_PRICE', 'large_area_name', 'ken_name', 'small_area_name']
df_purchased_coupons_train = df_purchased_coupons_train[features]
df_purchased_coupons_train<categorify>
class DateAttribsAdder(BaseEstimator, TransformerMixin):
def __init__(self, date_field):
self.date_field = date_field
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X["month"] = X.loc[:,self.date_field].dt.month
X["week"] = X.loc[:,self.date_field].dt.week
X["weekday"] = X.loc[:,self.date_field].dt.weekday
X["day"] = X.loc[:,self.date_field].dt.day
X["hour"] = X.loc[:,self.date_field].dt.hour
X["minute"] = X.loc[:,self.date_field].dt.minute
X["minute_of_the_day"] = X["hour"] * 60 + X.loc[:,self.date_field].dt.minute
return X.drop([self.date_field, "dropoff_datetime", "minute"], axis=1, errors="ignore")
 date_attribs_adder = DateAttribsAdder("pickup_datetime")
 date_attribs_added = date_attribs_adder.transform(taxi)
 print(date_attribs_added.head())
df_coupon_list_test['USER_ID_hash'] = 'dummyuser'
df_coupon_list_test = df_coupon_list_test[features]
df_coupon_list_test<feature_engineering>
cat_attribs = ["vendor_id", "store_and_fwd_flag"]
taxi_cat = taxi[cat_attribs]
cat_encoder = OneHotEncoder()
taxi_1hot = cat_encoder.fit_transform(taxi_cat)
 print(taxi_1hot.toarray() )
df_combined = pd.concat([df_purchased_coupons_train, df_coupon_list_test], axis=0)
 df_combined['DISCOUNT_PRICE'] = 1 / np.log10(df_combined['DISCOUNT_PRICE'])
 features.extend(['DISCOUNT_PRICE'])
 df_combined<categorify>
%%time
class DistanceAttribsAdder(BaseEstimator, TransformerMixin):
def __init__(self, attrib_name = "distance"):
self.attrib_name = attrib_name
def fit(self, df):
return self
def transform(self, df, y=None):
R=6373.0
pickup_lat=np.radians(df['pickup_latitude'].values)
 pickup_lon=np.radians(df['pickup_longitude'].values)
 dropoff_lat=np.radians(df['dropoff_latitude'].values)
 dropoff_lon=np.radians(df['dropoff_longitude'].values)
 dlon = dropoff_lon - pickup_lon
dlat = dropoff_lat - pickup_lat
a = np.sin(dlat / 2)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
df["distance"] = R * c
df.boxplot(column="distance", return_type="axes");
plt.show()
return df
distance_attribs_adder = DistanceAttribsAdder()
taxi_distance = distance_attribs_adder.transform(taxi)
 print(taxi_distance.head())
 clip_outliers_in_distance = ClipOutliers("distance", lower_limit=None, upper_limit=400)
 taxi = clip_outliers_in_distance.fit_transform(taxi_distance )
categoricals = ['GENRE_NAME', 'large_area_name', 'ken_name', 'small_area_name']
df_combined_categoricals = df_combined[categoricals]
df_combined_categoricals = pd.get_dummies(df_combined_categoricals, dummy_na=False)
 df_combined_categoricals<concatenate>
class DropFeature(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X.drop([self.attribute_names],axis=1,errors="ignore" ).values
continuous = list(set(features)- set(categoricals))
df_combined = pd.concat([df_combined[continuous], df_combined_categoricals], axis=1)
 print(df_combined.isna().sum())
 NAN_SUBSTITUTION_VALUE = 1
df_combined = df_combined.fillna(NAN_SUBSTITUTION_VALUE)
 df_combined<drop_column>
class FeatureSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names].values
df_train = df_combined[df_combined['USER_ID_hash'] != 'dummyuser']
df_test = df_combined[df_combined['USER_ID_hash'] == 'dummyuser']
df_test = df_test.drop('USER_ID_hash', axis=1)
 display(df_train)
 display(df_test )<groupby>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
columns = X.columns
selected_features = [feature for feature in self.attribute_names if feature in columns ]
return X[selected_features]
df_train_dropped_coupons = df_train.drop('COUPON_ID_hash', axis=1)
 df_user_profiles = df_train_dropped_coupons.groupby('USER_ID_hash' ).mean()
df_user_profiles<define_variables>
class LogTransformer(BaseEstimator, TransformerMixin):
def __init__(self, attribute_name):
self.attribute_name = attribute_name
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X[self.attribute_name] = np.log(X[self.attribute_name].values)
 return X
FEATURE_WEIGHTS = {
'GENRE_NAME': 2,
'DISCOUNT_PRICE': 2,
'large_area_name': 0.5,
'ken_name': 1.5,
'small_area_name': 5
}<load_pretrained>
cat_pipeline = Pipeline([ 
('selector_cat', FeatureSelector(cat_attribs)) , 
("cat_trans", OneHotEncoder()),
])
def find_appropriate_weight(weights_dict, colname):
for col, weight in weights_dict.items() :
if col in colname:
return weight
raise ValueError<concatenate>
label_pipeline = Pipeline([ 
("clip_trip_duration", ClipOutliers("trip_duration", lower_limit=None, upper_limit=6000)) , 
('trip_log_transformer', LogTransformer("trip_duration")) , 
("feature_selector", FeatureSelector("trip_duration"))
] )
W_values = [find_appropriate_weight(FEATURE_WEIGHTS, colname)
 for colname in df_user_profiles.columns]
W = np.diag(W_values)
 W<prepare_x_and_y>
full_pipeline = FeatureUnion(transformer_list=[ 
("num_pipeline", num_pipeline), 
("cat_pipeline", cat_pipeline),
] )
df_test_only_features = df_test.drop('COUPON_ID_hash', axis=1)
 similarity_scores = np.dot(np.dot(df_user_profiles, W), df_test_only_features.T)
 similarity_scores<create_dataframe>
outliers_clipped = outlier_pipeline.fit_transform(train)
 taxi_prepared = full_pipeline.fit_transform(outliers_clipped)
 taxi_labels = label_pipeline.fit_transform(outliers_clipped)
 print(taxi_prepared.shape)
 taxi_prepared
s_coupons_ids = df_test['COUPON_ID_hash']
index = df_user_profiles.index
columns = pd.Series([s_coupons_ids[i] for i in range(0, similarity_scores.shape[1])], name='COUPON_ID_hash')
 df_results = pd.DataFrame(index=index, columns=columns, data=similarity_scores)
 df_results<sort_values>
outliers_clipped_val = outlier_pipeline.fit_transform(val_set)
 taxi_prepared_val = full_pipeline.fit_transform(outliers_clipped_val)
 taxi_labels_val = label_pipeline.fit_transform(outliers_clipped_val )
def get_top10_coupon_hashes_string(row):
sorted_row = row.sort_values()
return ' '.join(sorted_row.index[-10:][::-1].tolist() )<feature_engineering>
forest_reg = RandomForestRegressor(n_estimators=50,min_samples_leaf=10, min_samples_split=15, max_features='auto', max_depth=50, bootstrap=True, n_jobs=-1)
 forest_reg.fit(taxi_prepared, taxi_labels)
 predictions = forest_reg.predict(taxi_prepared_val)
 print(predictions[:5])
 print(taxi_labels_val[:5])
 score = np.sqrt(mean_squared_error(taxi_labels_val, predictions))
print("validation RMSE: {}".format(score))
output = df_results.apply(get_top10_coupon_hashes_string, axis=1)
 output<create_dataframe>
test = pd.read_csv(TEST_FILE_PATH, index_col="id", parse_dates=["pickup_datetime"] )
df_output = pd.DataFrame(data={'USER_ID_hash': output.index, 'PURCHASED_COUPONS': output.values})
 df_output<save_to_csv>
outliers_clipped_test = outlier_pipeline.fit_transform(test)
 test_prepared = full_pipeline.fit_transform(test)
 print(test_prepared.shape )
df_output_all = pd.merge(df_user_list, df_output, how='left', on='USER_ID_hash')
 df_output_all.to_csv('cosine_sim_python.csv', header=True, index=False, columns=['USER_ID_hash', 'PURCHASED_COUPONS'])
 df_output_all<install_modules>
pred_test = np.exp(forest_reg.predict(test_prepared))
!pip install turicreate<import_modules>
submit = pd.read_csv(SAMPLE_SUB_FILE_PATH)
 submit.head()
import os
import pandas as pd
import numpy as np
from zipfile import ZipFile
import turicreate<define_variables>
submit["trip_duration"] = pred_test
submit.head()
<load_pretrained><EOS> 
submit.to_csv("submission.csv", index=False )