Kaggle competition: New York City Taxi Trip Duration
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<set_options>
import os
import csv
import pandas as pd
import numpy as np
from matplotlib.pyplot import *
import matplotlib.pyplot as plt
from matplotlib import animation
from matplotlib import cm
from sklearn.cluster import KMeans
from sklearn.neighbors import KNeighborsClassifier
from dateutil import parser
import io
import base64
from IPython.display import HTML
from imblearn.under_sampling import RandomUnderSampler
from subprocess import check_output
import seaborn as sns
pd.options.display.precision = 15<train_model>
import zipfile
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0] 
<prepare_x_and_y>
with zipfile.ZipFile(".. /input/nyc-taxi-trip-duration/test.zip","r")as z:
z.extractall("." )
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01',
'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend', 'iqr',
'q999','q001','ave10'])
 y_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = seg['acoustic_data'].values
y = seg['time_to_failure'].values[-1]
y_train.loc[segment, 'time_to_failure'] = y
X_train.loc[segment, 'ave'] = x.mean()
X_train.loc[segment, 'std'] = x.std()
X_train.loc[segment, 'max'] = x.max()
X_train.loc[segment, 'min'] = x.min()
X_train.loc[segment, 'q95'] = np.quantile(x,0.95)
 X_train.loc[segment, 'q99'] = np.quantile(x,0.99)
 X_train.loc[segment, 'q05'] = np.quantile(x,0.05)
 X_train.loc[segment, 'q01'] = np.quantile(x,0.01)
 X_train.loc[segment, 'abs_max'] = np.abs(x ).max()
X_train.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_train.loc[segment, 'abs_std'] = np.abs(x ).std()
X_train.loc[segment, 'trend'] = add_trend_feature(x)
 X_train.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_train.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_train.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_train.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_train.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1 )<normalization>
with zipfile.ZipFile(".. /input/nyc-taxi-trip-duration/train.zip","r")as z:
z.extractall("." )
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train )<train_on_grid>
train = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.zip')
 test = pd.read_csv('.. /input/nyc-taxi-trip-duration/test.zip' )
scorer = make_scorer(mean_absolute_error, greater_is_better=False)
 parameters = [{ 'gamma': [0.6, 0.7, 0.8],
'C': [2.35, 2.4, 2.45, 2.5],
'nu': [0.85, 0.9, 0.95]}]
reg1 = GridSearchCV(NuSVR(kernel='rbf', tol=0.01), parameters, cv = 3, scoring=scorer)
 reg1.fit(X_train_scaled, y_train.values.flatten())
 y_pred1 = reg1.predict(X_train_scaled)
 print(reg1.best_params_)
 print(reg1.best_score_ )<train_on_grid>
train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)
 train.drop(['dropoff_datetime'], axis=1, inplace=True)
 train['month'] = train.pickup_datetime.dt.month
train['week'] = train.pickup_datetime.dt.week
train['weekday'] = train.pickup_datetime.dt.weekday
train['hour'] = train.pickup_datetime.dt.hour
train['minute'] = train.pickup_datetime.dt.minute
train['minute_oftheday'] = train['hour'] * 60 + train['minute']
train.drop(['minute'], axis=1, inplace=True)
 test['month'] = test.pickup_datetime.dt.month
test['week'] = test.pickup_datetime.dt.week
test['weekday'] = test.pickup_datetime.dt.weekday
test['hour'] = test.pickup_datetime.dt.hour
test['minute'] = test.pickup_datetime.dt.minute
test['minute_oftheday'] = test['hour'] * 60 + test['minute']
test.drop(['minute'], axis=1, inplace=True)
 train.drop(['pickup_datetime'], axis=1, inplace=True )
parameters = [{ 'gamma': [0.06, 0.1, 0.08, 0.09],
'alpha': [0.005, 0.01, 0.05]}]
reg2 = GridSearchCV(KernelRidge(kernel='rbf'), parameters, cv = 3, scoring=scorer)
 reg2.fit(X_train_scaled, y_train.values.flatten())
 y_pred2 = reg2.predict(X_train_scaled)
 print(reg2.best_params_)
 print(reg2.best_score_ )<compute_test_metric>
train = pd.concat([train, pd.get_dummies(train['store_and_fwd_flag'])], axis=1)
 test = pd.concat([test, pd.get_dummies(test['store_and_fwd_flag'])], axis=1)
 train.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 train = pd.concat([train, pd.get_dummies(train['vendor_id'])], axis=1)
 test = pd.concat([test, pd.get_dummies(test['vendor_id'])], axis=1)
 train.drop(['vendor_id'], axis=1, inplace=True )
score1 = mean_absolute_error(y_train.values.flatten() , y_pred1)
 print(f'Score1: {score1:0.3f}')
 score2 = mean_absolute_error(y_train.values.flatten() , y_pred2)
 print(f'Score2: {score2:0.3f}')
 score3 = mean_absolute_error(y_train.values.flatten() , y_pred1*0.5+y_pred2*0.5)
 print(f'Score3: {score3:0.3f}' )<load_from_csv>
def ft_haversine_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
train['distance'] = ft_haversine_distance(train['pickup_latitude'].values,
train['pickup_longitude'].values,
train['dropoff_latitude'].values,
train['dropoff_longitude'].values)
 test['distance'] = ft_haversine_distance(test['pickup_latitude'].values,
test['pickup_longitude'].values,
test['dropoff_latitude'].values,
test['dropoff_longitude'].values )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id' )<create_dataframe>
def ft_degree(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
train['direction'] = ft_degree(train['pickup_latitude'].values,
train['pickup_longitude'].values,
train['dropoff_latitude'].values,
train['dropoff_longitude'].values)
 test['direction'] = ft_degree(test['pickup_latitude'].values,
test['pickup_longitude'].values,
test['dropoff_latitude'].values,
test['dropoff_longitude'].values )
X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index )<feature_engineering>
train = train[(train.distance < 200)]
for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
X_test.loc[seg_id, 'trend'] = add_trend_feature(x)
 X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)
 X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)
 X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1 )<save_to_csv>
train['speed'] = train.distance / train.trip_duration
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = reg1.predict(X_test_scaled)*0.5 + reg2.predict(X_test_scaled)*0.5
submission.to_csv('submission.csv' )<set_options>
train = train[(train.speed < 30)]
train.drop(['speed'], axis=1, inplace=True )
%%html
<h1><b>LIVE EARTHQUACK MAP</b></h1>
<iframe width="800" height="600" src="https://ds.iris.edu/seismon/" allowfullscreen style="align:center;"></iframe><set_options>
feature_cols = ['passenger_count','pickup_longitude','pickup_latitude',
'dropoff_longitude','dropoff_latitude',
'N','Y','month','week','weekday','hour',
'minute_oftheday','distance','direction']
%%html
<h1> <strong><span style="color:blue;">Can We Predict Earthquakes?</span></strong></h1>
<iframe width="800" height="400" src="https://www.youtube.com/embed/uUEzGcRJIZE" style="align:center;" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> 
<set_options>
x_train = train[feature_cols]
y_train = np.log1p(train['trip_duration'])
 x_test = test[feature_cols]
%matplotlib inline
plt.style.use("fivethirtyeight")
<load_from_csv>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=1)
 Xcv,Xv,Zcv,Zv = train_test_split(x_valid, y_valid, test_size=0.4, random_state=1)
 data_tr = xgb.DMatrix(x_train, label=y_train)
 data_cv = xgb.DMatrix(Xcv , label=Zcv)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
 gc.collect()<compute_test_metric>
print('score = %1.5f, n_boost_round =%d.'%(clf.best_score,clf.best_iteration))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean() )<set_options>
data_test = xgb.DMatrix(x_test)
 ztest = clf.predict(data_test )
gc.collect()<prepare_x_and_y>
ytest = np.exp(ztest)-1
print(ytest[:10] )
<normalization><EOS> 
submission = pd.DataFrame({'id': test.id, 'trip_duration': ytest})
 submission.to_csv('submission.csv', index=False )