Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<predict_on_test>
warnings.filterwarnings('ignore')
 client = Client()
% matplotlib inline
print(os.listdir(".. /input"))
data_train_file_path = ".. /input/train.csv"
data_test_file_path = ".. /input/test.csv"
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
%%time
db_rows_count = 55423857
with open(data_train_file_path)as f:
db_rows_count = len(f.readlines())
 print("no.of rows in the training data : {0}
".format(db_rows_count))
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
train_types = {'fare_amount' : 'float32',
'pickup_datetime' : 'str',
'pickup_longitude' : 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32'}
test_types = { 'pickup_datetime' : 'str',
'key' : 'str',
'pickup_longitude' : 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32'}
train_cols = list(train_types.keys())
 test_cols = list(test_types.keys())
 BB =(-74.5, -72.8, 40.5, 41.8)
 frac = 0.00050
def select_within_boundingbox(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3])
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<load_from_csv>
def load_data(data_file_path, train_data='Y'):
if(train_data=='Y'):
df = dd.read_csv(data_file_path,usecols=train_cols, dtype=train_types)
 column_names = ["fare_amount",
"pickup_longitude",
"pickup_latitude",
"dropoff_longitude",
"dropoff_latitude"]
df = df.sample(frac=0.04)
 df = df.dropna(how="any", subset = train_cols)
 df = df[(df[column_names] != 0 ).all(axis=1)]
df = df.loc[(df.fare_amount > 0)&(df.fare_amount < 100)&
~(((df.pickup_longitude - df.dropoff_longitude)== 0)& 
(( df.pickup_latitude - df.dropoff_latitude)== 0)) ]
df = df[select_within_boundingbox(df, BB)]
if(train_data == 'N'):
df = dd.read_csv(data_file_path,usecols=test_cols, dtype=test_types)
 df = dd.concat([
df,dd.to_datetime(df['pickup_datetime'] ).apply(
 lambda x: pd.Series([x.year, x.month, x.day, x.weekday() , x.hour, x.minute],
index=['pickup_year', 'pickup_month', 'pickup_dd' ,'pickup_weekday', 'pickup_hour', 'pickup_minute'])) ], axis=1)
 df = client.persist(df)
 return df
train_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 question_p="identify the positive sentence? "
question_n="identify the negative sentence?"
question_neu="identify the neutral sentence? "<feature_engineering>
df = load_data(data_train_file_path, 'Y')
 progress(df )
def make_question(x):
if x=="positive":
return question_p
elif x=="negative":
return question_n
else:
return question_neu
train_df["sentiment"]=train_df["sentiment"].apply(make_question)
 train_df["sentiment"]<correct_missing_values>
after = len(df)
 print('
\t actual : {0}
\t after applying filters : {1}
\t dropped rows: {2} '.format(db_rows_count, after, db_rows_count-after))
train_df=train_df.dropna()<feature_engineering>
def distance_haversine(lon1, lat1, lon2, lat2):
p = 0.017453292519943295
a = 0.5 - cos(( lat2 - lat1)* p)/2 + cos(lat1 * p)* cos(lat2 * p)*(1 - cos(( lon2 - lon1)* p)) / 2
return 12742 * asin(sqrt(a)) * 0.62137
def answer_index(x):
return x["text"].find(str(x["selected_text"]))
def answer_end(x):
return x["text"].find(str(x["selected_text"])) +len(str(x["selected_text"])) -1
train_df["answer_index"]=train_df.apply(answer_index,axis=1)
 train_df["answer_end"]=train_df.apply(answer_end,axis=1 )<feature_engineering>
def distance_rows(p_lon, p_lat, d_lon, d_lat):
nyc_coord =(40.7141667, -74.0063889,)
 jfk_coord =(40.639722, -73.778889)
 ewr_coord =(40.6925, -74.168611)
 lga_coord =(40.77725, -73.872611)
 distance_between_pickup_dropoff = distance_haversine(p_lon, p_lat, d_lon, d_lat)
 distance_between_pickup_jfk = distance_haversine(p_lon, p_lat, jfk_coord[1], jfk_coord[0])
 distance_between_dropoff_jfk = distance_haversine(jfk_coord[1], jfk_coord[0], d_lon, d_lat)
 distance_between_pickup_ewr = distance_haversine(p_lon, p_lat, ewr_coord[1], ewr_coord[0])
 distance_between_dropoff_ewr = distance_haversine(ewr_coord[1], ewr_coord[0], d_lon, d_lat)
 distance_between_pickup_lga = distance_haversine(p_lon, p_lat, lga_coord[1], lga_coord[0])
 distance_between_dropoff_lga = distance_haversine(lga_coord[1], lga_coord[0], d_lon, d_lat)
 distance_between_citycenter_pickup = distance_haversine(nyc_coord[0], nyc_coord[1],p_lon, p_lat)
 longitude_diff = p_lon - d_lon
latitude_diff = p_lat - d_lat
return [distance_between_pickup_dropoff,
distance_between_pickup_jfk,
distance_between_dropoff_jfk,
distance_between_pickup_ewr,
distance_between_dropoff_ewr,
distance_between_pickup_lga,
distance_between_dropoff_lga,
distance_between_citycenter_pickup,
longitude_diff,
latitude_diff]
def calculate_coordinates_distance(df, train_data='Y'):
column_names = ['distance_between_pickup_dropoff',
'distance_between_pickup_jfk',
'distance_between_dropoff_jfk',
'distance_between_pickup_ewr',
'distance_between_dropoff_ewr',
'distance_between_pickup_lga',
'distance_between_dropoff_lga',
'distance_between_citycenter_pickup',
'longitude_diff',
'latitude_diff']
df = dd.concat([df,df[["pickup_longitude","pickup_latitude", "dropoff_longitude","dropoff_latitude"]].apply(lambda x: pd.Series(distance_rows(*x),index=column_names), axis=1)], axis=1)
 if(train_data == 'Y'):
df = df.loc[df.distance_between_pickup_dropoff>0.05]
df['fare_per_mile'] = df.fare_amount/df.distance_between_pickup_dropoff
df = df.reset_index(drop=True)
 df = client.persist(df)
 return df
train_data=[]
def make_train_data(df):
for ind in df.index:
main_d=dict()
context=df.loc[ind,"text"].lower()
main_d["context"]=context
qas=[]
inside_qas={}
inside_qas["id"]=df.loc[ind,"textID"]
inside_qas["question"]=df.loc[ind,"sentiment"]
inside_answer=[{"text":df.loc[ind,"selected_text"].lower() ,"answer_start": int(df.loc[ind,"answer_index"]),"answer_end":int(df.loc[ind,"answer_end"])}]
inside_qas["answers"]=inside_answer
inside_qas["is_impossible"]=False
qas.append(inside_qas)
 main_d["qas"]=qas
train_data.append(main_d)
 return train_data 
<create_dataframe>
df = calculate_coordinates_distance(df, train_data='Y')
 progress(df )
train_data=make_train_data(train_df )<load_from_disk>
@dask.delayed
def round_decimals(x, x_decimals=2):
return x.round(x_decimals)
 @dask.delayed
def math_sqrt(x):
return math.sqrt(x )
with open('/kaggle/working/train.json', 'w')as outfile:
json.dump(train_data, outfile)
<load_pretrained>
fare_amount_mean = df["fare_amount"].mean()
fare_amount_standard_deviation = math_sqrt(((df["fare_amount"] - fare_amount_mean)** 2 ).mean())
 print("average fair amount(mean): ${0:.2f}".format(fare_amount_mean.compute()))
print("fare amount standard deviation : ${0:.2f}
".format(fare_amount_standard_deviation.compute()))
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model=QuestionAnsweringModel("distilbert",MODEL_PATH,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 2,
'max_seq_length': 128,
'doc_stride': 64,
'fp16': False,
},
use_cuda=True)
<train_model>
df_day = df.loc[(df.pickup_hour >=6)&(df.pickup_hour <16)]
df_peak_hours = df.loc[(( df.pickup_hour >=16)&(df.pickup_hour <20)) ]
df_night = df.loc[~(( df.pickup_hour >=6)&(df.pickup_hour <20)) ]
df_between_airports = df.loc[(((df.distance_between_pickup_jfk < 2)|(df.distance_between_pickup_ewr < 2)|(df.distance_between_pickup_lga < 2)) & 
(( df.distance_between_dropoff_jfk < 2)|(df.distance_between_dropoff_lga < 2)|(df.distance_between_dropoff_ewr < 2)))]
df_airport_pickup = df.loc[(( df.distance_between_pickup_jfk < 2)|(df.distance_between_pickup_ewr < 2)|(df.distance_between_pickup_lga < 2)) ]
df_airport_dropoff = df.loc[(( df.distance_between_dropoff_jfk < 2)|(df.distance_between_dropoff_lga < 2)|(df.distance_between_dropoff_ewr < 2)) ]
BB_manhattan =(-74.025, 40.7, -73.925, 40.8)
 df_jfk_manhattan = df[(select_within_boundingbox(df, BB_manhattan)& 
(( df.distance_between_pickup_jfk < 2)|(df.distance_between_dropoff_jfk < 2)))]
df = df.reset_index(drop=True )
model.train_model('/kaggle/working/train.json' )<feature_engineering>
%%time
fare_amount_per_mile = df.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_day = df_day.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_peak_hours = df_peak_hours.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_night = df_night.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_between_airports = df_between_airports.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_airport_pickup = df_airport_pickup.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_airport_dropoff = df_airport_dropoff.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_jfk_manhattan = df_jfk_manhattan.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_weekday = df.loc[df.pickup_weekday<=4].fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_weekend = df.loc[df.pickup_weekday>=5].fare_per_mile.mean().compute().round(2)
 avg_data = pd.DataFrame({'fare':[
fare_amount_per_mile_between_airports,
fare_amount_per_mile_jfk_manhattan,
fare_amount_per_mile_airport_pickup,
fare_amount_per_mile_airport_dropoff,
fare_amount_per_mile_weekend,
fare_amount_per_mile_weekday,
fare_amount_per_mile_peak_hours,
fare_amount_per_mile_night,
fare_amount_per_mile_day,
fare_amount_per_mile
]}, index = [
'between_airports',
'jfk_manhattan',
'airport_pickup',
'airport_dropoff',
'week end',
'week day',
'peak hours(4-8pm)',
'night_ride',
'day_ride',
'all_day']
).dropna()
test_data=[]
def make_test_data(df):
for ind in df.index:
main_d=dict()
context=df.loc[ind,"text"].lower()
main_d["context"]=context
qas=[]
inside_qas={}
inside_qas["id"]=df.loc[ind,"textID"]
inside_qas["question"]=df.loc[ind,"sentiment"]
inside_qas["is_impossible"]=False
qas.append(inside_qas)
 main_d["qas"]=qas
test_data.append(main_d)
 return test_data 
<load_from_csv>
fare_per_mile_yr = df.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_day_yr = df_day.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_peak_hours_yr = df_peak_hours.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_night_yr = df_night.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_weekday_yr = df.loc[df.pickup_weekday<=4].groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_weekend_yr = df.loc[df.pickup_weekday>=5].groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_airport_pickup_yr = df_airport_pickup.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_airport_dropoff_yr = df_airport_dropoff.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_between_airports_yr = df_between_airports.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_jfk_manhattan_yr = df_jfk_manhattan.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_mile = pd.concat([
pd.DataFrame({'year':fare_per_mile_yr.index, 'avg_fare':fare_per_mile_yr.values, 'type':'overall'}),
pd.DataFrame({'year':fare_per_mile_day_yr.index, 'avg_fare':fare_per_mile_day_yr.values, 'type':'day_time'}),
pd.DataFrame({'year':fare_per_mile_peak_hours_yr.index, 'avg_fare':fare_per_mile_peak_hours_yr.values, 'type':'peak_hours'}),
pd.DataFrame({'year':fare_per_mile_night_yr.index, 'avg_fare':fare_per_mile_night_yr.values, 'type':'night_time'}),
pd.DataFrame({'year':fare_per_mile_weekday_yr.index, 'avg_fare':fare_per_mile_weekday_yr.values, 'type':'weekend'}),
pd.DataFrame({'year':fare_per_mile_weekend_yr.index, 'avg_fare':fare_per_mile_weekend_yr.values, 'type':'weekday'}),
pd.DataFrame({'year':fare_per_mile_airport_pickup_yr.index, 'avg_fare':fare_per_mile_airport_pickup_yr.values, 'type':'airport_pickup'}),
pd.DataFrame({'year':fare_per_mile_airport_dropoff_yr.index, 'avg_fare':fare_per_mile_airport_dropoff_yr.values, 'type':'airport_dropoff'}),
pd.DataFrame({'year':fare_per_mile_jfk_manhattan_yr.index, 'avg_fare':fare_per_mile_jfk_manhattan_yr.values, 'type':'jfk_manhattan'}),
pd.DataFrame({'year':fare_per_mile_between_airports_yr.index, 'avg_fare':fare_per_mile_between_airports_yr.values, 'type':'between_airports'}),
] ).reset_index(drop = True)
test_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 test_df["sentiment"]=test_df["sentiment"].apply(make_question)
 test_data=make_test_data(test_df)
 test_data_json=json.dumps(test_data)
 with open('/kaggle/working/test.json', 'w')as outfile:
json.dump(test_data, outfile )<predict_on_test>
%%time
X = df.drop(['fare_amount', 'fare_per_mile', 'pickup_datetime'], axis=1)
 y = df.fare_amount
X_train, X_test = X.random_split([0.7, 0.3], random_state=0)
 y_train, y_test = y.random_split([0.7, 0.3], random_state=0)
predictions=model.predict(test_data )<save_to_csv>
def dxgb_evaluate() :
params = {'eval_metric' : 'rmse'
,'num_boost_round' : 100
,'max_depth' : 7
,'seed' : 0
,'subsample' : 0.8
,'silent' : True
,'gamma' : 1
,'colsample_bytree' : 0.9
,'nfold' : 3
,'boosting_type' : 'gbdt'
, 'seed' : 0
}
bst = dxgb.train(client, params, X_train, y_train)
 del(params)
 return bst
predictions_df = pd.DataFrame.from_dict(predictions)
 sub_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv")
 sub_df['selected_text'] = predictions_df['answer']
sub_df.to_csv('submission.csv', index=False )<import_modules>
bst = dxgb_evaluate()
import os
import pandas as pd
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPool2D, BatchNormalization, Activation
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import plot_model
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools 
<define_variables>
X_train_predictions = dxgb.predict(client, bst, X_train)
 X_test_predictions = dxgb.predict(client, bst, X_test )
path_main = '/kaggle/input/Kannada-MNIST'
path_train = os.path.join(path_main, 'train.csv')
 path_test = os.path.join(path_main, 'test.csv')
 path_sample_sub = os.path.join(path_main, 'sample_submission.csv')
 path_dig_mnist = os.path.join(path_main, 'Dig-MNIST.csv' )<load_from_csv>
print("\033[1;37;40m\033[2;37:40mdata category \t\t\trmse-score\033[0m")
 print('train test split \t\t\033[0;37;41m {0:.2f} \033[0m'.format(np.sqrt(mean_squared_error(y_test, X_test_predictions))))
print('train split \t\t\t\033[0;37;41m {0:.2f} \033[0m
'.format(np.sqrt(mean_squared_error(y_train, X_train_predictions))))
train_dane = pd.read_csv(path_train)
 test_dane = pd.read_csv(path_test)
 sample_dane = pd.read_csv(path_sample_sub)
 dig_mnist_dane = pd.read_csv(path_dig_mnist )<data_type_conversions>
%%time
df_test = load_data(data_test_file_path, train_data='N')
 df_test = calculate_coordinates_distance(df_test, train_data='N')
 df_test_key = df_test.key
df_test = df_test.drop(['key', 'pickup_datetime'], axis=1 )
X_train = train_dane.iloc[:,1:].values.astype('float32')
 y_train = train_dane.iloc[:,0].values.astype('int32')
 X_test = test_dane.iloc[:,1:].values.astype('float32')
 Dig_mnist_im = dig_mnist_dane.iloc[:,1:].values.astype('float32')
 X_train = X_train.reshape(X_train.shape[0], 28, 28)
 X_test = X_test.reshape(X_test.shape[0], 28, 28)
 Dig_mnist_im = Dig_mnist_im.reshape(Dig_mnist_im.shape[0], 28, 28 )<data_type_conversions>
test_predictions = dxgb.predict(client, bst, df_test )
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28,1)
 mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
y_train= to_categorical(y_train)
 num_classes = y_train.shape 
<split>
submission_predictions = pd.DataFrame({'key': df_test_key.compute() , 'fare_amount': test_predictions.compute() })
 submission_predictions.to_csv('submission.csv', index=False )