Kaggle competition: New York City Taxi Fare Prediction
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 test.head()<drop_column>
train_df=pd.read_csv(".. /input/train.csv",nrows=1000000)
 test_df=pd.read_csv(".. /input/test.csv" )
train=train.drop('PassengerId',axis=1)
 PassengerId=test['PassengerId']
test=test.drop('PassengerId',axis=1)
 Survived=train['Survived'] 
<feature_engineering>
train_df.isnull().sum()
o=[]
c=['SibSp','Age','Parch','Fare']
for f in c:
q1=np.percentile(train[f],25)
 q3=np.percentile(train[f],75)
 iq=q3-q1
iqs=1.5*iq
oi=train[(train[f]<q1-iqs)|(train[f]>q3+iqs)].index
o.extend(oi)
 o=Counter(o)
 mo=list(k for k,v in o.items() if v>2)
 train=train.drop(mo,axis=0)
 train.shape 
<groupby>
train_df= train_df.drop(train_df[train_df.isnull().any(1)].index, axis = 0 )
print(train[['Pclass','Survived']].groupby(['Pclass'],as_index=False ).mean() )<concatenate>
Counter(train_df['fare_amount']<0 )
tlen=len(train)
 data=pd.concat(objs=[train,test],axis=0 ).reset_index(drop=True)
 data['size']=data['SibSp']+data['Parch']+1 
<feature_engineering>
train_df= train_df.drop(train_df[train_df['fare_amount']<0].index, axis = 0)
 train_df.shape
data['alone']=0
data.loc[data['size']==1,'alone']=1
data['sf']=0
data.loc[data['size']==2,'sf']=1
data['mf']=data['size'].apply(lambda s: 1 if 3<= s <= 4 else 0)
 data['bf']=data['size'].apply(lambda s: 1 if s>4 else 0)
<data_type_conversions>
Counter(train_df['passenger_count']>6 )
data['Sex']=data['Sex'].map({'male':1,'female':0} ).astype(int)
<count_missing_values>
train_df= train_df.drop(train_df[train_df['passenger_count']>6].index, axis = 0)
 train_df.shape
data.isnull().sum() 
<feature_engineering>
Counter(train_df['pickup_latitude']<-90 )
data['Embarked']=data['Embarked'].fillna('S')
<feature_engineering>
Counter(train_df['pickup_latitude']>90 )
l=data[data['Age'].isnull() ].index
for i in l:
am=data['Age'].median()
ap= data["Age"][(( data['SibSp'] == data.iloc[i]["SibSp"])&(data['Parch'] == data.iloc[i]["Parch"])&(data['Pclass'] == data.iloc[i]["Pclass"])) ].median()
if not np.isnan(ap):
data['Age'].iloc[i] = ap
else :
data['Age'].iloc[i] = am<feature_engineering>
train_df = train_df.drop(((train_df[train_df['pickup_latitude']<-90])|(train_df[train_df['pickup_latitude']>90])).index, axis=0 )
data['title'] = data['title'].replace(['Lady', 'Countess','Capt', 'Col','Don'\
, 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data['title'] = data['title'].replace('Mlle', 'Miss')
 data['title'] = data['title'].replace('Ms', 'Miss')
 data['title'] = data['title'].replace('Mme', 'Mrs')
<categorify>
Counter(train_df['pickup_longitude']<-180 )
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
data['title'] = data['title'].map(title_mapping)
<categorify>
Counter(train_df['pickup_longitude']>180 )
data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
<feature_engineering>
train_df = train_df.drop(( train_df[train_df['pickup_longitude']<-180] ).index, axis=0 )
Ticket = []
for i in list(data.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket.append("X")
 data["Ticket"] = Ticket
data["Ticket"].head() 
<categorify>
train_df['key']=pd.to_datetime(train_df['key'])
 train_df['pickup_datetime']=pd.to_datetime(train_df['pickup_datetime'] )
data["Pclass"] = data["Pclass"].astype("category")
 data = pd.get_dummies(data, columns = ["Pclass",'Ticket','Embarked','Cabin'] )<drop_column>
test_df['key']=pd.to_datetime(test_df['key'])
 test_df['pickup_datetime']=pd.to_datetime(test_df['pickup_datetime'] )
data=data.drop(['Name'],axis=1 )<split>
data=[train_df,test_df]
for i in data:
i['date']=i['pickup_datetime'].dt.day
i['month']=i['pickup_datetime'].dt.month
i['day_of_week']=i['pickup_datetime'].dt.dayofweek
i['hour']=i['pickup_datetime'].dt.hour
i['year']=i['pickup_datetime'].dt.year
train=data[:tlen]
test=data[tlen:] 
<prepare_x_and_y>
def sphere_distance(lat1,long1,lat2,long2):
data=[train_df,test_df]
for i in data:
R=6367
phi1 = np.radians(i[lat1])
 phi2 = np.radians(i[lat2])
 delta_phi = np.radians(i[lat2]-i[lat1])
 delta_lambda = np.radians(i[long2]-i[long1])
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 i['S_Distance'] = d
return d
xtrain=train.drop(['Survived'],axis=1 ).values
xtest=test.drop(['Survived'],axis=1 ).values
ytrain=train['Survived'].values<import_modules>
train_df.sort_values(['S_Distance','fare_amount'], ascending=False )
sv=SVC()<choose_model_class>
dis_0 = train_df.loc[(train_df['S_Distance'] == 0), ['S_Distance']]
dis_1 = train_df.loc[(train_df['S_Distance'] > 0)&(train_df['S_Distance'] <= 10), ['S_Distance']]
dis_2 = train_df.loc[(train_df['S_Distance'] > 10)&(train_df['S_Distance'] <= 50), ['S_Distance']]
dis_3 = train_df.loc[(train_df['S_Distance'] > 50)&(train_df['S_Distance'] <= 100), ['S_Distance']]
dis_4 = train_df.loc[(train_df['S_Distance'] > 100)&(train_df['S_Distance'] <= 200), ['S_Distance']]
dis_5 = train_df.loc[(train_df['S_Distance'] > 200)&(train_df['S_Distance'] <= 300), ['S_Distance']]
dis_6 = train_df.loc[(train_df['S_Distance'] > 300)&(train_df['S_Distance'] <= 500), ['S_Distance']]
dis_7 = train_df.loc[(train_df['S_Distance'] > 500), ['S_Distance']]
dis_0['bins']='0'
dis_1['bins']='0-10'
dis_2['bins']='11-50'
dis_3['bins']='51-100'
dis_4['bins']='101-200'
dis_5['bins']='201-300'
dis_6['bins']='301-500'
dis_7['bins']='>500'
dis_bin=pd.concat([dis_0,dis_1,dis_2,dis_3,dis_4,dis_5,dis_6,dis_7])
 dis_bin
kf=KFold(10,True,0 )<compute_train_metric>
x=Counter(dis_bin['bins'])
 x
cross_val_score(sv, xtrain, y = ytrain, scoring = "accuracy", cv = kf, n_jobs=4 ).mean()<train_model>
train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)]
scores=[]
kf=KFold(10,True,0)
 for(train_index,test_index)in kf.split(xtrain):
X_train, X_test, y_train, y_test = xtrain[train_index], xtrain[test_index], ytrain[train_index], ytrain[test_index]
sv.fit(X_train, y_train)
 scores.append(sv.score(X_test, y_test))
print(np.mean(scores)) 
<import_modules>
train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)]
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier,ExtraTreesClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
import xgboost as xgb<choose_model_class>
train_df = train_df.drop(train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)].index, axis=0 )
classifiers = [
SVC() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
LogisticRegression() ,
LinearDiscriminantAnalysis() ,
xgb.XGBClassifier() ]<compute_train_metric>
train_df = train_df.drop(train_df.loc[(( train_df['pickup_latitude']==0)&(train_df['pickup_longitude']==0)) &(( train_df['dropoff_latitude']!=0)&(train_df['dropoff_longitude']!=0)) &(train_df['fare_amount']==0)].index, axis=0)
for cf in classifiers:
print(cross_val_score(cf,xtrain,ytrain,cv=10 ).mean())
<define_variables>
high_distance = train_df.loc[(train_df['S_Distance']>200)&(train_df['fare_amount']!=0)]
n=xtrain.shape[0] 
<find_best_model_class>
high_distance['S_Distance'] = high_distance.apply(
 lambda row:(row['fare_amount'] - 2.50)/1.56,
axis=1
)
kf=KFold(10,True,0)
 sttrain=np.zeros(( n,1))
stest=np.zeros(( xtest.shape[0],1))
for cf in classifiers:
otr=np.zeros(( n,))
oof_test = np.zeros(( xtest.shape[0],))
oof_test_skf = np.empty(( 10, xtest.shape[0]))
for i,(train_index,test_index)in enumerate(kf.split(train)) :
X_train, X_test, y_train, y_test = xtrain[train_index], xtrain[test_index], ytrain[train_index], ytrain[test_index]
cf.fit(X_train,y_train)
 otr[test_index]=cf.predict(X_test)
 oof_test_skf[i, :] = cf.predict(xtest)
 oof_test[:] = oof_test_skf.mean(axis=0)
 otr=otr.reshape(-1,1)
 oof_test= oof_test.reshape(-1, 1)
 sttrain=np.append(sttrain, otr, axis=1)
 stest=np.append(stest, oof_test, axis=1)
 sttrain=sttrain[:,1:]
stest=stest[:,1:]<find_best_model_class>
train_df[train_df['S_Distance']==0]
xgc=xgb.XGBClassifier()
xgc.fit(sttrain,ytrain)
 pred=xgc.predict(stest ).astype(int )<save_to_csv>
train_df[(train_df['S_Distance']==0)&(train_df['fare_amount']==0)]
<import_modules>
train_df = train_df.drop(train_df[(train_df['S_Distance']==0)&(train_df['fare_amount']==0)].index, axis = 0 )
FileLinks('.')
<choose_model_class>
rush_hour = train_df.loc[(((train_df['hour']>=6)&(train_df['hour']<=20)) &(( train_df['day_of_week']>=1)&(train_df['day_of_week']<=5)) &(train_df['S_Distance']==0)&(train_df['fare_amount'] < 2.5)) ]
rush_hour
kfold = StratifiedKFold(n_splits=10 )<train_on_grid>
train_df=train_df.drop(rush_hour.index,axis=0 )
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kf, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(xtrain,ytrain)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_<train_on_grid>
non_rush_hour = train_df.loc[(((train_df['hour']<6)|(train_df['hour']>20)) &(( train_df['day_of_week']>=1)&(train_df['day_of_week']<=5)) &(train_df['S_Distance']==0)&(train_df['fare_amount'] < 3.0)) ]
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kf, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(xtrain,ytrain)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_<train_on_grid>
non_rush_hour = train_df.loc[(((train_df['hour']<6)|(train_df['hour']>20)) &(( train_df['day_of_week']>=1)&(train_df['day_of_week']<=5)) &(train_df['S_Distance']==0)&(train_df['fare_amount'] < 3.0)) ]
non_rush_hour
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kf, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(xtrain,ytrain)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_ 
<train_on_grid>
train_df.loc[(train_df['S_Distance']!=0)&(train_df['fare_amount']==0)]
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(xtrain,ytrain)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<train_model>
scenario_3 = train_df.loc[(train_df['S_Distance']!=0)&(train_df['fare_amount']==0)]
scenario_3
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best), 
('svc', SVMC_best),('gbc',GBC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(xtrain, ytrain )<save_to_csv>
scenario_3 = train_df.loc[(train_df['S_Distance']!=0)&(train_df['fare_amount']==0)]
test_Survived = pd.Series(votingC.predict(xtest), name="Survived" ).astype(int)
 results = pd.concat([PassengerId,test_Survived],axis=1)
 results.to_csv("ensemble_python_voting.csv",index=False )<import_modules>
scenario_3['fare_amount'] = scenario_3.apply(
 lambda row:(( row['S_Distance'] * 1.56)+ 2.50), axis=1
)
FileLinks('.' )<save_to_csv>
train_df.loc[(train_df['S_Distance']==0)&(train_df['fare_amount']!=0)]
StackingSubmission = pd.DataFrame({ 'PassengerId': PassengerId,
'Survived': pred })
 StackingSubmission.to_csv("StackingSubmission.csv", index=False)
 df=StackingSubmission<install_modules>
scenario_4 = train_df.loc[(train_df['S_Distance']==0)&(train_df['fare_amount']!=0)]
!pip install pycomp --upgrade --no-cache-dir<define_variables>
scenario_4.loc[(scenario_4['fare_amount']<=3.0)&(scenario_4['S_Distance']==0)]
filterwarnings('ignore')
 DATA_PATH = '.. /input/titanic'
TRAIN_FILENAME = 'train.csv'
TEST_FILENAME = 'test.csv'<load_from_csv>
scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['S_Distance']==0)]
df = pd.read_csv(os.path.join(DATA_PATH, TRAIN_FILENAME))
df.head()<categorify>
scenario_4_sub = scenario_4.loc[(scenario_4['fare_amount']>3.0)&(scenario_4['S_Distance']==0)]
feature_adder = CustomFeaturesTitanic(name_title=True, cabin_class=True, ticket_class=True)
 df_custom = feature_adder.fit_transform(df)
 df_custom.head()<count_duplicates>
scenario_4_sub['S_Distance'] = scenario_4_sub.apply(
 lambda row:(( row['fare_amount']-2.50)/1.56), axis=1
)
dup_dropper = EliminaDuplicatas()
df_nodup = dup_dropper.fit_transform(df_slct)
 print(f'Total of duplicates before: {df_slct.duplicated().sum() }')
 print(f'Total of duplicates after: {df_nodup.duplicated().sum() }' )<categorify>
train_df = train_df.drop(['key','pickup_datetime'], axis = 1)
 test_df = test_df.drop(['key','pickup_datetime'], axis = 1 )
cat_custom_features = ['Pclass']
mod_dict = {col: str for col in cat_custom_features}
print(f'Selected columns dtype before transformation:
')
 print(df_nodup.dtypes[cat_custom_features])
 dtype_mod = ModificaTipoPrimitivo(mod_dict=mod_dict)
 df_mod = dtype_mod.fit_transform(df_nodup)
 print(f'Selected columns dtype after transformation:
')
 print(df_mod.dtypes[cat_custom_features] )<normalization>
x_train = train_df.iloc[:,train_df.columns!='fare_amount']
y_train = train_df['fare_amount'].values
x_test = test_df
imputer = SimpleImputer(strategy='median')
 X_train_num_imp = imputer.fit_transform(X_train_num)
 X_train_num_imp = pd.DataFrame(X_train_num_imp, columns=num_features)
 print(f'Null data before imputer: {X_train_num.isnull().sum().sum() }')
 print(f'Null data after imputer: {X_train_num_imp.isnull().sum().sum() }' )<feature_engineering>
rg=RandomForestRegressor()
rg.fit(x_train,y_train)
 y_predict=rg.predict(x_test)
 y_predict
<normalization><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = y_predict
submission.to_csv('submission_1.csv', index=False)
 submission.head(10 )