Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<train_model>
%matplotlib inline
col = ['Pronoun-offset', 'A-offset', 'B-offset', 'section_min', 'Pronoun-offset2', 'A-offset2', 'B-offset2', 'section_max',
'A-poss', 'B-poss', 'A-dist', 'B-dist', 'A-dist_abs', 'B-dist_abs', 'A_max', 'A_min', 'B_max', 'B_min', 'wc']
x1, x2, y1, y2 = model_selection.train_test_split(train[col].fillna(-1), train[['A', 'B', 'NEITHER']], test_size=0.2, random_state=1)
 etc = multiclass.OneVsRestClassifier(ensemble.ExtraTreesClassifier(n_jobs=-1, max_depth=9, n_estimators=1000, random_state=30))
etc.fit(x1, y1)
 print('log_loss', metrics.log_loss(y2, etc.predict_proba(x2)))
 etc.fit(train[col].fillna(-1), train[['A', 'B', 'NEITHER']])
 results = etc.predict_proba(test[col])
 test['A'] = results[:,0]
test['B'] = results[:,1]
test['NEITHER'] = results[:,2]
test[['ID', 'A', 'B', 'NEITHER']].to_csv('submission.csv', index=False )<prepare_x_and_y>
INPUT_PATH = Path('.. ','input')
 TRAIN_PATH = Path(INPUT_PATH, 'train.csv')
 TEST_PATH = Path(INPUT_PATH, 'test.csv')
 train = pd.read_csv(TRAIN_PATH, index_col='id', parse_dates=['pickup_datetime', 'dropoff_datetime'])
 train.head()
data_set = pd.read_csv(".. /input/names-dataset/names_dataset.csv")
 xfeatures = data_set["name"]
cv = CountVectorizer()
X = cv.fit_transform(xfeatures)
 data_set.sex.replace({'F':0,'M':1},inplace=True)
 X = X
data_set.drop_duplicates(keep="first", inplace=True)
 y =data_set.sex
print("ty",Counter(y))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
 clf = MultinomialNB()
clf.fit(X, y)
 def Predict(data):
test_name = [data]
vector = cv.transform(test_name ).toarray()
result = clf.predict(vector)[0]
return result 
<load_from_csv>
test = pd.read_csv(TEST_PATH, index_col='id', parse_dates=['pickup_datetime'])
 test.head()
train_set = pd.read_csv(".. /input/gendered-pronoun-resolution/test_stage_1.tsv", encoding="utf-8", error_bad_lines=False, delimiter='\t')
 train_set["A"] = train_set["A"].apply(Predict)
 train_set["B"] = train_set["B"].apply(Predict)
<drop_column>
def extract_date_info(df, cols):
for col in cols:
df[col + '_month'] = df[col].dt.month
df[col + '_week'] = df[col].dt.week
df[col + '_dow'] = df[col].dt.dayofweek
df[col + '_hour'] = df[col].dt.hour
df[col + '_date'] = df[col].dt.date
return df
train_set = train_set[["ID", "A", 'B'] ]<concatenate>
train = extract_date_info(train, ['pickup_datetime', 'dropoff_datetime'])
 test = extract_date_info(test, ['pickup_datetime'])
 train.head()
train_set = train_set.assign(NEITHER=lambda r: 1-r.A-r.B )<data_type_conversions>
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train_set["NEITHER"] = abs(train_set["NEITHER"].astype(int))<save_to_csv>
train['distance'] = train.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1 )
train_set.to_csv('sub.csv', index=False )<import_modules>
from concurrent.futures import ProcessPoolExecutor<import_modules>
X = train.drop(columns=['pickup_datetime', 'dropoff_datetime', 'trip_duration',
'pickup_datetime_date', 'dropoff_datetime_month',
'dropoff_datetime_week','dropoff_datetime_dow',
'dropoff_datetime_hour', 'dropoff_datetime_date'] )
from concurrent.futures import ProcessPoolExecutor
import itertools
import functools
import time
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
import sklearn
from sklearn.linear_model import LinearRegression, Ridge, Lasso, LassoCV, ElasticNet, ElasticNetCV
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold, cross_val_score, GridSearchCV
from sklearn.metrics import roc_curve, make_scorer, mean_squared_error
from sklearn.preprocessing import RobustScaler
from mlxtend.regressor import StackingCVRegressor
import lightgbm as lgb
import xgboost as xgb<load_from_csv>
le = LabelEncoder()
le.fit(X['store_and_fwd_flag'])
 X['store_and_fwd_flag'] = le.transform(X['store_and_fwd_flag'])
 y = np.log1p(train['trip_duration'] )
%%script false --no-raise-error
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv' )<load_from_csv>
X_test = test.drop(columns=['pickup_datetime', 'pickup_datetime_date'])
 X_test['store_and_fwd_flag'] = le.transform(X_test['store_and_fwd_flag'])
 X_test['distance'] = test.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1)
 X_test.info()
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<prepare_x_and_y>
rf = RandomForestRegressor(n_estimators=100, n_jobs=-1)
train_ids = train.Id
test_ids = test.Id
train_y = train.pop('SalePrice' )<concatenate>
rf.fit(X, y)
 log_pred = rf.predict(X_test)
 pred = np.exp(log_pred)- np.ones(len(log_pred))
def allyficy(train, test):
return pd.concat([train, test] ).reset_index(drop=True )<data_type_conversions>
SUBMIT_PATH = Path(INPUT_PATH, 'sample_submission.csv')
 submit = pd.read_csv(SUBMIT_PATH)
 arr_id = submit['id']
submission = pd.DataFrame({'id': arr_id, 'trip_duration': pred})
 submission.head()
class Get_rid_of_Nans(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def Stringificy(self, X):
X['MSZoning'] = X.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
X['YrSold'] = X.YrSold.astype(str)
 X['YearRemodAdd'] = X.YearRemodAdd.astype(str)
 X['MoSold']= X['MoSold'].astype(str)
 X['YearBuilt']= X['YearBuilt'].astype(str)
 def Nans_from_str(self, X):
missing_val_col = ["Alley", "PoolQC", "MiscFeature", "Fence", "FireplaceQu", "GarageType", "GarageFinish", "GarageQual",
"GarageCond", 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType']
for i in missing_val_col:
X[i] = X[i].fillna('None')
 X['Electrical'] = X['Electrical'].fillna("SBrkr")
 X['Exterior1st'] = X['Exterior1st'].fillna(X['Exterior1st'].mode() [0])
 X['Exterior2nd'] = X['Exterior2nd'].fillna(X['Exterior2nd'].mode() [0])
 X['Functional'] = X['Functional'].fillna(X['Functional'].mode() [0])
 X['SaleType'] = X['SaleType'].fillna(X['SaleType'].mode() [0])
 X['Utilities'] = X['Utilities'].fillna(X['Utilities'].mode() [0])
 X['KitchenQual'] = X['KitchenQual'].fillna(X['KitchenQual'].mode() [0])
 def Nans_from_int(self, X):
missing_val_col2 = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'GarageYrBlt',
'GarageArea', 'GarageCars', 'MasVnrArea']
for i in missing_val_col2:
X[i] = X[i].fillna(0)
 def Nans_from_float(self, X):
X['LotFrontage'] = X.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
def transform(self, X, y=None):
self.X = X.copy()
self.Stringificy(self.X)
 self.Nans_from_int(self.X)
 self.Nans_from_str(self.X)
 self.Nans_from_float(self.X)
 return self.X
def fit_transform(self, X, y=None):
return self.fit(X ).self.transform(X )<categorify>
submission.to_csv("submission.csv", index=False )
<train_model><EOS> 
submission.to_csv("submission.csv", index=False )