Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<feature_engineering>
%matplotlib inline
train = pd.read_csv(".. /input/train.csv")
 testfile = pd.read_csv(".. /input/test.csv")
 train.head(6 )
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )<drop_column>
train = train[(train.trip_duration < 10000)&(train.trip_duration > 100)]
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )<feature_engineering>
train.drop(['id'], axis=1, inplace=True)
 train.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 testfile.drop(['store_and_fwd_flag'], axis=1, inplace=True )
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )<count_values>
train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)
 testfile['pickup_datetime'] = pd.to_datetime(testfile.pickup_datetime)
 train.info()
train['WindSpeed'].value_counts()<feature_engineering>
train['week'] = train.pickup_datetime.dt.week
train['weekday'] = train.pickup_datetime.dt.weekday
train['hour'] = train.pickup_datetime.dt.hour
train.drop(['pickup_datetime'], axis=1, inplace=True)
 train.drop(['dropoff_datetime'], axis=1, inplace=True)
 testfile['week'] = testfile.pickup_datetime.dt.week
testfile['weekday'] = testfile.pickup_datetime.dt.weekday
testfile['hour'] = testfile.pickup_datetime.dt.hour
testfile.drop(['pickup_datetime'], axis=1, inplace=True )
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )<data_type_conversions>
y = train["trip_duration"]
train.drop(["trip_duration"], axis=1, inplace=True)
 X = train
X.shape, y.shape
def str_to_float(txt):
try:
return float(txt)
 except:
return -1<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )<drop_column>
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, random_state=42)
 X_train.shape, y_train.shape, X_valid.shape, y_valid.shape
train.drop('WindDirection', axis=1, inplace=True )<feature_engineering>
m1 = RandomForestRegressor(n_estimators=19, min_samples_split=2, min_samples_leaf=4, max_features='auto', max_depth=80, bootstrap=True)
 m1.fit(X_train, y_train)
 m1.score(X_valid, y_valid )
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right' )<feature_engineering>
print(np.sqrt(MSE(y_valid, m1.predict(X_valid))))
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )<feature_engineering>
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )<count_values>
test_columns = X_train.columns
predictions = m1.predict(testfile[test_columns] )
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]<categorify>
my_submission = pd.DataFrame({'id': testfile.id, 'trip_duration': np.exp(predictions)})
 my_submission.head()
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0<feature_engineering>
my_submission.to_csv("sub.csv", index=False )