Kaggle competition: New York City Taxi Trip Duration
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
%matplotlib inline
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
input_dir = '.. /input/'
working_dir = '.. /working/'<set_options>
train['trip_duration'] = np.log(train['trip_duration'].values )
%matplotlib inline<load_from_csv>
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['pickup_hour'] = train['pickup_datetime'].dt.hour
train['pickup_minute'] = train['pickup_datetime'].dt.minute
train['pickup_time']=train['pickup_hour']*60 +train['pickup_minute']
train = pd.read_csv(os.path.join(input_dir, 'train.csv'))
test = pd.read_csv(os.path.join(input_dir, 'test.csv'))
train.index = train['Id'].values
test.index = test['Id'].values
print(train.shape)
 print(test.shape )<categorify>
test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 test['pickup_hour'] = test['pickup_datetime'].dt.hour
test['pickup_minute'] = test['pickup_datetime'].dt.minute
test['pickup_time']=test['pickup_hour']*60 +test['pickup_minute']
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data<drop_column>
def haversine(lat1, lon1, lat2, lon2):
R = 6371800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train = train.query('parentesco1==1')
 train = train.drop('parentesco1', axis=1)
 test = test.drop('parentesco1', axis=1)
 print(train.shape )<prepare_x_and_y>
train['distance'] = train.apply(lambda row: haversine(row['pickup_latitude'],row['pickup_longitude'],row['dropoff_latitude'], row['dropoff_longitude']), axis=1)
 test['distance'] = test.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'],row['dropoff_latitude'], row['dropoff_longitude']), axis=1 )
def get_numeric(data, status_name):
status_cols = [s for s in data.columns.tolist() if status_name in s]
print('status column names')
 print(status_cols)
 status_df = data[status_cols]
status_df.columns = list(range(status_df.shape[1]))
status_numeric = status_df.idxmax(1)
 status_numeric.name = status_name
data = pd.concat([data, status_numeric], axis=1)
 return data<data_type_conversions>
X = train[['distance','passenger_count','pickup_latitude','dropoff_latitude','pickup_longitude','dropoff_longitude','pickup_time']
]
y = train['trip_duration']
X.shape, y.shape
status_name_list = ['epared', 'etecho', 'eviv', 'instlevel']
for status_name in status_name_list:
train = get_numeric(train, status_name)
 test = get_numeric(test, status_name )<define_variables>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import SGDRegressor
from sklearn.model_selection import ShuffleSplit
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'paredother', 'pisoother', 'abastaguano',
'energcocinar1', 'techootro', 'sanitario6', 'elimbasu6',
'estadocivil7', 'parentesco12', 'tipovivi5',
'lugar1', 'area1', 'female', 'epared1', 'epared2',
'epared3', 'etecho1', 'etecho2', 'etecho3',
'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',
'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',
'instlevel7', 'instlevel8', 'instlevel9']
SQB_cols = [s for s in train.columns.tolist() if 'SQB' in s]
parentesco_cols = [s for s in train.columns.tolist() if 'parentesco' in s]
needless_cols.extend(SQB_cols)
 needless_cols.extend(parentesco_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.10, random_state=42)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
ori_train = pd.read_csv(os.path.join(input_dir, 'train.csv'))
ori_train_X = ori_train.drop(['Id', 'Target', 'idhogar'], axis=1)
 train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)
 print('feature columns
{} -> {}'.format(ori_train_X.shape[1], train_X.shape[1]))<prepare_x_and_y>
m1 = RandomForestRegressor(n_estimators=30, min_samples_leaf= 3)
 m1.fit(X_train, y_train)
train_Id = train['Id']
train_idhogar = train['idhogar']
train_y = train['Target']
train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)
 test_Id = test['Id']
test_idhogar = test['idhogar']
test_X = test.drop(['Id', 'idhogar'], axis=1)
 all_Id = pd.concat([train_Id, test_Id], axis=0, sort=False)
 all_idhogar = pd.concat([train_idhogar, test_idhogar], axis=0, sort=False)
 all_X = pd.concat([train_X, test_X], axis=0, sort=False )<train_on_grid>
X_test=m1.predict(test[['distance','passenger_count','pickup_latitude','dropoff_latitude','pickup_longitude','dropoff_longitude','pickup_time']])
 X_test, len(X_test )
X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.1, random_state=0)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train)
<load_pretrained>
submit = pd.read_csv('.. /input/sample_submission.csv' )
with open(os.path.join(working_dir, '20180801_lgbm.pickle'), mode='wb')as f:
pickle.dump(gbm, f )<compute_train_metric>
my_submission = pd.DataFrame({'id': test.id, 'trip_duration':np.exp(X_test)} )
<save_to_csv><EOS> 
my_submission.to_csv('submission.csv', index=False )