Kaggle competition: IA1819
<SOS> metric: f_{beta} Kaggle data source: ia1819<init_hyperparams>
import numpy as np
import pandas as pd
from sklearn.neighbors import KNeighborsClassifier
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'nthread': 4,
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
'num_rounds':50000
}
cols = list(train_x.select_dtypes(exclude=['float64'] ).columns)
 train_set = lgbm.Dataset(train_x, train_y, silent=False,categorical_feature=cols)
 valid_set = lgbm.Dataset(val_x, val_y, silent=False,categorical_feature=cols)
 model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=500,verbose_eval=500, valid_sets=valid_set )<predict_on_test>
traindf = pd.read_csv(".. /input/train.csv" )
pred_train = model.predict(train_x, num_iteration = model.best_iteration)
 pred_val = model.predict(val_x, num_iteration = model.best_iteration)
 pred_test = model.predict(test_x, num_iteration = model.best_iteration )<set_options>
traindf.drop(['Gender'], axis=1, inplace=True )
gc.collect()<compute_test_metric>
traindata = traindf.values
rmse_train = np.sqrt(mean_squared_error(train_y, pred_train))
rmse_val = np.sqrt(mean_squared_error(val_y, pred_val))
print(rmse_train, rmse_val )<save_to_csv>
descriptores,clases = traindata[:,0:-1],traindata[:,-1]
final = pd.DataFrame({'key':test.timestamp_with_key, 'fare_amount':pred_test}, columns = ['key', 'fare_amount'])
 final.to_csv('submission_lgb.csv', index = False)
 final.head()<load_from_csv>
clases = [1 if clase==True else 0 for clase in clases]
datatypes = {'key': 'str',
'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
train_df = pd.read_csv('.. /input/train.csv', nrows=5000000, dtype=datatypes )<train_model>
knn = KNeighborsClassifier(n_neighbors=1 )
def manhattan_distance(lat1, long1, lat2, long2):
diff_lat = abs(lat1 - lat2)
 diff_long = abs(long1 - long2)
 return(diff_lat + diff_long)
 def distance_between_points(df):
df['diff_lat'] = abs(df['dropoff_latitude'] - df['pickup_latitude'])
 df['diff_long'] = abs(df['dropoff_longitude'] - df['pickup_longitude'])
 df['manhattan_dist'] = df['diff_lat'] + df['diff_long']
jfk = [40.6413, -73.7781]
lga = [40.7769, -73.8740]
ewr = [40.6895, -74.1745]
df['jfk_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], jfk[0], jfk[1])
 df['jfk_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], jfk[0], jfk[1])
 df['lga_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], lga[0], lga[1])
 df['lga_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], lga[0], lga[1])
 df['ewr_dist_pickup'] = manhattan_distance(df['pickup_latitude'], df['pickup_longitude'], ewr[0], ewr[1])
 df['ewr_dist_dropoff'] = manhattan_distance(df['dropoff_latitude'], df['dropoff_longitude'], ewr[0], ewr[1])
 distance_between_points(train_df )<feature_engineering>
knn.fit(descriptores,clases )
def extract_date_details(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S UTC')
 df['year'] = df['pickup_datetime'].apply(lambda date: date.year)
 df['month'] = df['pickup_datetime'].apply(lambda date: date.month)
 df['day'] = df['pickup_datetime'].apply(lambda date: date.weekday())
 df['hour'] = df['pickup_datetime'].apply(lambda date: date.hour)
 extract_date_details(train_df)
 train_df<drop_column>
testdf = pd.read_csv(".. /input/test.csv")
 testdf.drop(['Gender'], axis=1, inplace=True)
 testdata = testdf.values
def remove_outliers(df):
df = df.dropna()
df = df[(df['diff_lat'] < 5.0)&(df['diff_long'] < 5.0)]
df = df[(df['diff_lat'] >.001)&(df['diff_long'] >.001)]
df = df[(df['pickup_longitude'] < -72)&(df['pickup_longitude'] > -75)]
df = df[(df['pickup_latitude'] < 42)&(df['pickup_latitude'] > 39)]
df = df[(df['dropoff_longitude'] < -72)&(df['dropoff_longitude'] > -75)]
df = df[(df['dropoff_latitude'] < 42)&(df['dropoff_latitude'] > 39)]
df = df[(df['fare_amount'] > 2.50)&(df['fare_amount'] < 200)&(df['passenger_count'] <= 6)&(df['passenger_count'] > 0)]
return df
train_df = remove_outliers(train_df)
 len(train_df )<categorify>
predicciones = knn.predict(testdata )
def convert_to_one_hot(column, num_buckets, df, starting_index = 0):
df_size = df.shape[0]
one_hots = np.zeros(( df_size, num_buckets), dtype='byte')
 one_hots[np.arange(df_size), df[column].values - starting_index] = 1
return one_hots<categorify>
predicciones = [True if prediccion==1 else False for prediccion in predicciones]
year = convert_to_one_hot('year', 7, train_df, 2009)
 hour = convert_to_one_hot('hour', 24, train_df, 0 )<define_variables>
soldf = pd.DataFrame(list(enumerate(predicciones)) )
def bucketize_feature(df,column):
buckets = train_df[column].quantile([.1,.2,.3,.4,.5,.6,.7,.8,.9] ).values
bins = np.array(df[column].values)
 lower_bound = -100000
for i in range(buckets.shape[0]):
upper_bound = buckets[i]
bins[(bins >= lower_bound)&(bins < upper_bound)] = i
lower_bound = upper_bound
bins[(bins < 0)|(bins > 8)] = 9
bins = np.array(bins, dtype='byte')
 return bins
p_long = bucketize_feature(train_df, 'pickup_longitude')
 p_lat = bucketize_feature(train_df, 'pickup_latitude')
 d_long = bucketize_feature(train_df, 'dropoff_longitude')
 d_lat = bucketize_feature(train_df, 'dropoff_latitude' )<concatenate>
soldf.columns = ['Id','Prediction']
def feature_cross(a1, a2):
rows = a1.shape[0]
cols = 100
cross = np.zeros(( rows, cols), dtype='byte')
 cross[np.arange(rows),(a1 * 10)+ a2] = 1
return cross
p_lat_x_long = feature_cross(p_lat, p_long)
 d_lat_x_long = feature_cross(d_lat, d_long )<concatenate>
soldf.to_csv("submission.csv",sep=',',index=False )