Kaggle competition: NBA Rookies
def calculate_direction(d_lon, d_lat):
result = np.zeros(len(d_lon))
l = np.sqrt(d_lon**2 + d_lat**2)
 result[d_lon>0] =(180/np.pi)*np.arcsin(d_lat[d_lon>0]/l[d_lon>0])
 idx =(d_lon<0)&(d_lat>0)
 result[idx] = 180 -(180/np.pi)*np.arcsin(d_lat[idx]/l[idx])
 idx =(d_lon<0)&(d_lat<0)
 result[idx] = -180 -(180/np.pi)*np.arcsin(d_lat[idx]/l[idx])
 return result 
<categorify>
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
def time_type_of_the_day(hour):
time_of_the_day = 2
if(hour >= 4 and hour < 7):
time_of_the_day = 1
elif(hour >= 7 and hour < 12):
time_of_the_day = 2
elif(hour >= 12 and hour < 16):
time_of_the_day = 3
elif(hour >= 16 and hour < 20):
time_of_the_day = 4
elif(hour >= 20 and hour < 23):
time_of_the_day = 5
elif(hour >= 23 and hour < 4):
time_of_the_day = 6
else:
time_of_the_day = 2
return time_of_the_day 
<define_variables>
def classification_reports(classifier,features,labels):
size_data = len(labels)
 count_class_1 = sum(labels)
 count_class_0 = size_data - count_class_1
print(' class 1 : ', count_class_1)
 print(' class 0 : ', count_class_0)
 y = classifier.predict(features)
 fpr, tpr, thresholds = metr.roc_curve(labels, y)
 print("Confusion Matrix:
",metr.confusion_matrix(labels,y))
score=metr.accuracy_score(labels,y)
 print("Score: ",score)
 auc=metr.roc_auc_score(labels,y)
 print("AUC: ",auc)
 print(metr.classification_report(labels,y))
plt.figure()
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
 Legend = "FirstStage"
plt.plot(fpr, tpr,color='darkorange')
 plt.show()
return fpr,tpr,score
def peak_hours_of_the_day(hour, weekday):
peak_hours = 0
if(0 <= weekday <= 4)and(16 <= hour <= 20):
peak_hours = 1
return peak_hours 
<define_variables>
def pre_process(features):
del features['Name']
del features['PlayerID']
x = np.isfinite(features)
 features[(x == False)&(features > 0)] = 10000000
features[(x == False)&(features < 0)] = -10000000
imputer = Imputer(missing_values='NaN', strategy='mean', axis=0 ).fit(features)
 features= imputer.transform(features)
 features = normalize(features, axis=0,norm='max')
 return features
def surcharge_hours_of_the_day(hour):
surcharge_hours = 0
if(hour >= 20 or hour <= 6):
surcharge_hours = 1
return surcharge_hours 
<feature_engineering>
def pre_process_pca(features,test):
Data = np.concatenate(( features,test))
Data = PCA(n_components=.95 ).fit_transform(Data)
 return Data[0:900,:],Data[900:1340,:]
def add_travel_vector_features(df):
df['delta_longitude'] = df.pickup_longitude - df.dropoff_longitude
df['delta_latitude'] = df.pickup_latitude - df.dropoff_latitude
df['pickup_x'] = np.cos(df.pickup_latitude)* np.cos(df.pickup_longitude)
 df['pickup_y'] = np.cos(df.pickup_latitude)* np.sin(df.pickup_longitude)
 df['pickup_z'] = np.sin(df.pickup_latitude)
 df['dropoff_x'] = np.cos(df.dropoff_latitude)* np.cos(df.dropoff_longitude)
 df['dropoff_y'] = np.cos(df.dropoff_latitude)* np.sin(df.dropoff_longitude)
 df['dropoff_z'] = np.sin(df.dropoff_latitude)
<feature_engineering>
def pre_process_manifold(features,test):
n_neighbors = 15
Data = np.concatenate(( features, test))
Data = manifold.Isomap(n_neighbors, n_components=15 ).fit_transform(Data)
 return Data[0:900,:],Data[900:1340,:]
def parse_and_extract_date_time_components(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S %Z')
 df['year'] = df['pickup_datetime'].apply(lambda x: x.year)
 df['month'] = df['pickup_datetime'].apply(lambda x: x.month)
 df['day'] = df['pickup_datetime'].apply(lambda x: x.day)
 df['hour'] = df['pickup_datetime'].apply(lambda x: x.hour)
 df['minute'] = df['pickup_datetime'].apply(lambda x: x.minute)
 df['weekday'] = df['pickup_datetime'].apply(lambda x: x.dayofweek)
 df['weekofyear'] = df['pickup_datetime'].apply(lambda x: x.weekofyear)
 df['dayofyear'] = df['pickup_datetime'].apply(lambda x: x.dayofyear)
 df['quarterofyear'] = df['pickup_datetime'].apply(lambda x: x.quarter)
 df['days_in_month'] = df['pickup_datetime'].apply(lambda x: x.days_in_month)
 df['linear_time'] = df['pickup_datetime'].apply(lambda x: x.value // 10 ** 9)
 df['time_type_of_the_day'] = df['pickup_datetime'].apply(lambda x: time_type_of_the_day(x.hour))
df['peak_hours'] = df['pickup_datetime'].apply(lambda x: peak_hours_of_the_day(x.hour, x.dayofweek))
df['surcharge'] = df['pickup_datetime'].apply(lambda x: surcharge_hours_of_the_day(x.hour))
df['hour_sin'] = np.sin(df.hour *(2.* np.pi / 24))
df['hour_cos'] = np.cos(df.hour *(2.* np.pi / 24))
df['minute_sin'] = np.sin(df.minute *(2.* np.pi / 60))
df['minute_cos'] = np.cos(df.minute *(2.* np.pi / 60))
df['time_type_of_day_sin'] = np.sin(df.time_type_of_the_day *(2.* np.pi / 6))
df['time_type_of_day_cos'] = np.cos(df.time_type_of_the_day *(2.* np.pi / 6))
df['month_sin'] = np.sin(( df.month - 1)*(2.* np.pi / 12))
df['month_cos'] = np.cos(( df.month - 1)*(2.* np.pi / 12))
df['day_sin'] = np.sin(( df.day - 1)*(2.* np.pi / df.days_in_month))
df['day_cos'] = np.cos(( df.day - 1)*(2.* np.pi / df.days_in_month))
df['weekday_sin'] = np.sin(( df.weekday - 1)*(2.* np.pi / 7))
df['weekday_cos'] = np.cos(( df.weekday - 1)*(2.* np.pi / 7))
df['weekofyear_sin'] = np.sin(( df.weekofyear - 1)*(2.* np.pi / 52))
df['weekofyear_cos'] = np.cos(( df.weekofyear - 1)*(2.* np.pi / 52))
df['dayofyear_sin'] = np.sin(( df.dayofyear - 1)*(2.* np.pi / 365))
df['dayofyear_cos'] = np.cos(( df.dayofyear - 1)*(2.* np.pi / 365))
df['quarterofyear_sin'] = np.sin(( df.quarterofyear - 1)*(2.* np.pi / 4))
df['quarterofyear_cos'] = np.cos(( df.quarterofyear - 1)*(2.* np.pi / 4))
del df['hour']
del df['minute']
del df['month']
del df['day']
del df['weekday']
del df['days_in_month']
del df['time_type_of_the_day']
del df['weekofyear']
del df['quarterofyear'] 
<compute_test_metric>
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn import svm
from sklearn import ensemble
from sklearn import naive_bayes
from sklearn import neural_network
from sklearn import neighbors
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import accuracy_score
def haversine_np(lon1, lat1, lon2, lat2):

 lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2.0)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2.0)**2
c = 2 * np.arcsin(np.sqrt(a))
miles = 3959 * c
return miles<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
fname = 'train.csv'
train_df = pd.read_csv('.. /input/train.csv', nrows = TRAINING_SIZE)
 print(train_df.isnull().sum())
 print(fname + ' Old size: %d' % len(train_df))
train_df = train_df.dropna(how = 'any', axis = 'rows')
 print(fname + ' New size: %d' % len(train_df))<feature_engineering>
labels = train.TARGET_5Yrs
features = train.iloc[:,0:21]
add_travel_vector_features(train_df)
 parse_and_extract_date_time_components(train_df)
 print(fname + ' Before dropping Zero pickup longitude and latitude - Old size: %d' % len(train_df))
train_df = train_df[(train_df.pickup_longitude != 0.) &(train_df.pickup_latitude != 0.) ]
print(fname + ' After dropping Zero pickup longitude and latitude - New size: %d' % len(train_df))
print(fname + ' Before dropping Zero dropoff longitude and latitude - Old size: %d' % len(train_df))
train_df = train_df[(train_df.dropoff_longitude != 0.) &(train_df.dropoff_latitude != 0.) ]
print(fname + ' After dropping Zero dropoff longitude and latitude - New size: %d' % len(train_df))
print(fname + ' Before dropping really large lat and lon Outliers - Old size: %d' % len(train_df))
train_df = train_df[(train_df.delta_longitude.abs() < 5.0)&(train_df.delta_latitude.abs() < 5.0)]
print(fname + ' After dropping really large lat and lon Outliers - New size: %d' % len(train_df))
print(fname + ' Before dropping zero delta lat and long values - Old size: %d' % len(train_df))
train_df = train_df[(train_df.delta_longitude != 0.) &(train_df.delta_latitude != 0.) ]
print(fname + ' After dropping delta lat and long values - New size: %d' % len(train_df))
print(fname + ' Before dropping NEGATIVE FARE VALUES - Old size: %d' % len(train_df))
train_df = train_df[(train_df.fare_amount > 0.) ]
print(fname + ' After dropping NEGATIVE FARE VALUES - New size: %d' % len(train_df))
print(fname + ' Before dropping Zero Passenger Counts - Old size: %d' % len(train_df))
train_df = train_df[(train_df.passenger_count > 0)&(train_df.passenger_count < 7)]
print(fname + ' After dropping Zero Passenger Counts - New size: %d' % len(train_df)) 
<feature_engineering>
features = pre_process(features)
 test = pre_process(test )
train_df['distance_in_miles'] = haversine_np(train_df.dropoff_longitude, train_df.dropoff_latitude, train_df.pickup_longitude, train_df.pickup_latitude)
 train_df['direction'] = calculate_direction(train_df.delta_longitude, train_df.delta_latitude)
 train_df['pickup_to_JFK_distance'] = haversine_np(train_df.pickup_longitude, train_df.pickup_latitude, JFK[0], JFK[1])
 train_df['dropoff_to_JFK_distance'] = haversine_np(train_df.dropoff_longitude, train_df.dropoff_latitude, JFK[0], JFK[1])
 train_df['pickup_to_EWR_distance'] = haversine_np(train_df.pickup_longitude, train_df.pickup_latitude, EWR[0], EWR[1])
 train_df['dropoff_to_EWR_distance'] = haversine_np(train_df.dropoff_longitude, train_df.dropoff_latitude, EWR[0], EWR[1])
 train_df['pickup_to_LGA_distance'] = haversine_np(train_df.pickup_longitude, train_df.pickup_latitude, LGA[0], LGA[1])
 train_df['dropoff_to_LGA_distance'] = haversine_np(train_df.dropoff_longitude, train_df.dropoff_latitude, LGA[0], LGA[1])
 train_df['pickup_to_NYC_distance'] = haversine_np(train_df.pickup_longitude, train_df.pickup_latitude, NYC[0], NYC[1])
 train_df['dropoff_to_NYC_distance'] = haversine_np(train_df.dropoff_longitude, train_df.dropoff_latitude, NYC[0], NYC[1])
 train_df['manhattan'] = minkowski_distance(train_df.pickup_longitude, train_df.dropoff_longitude,
train_df.pickup_latitude, train_df.dropoff_latitude, 1)
 train_df['euclidean'] = minkowski_distance(train_df.pickup_longitude, train_df.dropoff_longitude,
train_df.pickup_latitude, train_df.dropoff_latitude, 2)
 train_df['euclidean_3D'] = minkowski_distance_3D(train_df.pickup_x, train_df.dropoff_x,
train_df.pickup_y, train_df.dropoff_y,
train_df.pickup_z, train_df.dropoff_z, 2)
 train_df['manhattan_3D'] = minkowski_distance_3D(train_df.pickup_x, train_df.dropoff_x,
train_df.pickup_y, train_df.dropoff_y,
train_df.pickup_z, train_df.dropoff_z, 1)

print(fname + ' Before dropping HIGH FARE AMOUNTS - Old size: %d' % len(train_df))
train_df = train_df[(train_df.fare_amount > 1.) &(train_df.fare_amount < 251.) ]
print(fname + ' After dropping HIGH FARE AMOUNTS - New size: %d' % len(train_df)) 
<prepare_x_and_y>
classifier1 = neural_network.MLPClassifier(hidden_layer_sizes=(200,10),random_state=0)
 classifier2 = naive_bayes.GaussianNB()
classifier3 =neighbors.KNeighborsClassifier(n_neighbors=1)
 classifier4= svm.NuSVC(kernel='rbf',random_state=0)
 classifier5 = neural_network.MLPClassifier(hidden_layer_sizes=(20,5),random_state=0)
 classifier6 = svm.SVC(kernel='rbf', C=.5,random_state=0)
 classifier7 = ensemble.AdaBoostClassifier(n_estimators=60,learning_rate=.3,random_state=0)
 classifier8 = ensemble.GradientBoostingClassifier(n_estimators=20,max_depth=30,random_state=0)
 classifier9 = ensemble.RandomForestClassifier(n_estimators=60,random_state=0)
 classifier10 = ensemble.ExtraTreesClassifier(n_estimators=10,random_state=0)
 classifier11 = ensemble.BaggingClassifier(n_estimators=10,random_state=0)
 classifier13 = neighbors.KNeighborsClassifier(n_neighbors=3)
 classifier14 = QuadraticDiscriminantAnalysis()
training_set = train_df.loc[:, [
'passenger_count',
'distance_in_miles',
'pickup_x',
'pickup_y',
'pickup_z',
'dropoff_x',
'dropoff_y',
'dropoff_z',
'pickup_to_JFK_distance',
'dropoff_to_JFK_distance',
'pickup_to_NYC_distance',
'dropoff_to_NYC_distance',
'delta_longitude',
'delta_latitude',
'direction',
'year',
'linear_time',
'peak_hours',
'surcharge',
'hour_sin',
'hour_cos',
'minute_sin',
'minute_cos',
'time_type_of_day_sin',
'time_type_of_day_cos',
'month_sin',
'month_cos',
'day_sin',
'day_cos',
'weekday_sin',
'weekday_cos',
'weekofyear_sin',
'weekofyear_cos',
'quarterofyear_sin',
'quarterofyear_cos'
]].values
training_label = train_df.loc[:, 'fare_amount'].values
mean = training_set.mean(axis = 0)
 training_set -= mean
std = training_set.std(axis = 0)
 training_set /= std 
<define_variables>
classifier12 = ensemble.IsolationForest(n_estimators=100,bootstrap="true",contamination=0.1,random_state=0)
 classifier12.fit(features, labels)
 result12 = classifier12.predict(test)
 outlier = classifier12.predict(features)
 delfeatures = np.where(outlier == -1 )
num_validation_samples = VALIDATION_SIZE
if num_validation_samples >= len(training_set):
num_validation_samples = int(len(training_set)* 0.2)
 validation_data = training_set[:num_validation_samples]
training_set = training_set[num_validation_samples:]
validation_targets = training_label[:num_validation_samples]
training_label = training_label[num_validation_samples:] 
<choose_model_class>
for i in np.int32(delfeatures):
features[i,:] = np.sqrt(features[i,:] )
NN_INPUT_SIZE = training_set.shape[1]
if preTrainedModel == True:
model = load_model('./GoodModelRepo/0_NYCTFP_Model_8.h5')
 else:
model = models.Sequential()
model.add(layers.Dense(128, kernel_initializer = 'uniform', activation = 'relu', input_shape =(NN_INPUT_SIZE,)))
 model.add(layers.Dense(128, kernel_initializer = 'uniform', activation = 'relu'))
model.add(layers.Dense(64, kernel_initializer = 'uniform', activation = 'relu'))
model.add(layers.Dense(32, kernel_initializer = 'uniform', activation = 'relu'))

 model.add(layers.Dense(1))
model.compile(optimizer = 'rmsprop', loss = 'mse', metrics = ['mae'])
 if loadOnlyWeights == True:
model.load_weights('./GoodModelRepo/0_NYCTFP_Weights_8.h5')
<train_model>
classifier = ensemble.VotingClassifier\ 
(estimators=[('mlp',classifier1),('svm',classifier4),('knn3',classifier13), 
('knn1',classifier3),('nb',classifier2),('mlpmini', classifier5), 
('svm.5',classifier6),('ada',classifier7),('gb',classifier8), 
('rf', classifier9),('et',classifier10),('bag',classifier11), 
('quad',classifier14)], voting='hard',flatten_transform='true',
weights=[2,2,1,1,1,1,1,1,1,1,1,1,1] 
)
 classifier.fit(features, labels)
 result = classifier.predict(test)
 result[result==-1]=0
checkpointer = ModelCheckpoint(filepath = 'NYCTFP_Ref_Model.h5', monitor = 'val_mean_absolute_error', verbose = 1, save_best_only = True, mode = 'min')
 test_history = model.fit(training_set, training_label, shuffle=True,
validation_data =(validation_data, validation_targets),
epochs = EPOCH, batch_size = BATCH_SIZE,
callbacks = [checkpointer])
<save_to_csv>
scores = cross_val_score(classifier, features, labels, cv=10,scoring='f1')
 classification_reports(classifier,features,labels)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std() * 2))
print(scores)
model.save('NYCTFP_Model_12.h5')
 model.save_weights('NYCTFP_Weights_12.h5')
 pd.DataFrame(test_history.history ).to_csv("epoch_testing_history.csv", header = True, index = False)
<load_from_csv>
cols = { 'PlayerID': [i+901 for i in range(440)] , 'TARGET_5Yrs': result }
submission = pd.DataFrame(cols)
 submission.to_csv("subm.csv", index=False )