Kaggle competition: ML in biology
manhattan = train_df['manhattan_dist'].values.reshape(len(train_df), 1)
 jfk_p = train_df['jfk_dist_pickup'].values.reshape(len(train_df), 1)
 jfk_d = train_df['jfk_dist_dropoff'].values.reshape(len(train_df), 1)
 lga_p = train_df['lga_dist_pickup'].values.reshape(len(train_df), 1)
 lga_d = train_df['lga_dist_dropoff'].values.reshape(len(train_df), 1)
 ewr_p = train_df['ewr_dist_pickup'].values.reshape(len(train_df), 1)
 ewr_d = train_df['ewr_dist_dropoff'].values.reshape(len(train_df), 1)
 train_X = np.concatenate(( p_lat_x_long, d_lat_x_long, year, hour, manhattan, jfk_p, jfk_d, lga_p, lga_d, ewr_p, ewr_d), axis=1)
 train_y = train_df['fare_amount'].values
print(train_X.shape)
 print(train_y.shape )<load_from_csv>
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white' )
validate_df = pd.read_csv('.. /input/train.csv', skiprows=range(1,10000001), nrows=10000, dtype=datatypes )<categorify>
sample = pd.read_csv('/kaggle/input/mlbio1/sample_submission.csv')
 test = pd.read_csv('/kaggle/input/mlbio1/test.csv')
 train = pd.read_csv('/kaggle/input/mlbio1/train.csv')
distance_between_points(validate_df)
 validate_df = remove_outliers(validate_df)
 def extract_features(df):
extract_date_details(df)
 p_lo = bucketize_feature(df, 'pickup_longitude')
 p_la = bucketize_feature(df, 'pickup_latitude')
 d_lo = bucketize_feature(df, 'dropoff_longitude')
 d_la = bucketize_feature(df, 'dropoff_latitude')
 p_la_x_lo = feature_cross(p_la, p_lo)
 d_la_x_lo = feature_cross(d_la, d_lo)
 yr = convert_to_one_hot('year', 7, df, 2009)
 hr = convert_to_one_hot('hour', 24, df, 0)
 manhattan = df['manhattan_dist'].values.reshape(len(df), 1)
 jfk_p = df['jfk_dist_pickup'].values.reshape(len(df), 1)
 jfk_d = df['jfk_dist_dropoff'].values.reshape(len(df), 1)
 lga_p = df['lga_dist_pickup'].values.reshape(len(df), 1)
 lga_d = df['lga_dist_dropoff'].values.reshape(len(df), 1)
 ewr_p = df['ewr_dist_pickup'].values.reshape(len(df), 1)
 ewr_d = df['ewr_dist_dropoff'].values.reshape(len(df), 1)
 print(p_la_x_lo.shape)
 print(d_la_x_lo.shape)
 print(yr.shape)
 print(hr.shape)
 print(manhattan.shape)
 X = np.concatenate(( p_la_x_lo, d_la_x_lo, yr, hr, manhattan, jfk_p, jfk_d, lga_p, lga_d, ewr_p, ewr_d), axis=1)
 return X
X = extract_features(validate_df)
 true_y = validate_df['fare_amount'].values<choose_model_class>
total = train.isnull().sum().sort_values(ascending=False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
model = tf.keras.Sequential()
model.add(layers.Dense(128, activation='relu', input_dim=238))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer='adam',
loss='mse',
metrics=['mae'])
 model.fit(train_X, train_y, epochs=15, batch_size=256,validation_split=0.1 )<predict_on_test>
train = train.drop('smoking_status', 1)
 test = test.drop('smoking_status', 1 )
result = model.predict(X ).flatten()<compute_test_metric>
median_bmi = train['bmi'].median()
train['bmi'] = train['bmi'].fillna(median_bmi)
 test['bmi'] = test['bmi'].fillna(median_bmi )
mean_y = np.mean(train_df['fare_amount'].values)
 diff = true_y - result
mse = np.sum(diff ** 2)/ len(diff)
 rmse = np.sqrt(mse)
 print(rmse )<predict_on_test>
def cross_validation_for_roc_auc(clf, X, y ,cv=5):
X = np.array(X.copy())
 y = np.array(y.copy())
 kf = KFold(n_splits=cv)
 kf.get_n_splits(X)
 scores = []
for train_index, test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
clf.fit(X_train, y_train)
 prediction_on_this_fold = clf.predict_proba(X_test)[:,1]
score = roc_auc_score(y_score=prediction_on_this_fold, y_true=y_test)
 scores.append(score)
 return scores
distance_between_points(test_df)
 X_test = extract_features(test_df)
 pred_y_test = model.predict(X_test ).flatten()<load_from_csv>
def calc_smooth_mean(train, test, by, on, m):
mean = train[on].mean()
agg = train.groupby(by)[on].agg(['count', 'mean'])
 counts = agg['count']
means = agg['mean']
smooth =(counts * means + m * mean)/(counts + m)
 return train[by].map(smooth), test[by].map(smooth )
sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<save_to_csv>
for const in range(0, 21, 5):
lgb = LGBMClassifier(n_estimators=50, max_depth=3)
 sgd = SGDClassifier(loss='log', penalty = 'elasticnet')
 train['gender_target_enc'],test['gender_target_enc'] = \
calc_smooth_mean(train, test, by='gender',on='stroke', m=const)
 print(const, np.mean(cross_validation_for_roc_auc(lgb, train[['gender_target_enc']] , train['stroke'])) ,
np.mean(cross_validation_for_roc_auc(sgd, train[['gender_target_enc']] , train['stroke'])) )
sample_submission['fare_amount'] = pd.Series(pred_y_test)
 sample_submission.to_csv('nn_submission.csv', index=False )<set_options>
train = train.drop('gender_target_enc', 1)
 test = test.drop('gender_target_enc', 1)
 train = train.drop('gender', 1)
 test = test.drop('gender', 1 )
warnings.filterwarnings('ignore' )<load_from_csv>
train['work_type_target_enc'],test['work_type_target_enc'] = \
calc_smooth_mean(train, test, by='work_type',on='stroke', m=0 )
train_data = pd.read_csv('.. /input/train.csv',nrows=20000)
 test_data = pd.read_csv('.. /input/test.csv')
 train_data.head()
test_data.head()<data_type_conversions>
for i in train['work_type'].unique() :
train['work_type_is_{}'.format(i)] =(train['work_type'] == i)*1
test['work_type_is_{}'.format(i)] =(test['work_type'] == i)*1
train = train.drop('work_type', 1 )
def changeDataType(dataset):
dataset['passenger_count'] = dataset.passenger_count.astype('uint8')
 dataset['pickup_longitude'] = dataset.pickup_longitude.astype('float32')
 dataset['pickup_latitude'] = dataset.pickup_latitude.astype('float32')
 dataset['dropoff_longitude'] = dataset.dropoff_longitude.astype('float32')
 dataset['dropoff_latitude'] = dataset.dropoff_latitude.astype('float32')
 dataset['pickup_datetime'] = pd.to_datetime(arg=dataset['pickup_datetime'],format='%Y-%m-%d %H:%M:%S UTC')
 dataset.info()
changeDataType(train_data)
 print("--"*40)
 changeDataType(test_data)
 train_data['fare_amount'] = train_data.fare_amount.astype('float32')
<count_missing_values>
train['ever_married'] = train['ever_married'].replace({'Yes':1, 'No':0 })
 test['ever_married'] = test['ever_married'].replace({'Yes':1, 'No':0 })
 train['Residence_type'] = train['Residence_type'].replace({'Urban':1, 'Rural':0 })
 test['Residence_type'] = test['Residence_type'].replace({'Urban':1, 'Rural':0 } )
train_data.isnull().sum()
train_data = train_data.dropna(axis=0)
 train_data.isnull().sum()<count_values>
all_features = [ 'age', 'hypertension', 'heart_disease', 'ever_married',
'Residence_type', 'avg_glucose_level', 'bmi', 'work_type_target_enc',
'work_type_is_children',
'work_type_is_Private', 'work_type_is_Never_worked',
'work_type_is_Self-employed', 'work_type_is_Govt_job']
p = pd.cut(train_data.fare_amount,3)
 p.value_counts()<filter>
sgd = SGDClassifier(max_iter=1000, loss='log', penalty = 'elasticnet')
 lgb = LGBMClassifier()
rndf = RandomForestClassifier()
train_data = train_data[train_data.fare_amount<400]<drop_column>
for f in all_features:
scores = cross_validation_for_roc_auc(sgd, train[[f]] , train['stroke'])
 print(f, np.mean(scores))
train_data = train_data.drop(( train_data[train_data['pickup_latitude']<-90] |(train_data[train_data['pickup_latitude']>90])).index,axis=0)
 train_data = train_data.drop(( train_data[train_data['pickup_longitude']<-180] |(train_data[train_data['pickup_longitude']>180])).index,axis=0)
 train_data = train_data.drop(( train_data[train_data['dropoff_longitude']<-180] |(train_data[train_data['dropoff_longitude']>180])).index,axis=0)
 train_data = train_data.drop(( train_data[train_data['dropoff_latitude']<-90] |(train_data[train_data['dropoff_latitude']>90])).index,axis=0 )<normalization>
features_sgd=['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc', 'work_type_is_Self-employed', 'work_type_is_children']
def degree_to_radion(degree):
return degree*(np.pi/180)
 def calculate_distance(pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude):
from_lat = degree_to_radion(pickup_latitude)
 from_long = degree_to_radion(pickup_longitude)
 to_lat = degree_to_radion(dropoff_latitude)
 to_long = degree_to_radion(dropoff_longitude)
 radius = 6371.01
lat_diff = to_lat - from_lat
long_diff = to_long - from_long
a = np.sin(lat_diff / 2)**2 + np.cos(degree_to_radion(from_lat)) * np.cos(degree_to_radion(to_lat)) * np.sin(long_diff / 2)**2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
return radius * c<train_model>
for f in all_features:
scores = cross_validation_for_roc_auc(lgb, train[[f]] , train['stroke'])
 print(f, np.mean(scores))
train_data['distance'] = calculate_distance(train_data.pickup_latitude, train_data.pickup_longitude, train_data.dropoff_latitude, train_data.dropoff_longitude )<compute_test_metric>
features_lgb = ['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc', 'bmi', 'work_type_is_Self-employed', 'work_type_is_children']
test_data['distance'] = calculate_distance(test_data.pickup_latitude, test_data.pickup_longitude, test_data.dropoff_latitude, test_data.dropoff_longitude )<count_values>
for f in all_features:
scores = cross_validation_for_roc_auc(rndf, train[[f]] , train['stroke'])
 print(f, np.mean(scores))
p = pd.cut(train_data.distance,10)
 p.value_counts()<filter>
features_rndf = ['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc','bmi', 'work_type_is_Self-employed', 'work_type_is_children']
train_data = train_data.loc[train_data.distance<200]<drop_column>
sgd = SGDClassifier(max_iter=2000, loss='log', penalty = 'elasticnet')
 scores = cross_validation_for_roc_auc(sgd, train[features_sgd] , train['stroke'])
 np.mean(scores )
train_data = train_data.drop(columns='key' )<drop_column>
lgb = LGBMClassifier(n_estimators=45,max_depth=3)
 scores = cross_validation_for_roc_auc(lgb, train[features_lgb] , train['stroke'])
 np.mean(scores )
test_data_key = test_data['key']
test_data = test_data.drop(columns='key' )<feature_engineering>
rndf = RandomForestClassifier(n_estimators=530, max_depth=6)
 scores = cross_validation_for_roc_auc(rndf, train[features_rndf] , train['stroke'])
 np.mean(scores )
data = [train_data,test_data]
for i in data:
i['Year'] = i['pickup_datetime'].dt.year
i['Month'] = i['pickup_datetime'].dt.month
i['Date'] = i['pickup_datetime'].dt.day
i['Day of Week'] = i['pickup_datetime'].dt.dayofweek
i['Hour'] = i['pickup_datetime'].dt.hour<filter>
lgb.fit(train[features_rndf], train['stroke'])
 Y_pred = pd.DataFrame(lgb.predict_proba(test[features_rndf])) [1]
submission = pd.DataFrame({
"id": test["id"],
"stroke": Y_pred
})
 submission.to_csv(".. /working/submit.csv", index=False)
 submission.sample(10 )