Kaggle competition: APTOS 2019 Blindness Detection
scaler = StandardScaler()
sample = scaler.fit_transform(sample )<define_variables>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sample = train[:50000][['dropoff_longitude','dropoff_latitude']]<train_model>
from fastai import *
from fastai.vision import *
import torch
model = KMeans(n_clusters = 5)
 model.fit(train[['pickup_longitude','pickup_latitude']])
 pickup_labels = model.predict(train[['pickup_longitude','pickup_latitude']] )<categorify>
path = Path('.. /input/aptos2019-blindness-detection')
 path_train = path/'train_images'
path_test = path/'test_images'
path, path_train, path_test
train['pickup_cluster'] = pickup_labels
pickup_cluster = pd.get_dummies(train['pickup_cluster'],prefix='pickup_cluster',drop_first=True)
 train = pd.concat([train,pickup_cluster],axis=1 ).drop('pickup_cluster',axis=1 )<predict_on_test>
labels = pd.read_csv(path/'train.csv')
 labels.head()
model2 = KMeans(n_clusters = 8)
 model2.fit(train[['dropoff_longitude','dropoff_latitude']])
 dropoff_labels = model2.predict(train[['dropoff_longitude','dropoff_latitude']] )<categorify>
tfms = get_transforms(
 do_flip=True,
flip_vert=True,
max_warp=0.1,
max_rotate=360.,
max_zoom=1.1,
max_lighting=0.1,
p_lighting=0.5
)
train['dropoff_cluster'] = dropoff_labels
dropoff_cluster = pd.get_dummies(train['dropoff_cluster'],prefix='dropoff_cluster',drop_first=True)
 train = pd.concat([train,dropoff_cluster],axis=1 ).drop('dropoff_cluster',axis=1 )<feature_engineering>
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081] )
train['pickup_cluster'] = pickup_labels
train['dropoff_cluster'] = dropoff_labels<groupby>
test_labels = pd.read_csv(path/'sample_submission.csv')
 test = ImageList.from_df(test_labels, path = path_test, suffix = '.png' )
train.groupby(['pickup_cluster','dropoff_cluster'])['fare_amount'].agg(['mean','std', 'count'] )<predict_on_test>
src =(ImageList.from_df(labels, path = path_train, suffix = '.png')
.split_by_rand_pct(seed = 42)
.label_from_df(cols = 'diagnosis')
.add_test(test))
pickup_test_labels = model.predict(test[['pickup_longitude','pickup_latitude']])
 test['pickup_cluster'] = pickup_test_labels
pickup_cluster = pd.get_dummies(test['pickup_cluster'],prefix='pickup_cluster',drop_first=True)
 test = pd.concat([test,pickup_cluster],axis=1 ).drop('pickup_cluster',axis=1 )<predict_on_test>
data =(
 src.transform(
 tfms,
size = 512,
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros' 
)
.databunch(bs=8)
.normalize(aptos19_stats))
dropoff_test_labels = model2.predict(test[['dropoff_longitude','dropoff_latitude']])
 test['dropoff_cluster'] = dropoff_test_labels
dropoff_cluster = pd.get_dummies(test['dropoff_cluster'],prefix='dropoff_cluster',drop_first=True)
 test = pd.concat([test,dropoff_cluster],axis=1 ).drop('dropoff_cluster',axis=1 )<categorify>
kappa = KappaScore()
kappa.weights = "quadratic"
train = pd.get_dummies(train,columns=['month','hour','day_of_week'],drop_first=True)
 test = pd.get_dummies(test,columns=['month','hour','day_of_week'],drop_first=True )<import_modules>
learn = cnn_learner(
 data,
models.resnet152,
metrics = [accuracy, kappa],
model_dir = Path('.. /kaggle/working'),
path = Path(".")
 )
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn import metrics<drop_column>
learn.fit_one_cycle(3 )
train = train.drop(['pickup_cluster','dropoff_cluster'],axis=1 )<split>
learn.save('resnet152-1' )
X = train.drop('fare_amount',axis=1)
 y = train[['fare_amount']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42 )<train_model>
learn.load('resnet152-1');
clf = RandomForestRegressor(n_estimators=30,min_samples_split=5,min_samples_leaf=3,random_state=21)
 clf.fit(X_train,y_train )<predict_on_test>
learn.fit_one_cycle(3, slice(1e-6,1e-5))
predictions = clf.predict(X_test)
 error = np.sqrt(metrics.mean_squared_error(y_test,predictions))
print(error )<save_to_csv>
learn.save('resnet152-2' )
predictions = clf.predict(test)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = predictions
submission.to_csv('submission.csv',index=False )<set_options>
learn.load('resnet152-2');
warnings.filterwarnings('ignore')
 client = Client()
% matplotlib inline
print(os.listdir(".. /input"))
data_train_file_path = ".. /input/train.csv"
data_test_file_path = ".. /input/test.csv"<load_from_csv>
data =(
 src.transform(
 tfms,
size = 1024,
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros' 
)
.databunch(bs=4)
.normalize(aptos19_stats))
%%time
db_rows_count = 55423857
with open(data_train_file_path)as f:
db_rows_count = len(f.readlines())
 print("no.of rows in the training data : {0}
".format(db_rows_count))<define_variables>
learn.data = data
train_types = {'fare_amount' : 'float32',
'pickup_datetime' : 'str',
'pickup_longitude' : 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32'}
test_types = { 'pickup_datetime' : 'str',
'key' : 'str',
'pickup_longitude' : 'float32',
'pickup_latitude' : 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude' : 'float32'}
train_cols = list(train_types.keys())
 test_cols = list(test_types.keys())
 BB =(-74.5, -72.8, 40.5, 41.8)
 frac = 0.00050
def select_within_boundingbox(df, BB):
return(df.pickup_longitude >= BB[0])&(df.pickup_longitude <= BB[1])& \ 
(df.pickup_latitude >= BB[2])&(df.pickup_latitude <= BB[3])& \ 
(df.dropoff_longitude >= BB[0])&(df.dropoff_longitude <= BB[1])& \ 
(df.dropoff_latitude >= BB[2])&(df.dropoff_latitude <= BB[3])
<load_from_csv>
learn.fit_one_cycle(4, 2e-4 )
def load_data(data_file_path, train_data='Y'):
if(train_data=='Y'):
df = dd.read_csv(data_file_path,usecols=train_cols, dtype=train_types)
 column_names = ["fare_amount",
"pickup_longitude",
"pickup_latitude",
"dropoff_longitude",
"dropoff_latitude"]
df = df.sample(frac=0.04)
 df = df.dropna(how="any", subset = train_cols)
 df = df[(df[column_names] != 0 ).all(axis=1)]
df = df.loc[(df.fare_amount > 0)&(df.fare_amount < 100)&
~(((df.pickup_longitude - df.dropoff_longitude)== 0)& 
(( df.pickup_latitude - df.dropoff_latitude)== 0)) ]
df = df[select_within_boundingbox(df, BB)]
if(train_data == 'N'):
df = dd.read_csv(data_file_path,usecols=test_cols, dtype=test_types)
 df = dd.concat([
df,dd.to_datetime(df['pickup_datetime'] ).apply(
 lambda x: pd.Series([x.year, x.month, x.day, x.weekday() , x.hour, x.minute],
index=['pickup_year', 'pickup_month', 'pickup_dd' ,'pickup_weekday', 'pickup_hour', 'pickup_minute'])) ], axis=1)
 df = client.persist(df)
 return df 
<load_pretrained>
learn.save('resnet152-3' )
df = load_data(data_train_file_path, 'Y')
 progress(df )<train_model>
learn.load('resnet152-3');
after = len(df)
 print('
\t actual : {0}
\t after applying filters : {1}
\t dropped rows: {2} '.format(db_rows_count, after, db_rows_count-after))<compute_test_metric>
learn.fit_one_cycle(6, 2e-5 )
def distance_haversine(lon1, lat1, lon2, lat2):
p = 0.017453292519943295
a = 0.5 - cos(( lat2 - lat1)* p)/2 + cos(lat1 * p)* cos(lat2 * p)*(1 - cos(( lon2 - lon1)* p)) / 2
return 12742 * asin(sqrt(a)) * 0.62137<compute_test_metric>
learn.save('resnet152-4' )
def distance_rows(p_lon, p_lat, d_lon, d_lat):
nyc_coord =(40.7141667, -74.0063889,)
 jfk_coord =(40.639722, -73.778889)
 ewr_coord =(40.6925, -74.168611)
 lga_coord =(40.77725, -73.872611)
 distance_between_pickup_dropoff = distance_haversine(p_lon, p_lat, d_lon, d_lat)
 distance_between_pickup_jfk = distance_haversine(p_lon, p_lat, jfk_coord[1], jfk_coord[0])
 distance_between_dropoff_jfk = distance_haversine(jfk_coord[1], jfk_coord[0], d_lon, d_lat)
 distance_between_pickup_ewr = distance_haversine(p_lon, p_lat, ewr_coord[1], ewr_coord[0])
 distance_between_dropoff_ewr = distance_haversine(ewr_coord[1], ewr_coord[0], d_lon, d_lat)
 distance_between_pickup_lga = distance_haversine(p_lon, p_lat, lga_coord[1], lga_coord[0])
 distance_between_dropoff_lga = distance_haversine(lga_coord[1], lga_coord[0], d_lon, d_lat)
 distance_between_citycenter_pickup = distance_haversine(nyc_coord[0], nyc_coord[1],p_lon, p_lat)
 longitude_diff = p_lon - d_lon
latitude_diff = p_lat - d_lat
return [distance_between_pickup_dropoff,
distance_between_pickup_jfk,
distance_between_dropoff_jfk,
distance_between_pickup_ewr,
distance_between_dropoff_ewr,
distance_between_pickup_lga,
distance_between_dropoff_lga,
distance_between_citycenter_pickup,
longitude_diff,
latitude_diff]
def calculate_coordinates_distance(df, train_data='Y'):
column_names = ['distance_between_pickup_dropoff',
'distance_between_pickup_jfk',
'distance_between_dropoff_jfk',
'distance_between_pickup_ewr',
'distance_between_dropoff_ewr',
'distance_between_pickup_lga',
'distance_between_dropoff_lga',
'distance_between_citycenter_pickup',
'longitude_diff',
'latitude_diff']
df = dd.concat([df,df[["pickup_longitude","pickup_latitude", "dropoff_longitude","dropoff_latitude"]].apply(lambda x: pd.Series(distance_rows(*x),index=column_names), axis=1)], axis=1)
 if(train_data == 'Y'):
df = df.loc[df.distance_between_pickup_dropoff>0.05]
df['fare_per_mile'] = df.fare_amount/df.distance_between_pickup_dropoff
df = df.reset_index(drop=True)
 df = client.persist(df)
 return df 
<train_model>
learn.load('resnet152-4');
df = calculate_coordinates_distance(df, train_data='Y')
 progress(df )<normalization>
learn.load('resnet152-4');
@dask.delayed
def round_decimals(x, x_decimals=2):
return x.round(x_decimals)
 @dask.delayed
def math_sqrt(x):
return math.sqrt(x )<compute_test_metric>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
fare_amount_mean = df["fare_amount"].mean()
fare_amount_standard_deviation = math_sqrt(((df["fare_amount"] - fare_amount_mean)** 2 ).mean())
 print("average fair amount(mean): ${0:.2f}".format(fare_amount_mean.compute()))
print("fare amount standard deviation : ${0:.2f}
".format(fare_amount_standard_deviation.compute()))<filter>
submission = pd.read_csv(path/'sample_submission.csv')
 submission.head()
df_day = df.loc[(df.pickup_hour >=6)&(df.pickup_hour <16)]
df_peak_hours = df.loc[(( df.pickup_hour >=16)&(df.pickup_hour <20)) ]
df_night = df.loc[~(( df.pickup_hour >=6)&(df.pickup_hour <20)) ]
df_between_airports = df.loc[(((df.distance_between_pickup_jfk < 2)|(df.distance_between_pickup_ewr < 2)|(df.distance_between_pickup_lga < 2)) & 
(( df.distance_between_dropoff_jfk < 2)|(df.distance_between_dropoff_lga < 2)|(df.distance_between_dropoff_ewr < 2)))]
df_airport_pickup = df.loc[(( df.distance_between_pickup_jfk < 2)|(df.distance_between_pickup_ewr < 2)|(df.distance_between_pickup_lga < 2)) ]
df_airport_dropoff = df.loc[(( df.distance_between_dropoff_jfk < 2)|(df.distance_between_dropoff_lga < 2)|(df.distance_between_dropoff_ewr < 2)) ]
BB_manhattan =(-74.025, 40.7, -73.925, 40.8)
 df_jfk_manhattan = df[(select_within_boundingbox(df, BB_manhattan)& 
(( df.distance_between_pickup_jfk < 2)|(df.distance_between_dropoff_jfk < 2)))]
df = df.reset_index(drop=True )<feature_engineering>
preds = np.array(preds.argmax(1)).astype(int ).tolist()
preds[:5]
%%time
fare_amount_per_mile = df.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_day = df_day.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_peak_hours = df_peak_hours.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_night = df_night.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_between_airports = df_between_airports.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_airport_pickup = df_airport_pickup.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_airport_dropoff = df_airport_dropoff.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_jfk_manhattan = df_jfk_manhattan.fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_weekday = df.loc[df.pickup_weekday<=4].fare_per_mile.mean().compute().round(2)
 fare_amount_per_mile_weekend = df.loc[df.pickup_weekday>=5].fare_per_mile.mean().compute().round(2)
 avg_data = pd.DataFrame({'fare':[
fare_amount_per_mile_between_airports,
fare_amount_per_mile_jfk_manhattan,
fare_amount_per_mile_airport_pickup,
fare_amount_per_mile_airport_dropoff,
fare_amount_per_mile_weekend,
fare_amount_per_mile_weekday,
fare_amount_per_mile_peak_hours,
fare_amount_per_mile_night,
fare_amount_per_mile_day,
fare_amount_per_mile
]}, index = [
'between_airports',
'jfk_manhattan',
'airport_pickup',
'airport_dropoff',
'week end',
'week day',
'peak hours(4-8pm)',
'night_ride',
'day_ride',
'all_day']
).dropna()<groupby>
submission['diagnosis'] = preds
submission.head()
fare_per_mile_yr = df.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_day_yr = df_day.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_peak_hours_yr = df_peak_hours.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_night_yr = df_night.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_weekday_yr = df.loc[df.pickup_weekday<=4].groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_weekend_yr = df.loc[df.pickup_weekday>=5].groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_airport_pickup_yr = df_airport_pickup.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_airport_dropoff_yr = df_airport_dropoff.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_between_airports_yr = df_between_airports.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_per_mile_jfk_manhattan_yr = df_jfk_manhattan.groupby('pickup_year')['fare_per_mile'].mean().compute()
fare_mile = pd.concat([
pd.DataFrame({'year':fare_per_mile_yr.index, 'avg_fare':fare_per_mile_yr.values, 'type':'overall'}),
pd.DataFrame({'year':fare_per_mile_day_yr.index, 'avg_fare':fare_per_mile_day_yr.values, 'type':'day_time'}),
pd.DataFrame({'year':fare_per_mile_peak_hours_yr.index, 'avg_fare':fare_per_mile_peak_hours_yr.values, 'type':'peak_hours'}),
pd.DataFrame({'year':fare_per_mile_night_yr.index, 'avg_fare':fare_per_mile_night_yr.values, 'type':'night_time'}),
pd.DataFrame({'year':fare_per_mile_weekday_yr.index, 'avg_fare':fare_per_mile_weekday_yr.values, 'type':'weekend'}),
pd.DataFrame({'year':fare_per_mile_weekend_yr.index, 'avg_fare':fare_per_mile_weekend_yr.values, 'type':'weekday'}),
pd.DataFrame({'year':fare_per_mile_airport_pickup_yr.index, 'avg_fare':fare_per_mile_airport_pickup_yr.values, 'type':'airport_pickup'}),
pd.DataFrame({'year':fare_per_mile_airport_dropoff_yr.index, 'avg_fare':fare_per_mile_airport_dropoff_yr.values, 'type':'airport_dropoff'}),
pd.DataFrame({'year':fare_per_mile_jfk_manhattan_yr.index, 'avg_fare':fare_per_mile_jfk_manhattan_yr.values, 'type':'jfk_manhattan'}),
pd.DataFrame({'year':fare_per_mile_between_airports_yr.index, 'avg_fare':fare_per_mile_between_airports_yr.values, 'type':'between_airports'}),
] ).reset_index(drop = True)
<split>
warnings.simplefilter(action='ignore', category=FutureWarning )
<train_model><EOS> 
submission.to_csv('submission.csv', index = False )