Kaggle competition: DM-Assignment 1
<SOS> metric: categorizationaccuracy Kaggle data source: dm-assignment-1<save_to_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, RobustScaler, LabelEncoder
from sklearn.preprocessing import normalize, MinMaxScaler, MaxAbsScaler
from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN, Birch
if useBestModel == True:
del model
model = load_model('NYCTFP_Ref_Model.h5')
 predictions = model.predict(testing_set)
 outputKeys = np.reshape(test_df.iloc[:, 0].values,(len(test_df), 1))
outputdata = np.concatenate(( outputKeys, predictions), axis = 1)
 submission = pd.DataFrame(outputdata, columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False)
<load_from_csv>
def accuracy_score(list1, list2):
count = 0
for i in range(len(list1)) :
if(list1[i]==list2[i]):
count+= 1
print(count/len(list1))
<feature_engineering>
os.getcwd()
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
print(tf.__version__)
 assert tf.__version__ >= "1.8" or tf.__version__ >= "1.10"
tf.logging.set_verbosity(tf.logging.INFO)
 CSV_COLUMNS = ['fare_amount', 'pickup_datetime','pickup_longitude','pickup_latitude',
'dropoff_longitude','dropoff_latitude', 'passenger_count', 'key']
LABEL_COLUMN = 'fare_amount'
TRAIN_LINES = 55423856
DEBUG=False 
<feature_engineering>
df = pd.read_csv('/kaggle/input/dmassign1/data.csv',low_memory = False)
 df.head()
@contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s' )<feature_engineering>
df.replace('?',np.NaN,inplace = True )
PATH = '.. /input'
train_df = pd.read_csv(f'{PATH}/train.csv', nrows=10000)
 train_df['distance'] = np.sqrt(np.abs(train_df['pickup_longitude']-train_df['dropoff_longitude'])**2 +
np.abs(train_df['pickup_latitude']-train_df['dropoff_latitude'])**2)
 train_df.head()
train_df.describe()<define_variables>
le = LabelEncoder()
BATCH_SIZE=1
dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=f'{PATH}/train.csv',
batch_size=BATCH_SIZE,
column_names=None,
column_defaults=None,
label_name='fare_amount',
select_columns=[1, 2, 3, 4, 5, 6, 7],
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=None,
shuffle=True,
shuffle_buffer_size=10000,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100
)<categorify>
df['Col189'] = le.fit_transform(df['Col189'])
 df['Col190'] = df['Col190'].replace({'sacc1':1, 'sacc2':2, 'sacc4':3, 'sacc5':4})
 df['Col191'] = df['Col191'].replace({'time1':1, 'time2':2, 'time3':3})
 df['Col192'] = df['Col192'].replace({'p1':1,'p2':2,'p3':3,'p4':4,'p5':5,'p6':6,'p7':7,'p8':8,'p9':9,'p10':10})
 df['Col192'].fillna(df['Col192'].value_counts().idxmax() ,inplace = True)
 df['Col193'].fillna(df['Col193'].value_counts().idxmax() ,inplace = True )
def filter_data(features, label=None):
val = tf.greater(label , tf.constant(20.0, tf.float32))
return True
if DEBUG:
dataset = dataset.filter(filter_data)
 next_element = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
features, label = sess.run(next_element)
 print("Features:
", features, "
Label:
", label)
<data_type_conversions>
df['Col194'].fillna(df['Col194'].value_counts().idxmax() ,inplace = True)
 df['Col194'] = df['Col194'].replace({'ab':1, 'ac':2, 'ad':3})
 df['Col195'].fillna(df['Col195'].value_counts().idxmax() ,inplace = True)
 df['Col195'] = df['Col195'].replace({'Jb1':1, 'Jb2':2, 'Jb3':3, 'Jb4':4})
 df['Col196'].fillna(df['Col196'].value_counts().idxmax() ,inplace = True)
 df['Col196'] = df['Col196'].replace({'H1':1, 'H2':2, 'H3':3})
 df['Col197'] = df['Col197'].replace({'me':'ME', 'sm':'SM', 'M.E.':'ME','la':'LA'})
 df['Col197'].fillna(df['Col197'].value_counts().idxmax() ,inplace = True)
 df['Col197'] = df['Col197'].replace({'SM':1, 'ME':2, 'LA':3, 'XL':4})
 df = pd.get_dummies(data=df,columns = ['Col193','Col196','Col191'] )
def pd_weekDay(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.weekday.astype(np.int32)
 def pd_dayofYear(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.dayofyear.astype(np.int32)
 if DEBUG:
years=np.array([2018, 2018, 2018])
 months=np.array([8, 11, 1])
 days=np.array([20, 6, 8])
 print(pd_dayofYear(years, months, days)) 
<categorify>
cols = [col for col in df if col != 'Class'] + ['Class']
def tf_isAirport(latitude,longitude,airport_name='JFK'):
jfkcoord = tf.constant([-73.8352, -73.7401, 40.6195, 40.6659])
 ewrcoord = tf.constant([-74.1925, -74.1531, 40.6700, 40.7081])
 lgucoord = tf.constant([-73.8895, -73.8550, 40.7664, 40.7931])
 if airport_name=='JFK':
coord = jfkcoord
elif airport_name=='EWR':
coord = ewrcoord
elif airport_name=='LGU':
coord = lgucoord
else:
raise ValueError(f'Unknown NYC Airport {airport_name}')
 is_airport = \
tf.logical_and(
 tf.logical_and(
 tf.greater(longitude, coord[0]), tf.less(longitude, coord[1])
),
tf.logical_and(
 tf.greater(latitude, coord[2]), tf.less(latitude, coord[3])
)
)
 return is_airport 
<data_type_conversions>
df = df[cols]
def feat_eng_func(features, label=None):
print("Feature Engineered Label:", label)
 features['pickup_year'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 0, 4), tf.int32)
 features['pickup_month'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 5, 2), tf.int32)
 features['pickup_day'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 8, 2), tf.int32)
 features['pickup_hour'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 11, 2), tf.int32)
 features['pickup_weekday'] = tf.py_func(pd_weekDay,
[features['pickup_year'], features['pickup_month'], features['pickup_day']],
tf.int32,
stateful=False,
name='Weekday' 
)
 features['pickup_dense_year'] =(
 tf.cast(features['pickup_year'], tf.float32)+ \
tf.cast(features['pickup_month'], tf.float32)/ tf.constant(12.0, tf.float32)- \
tf.constant(2009.0, tf.float32)) / \
tf.constant(6.0, tf.float32)
 features['night'] = tf.cast(
 tf.logical_or(tf.greater(features['pickup_hour'], 19), tf.less(features['pickup_hour'], 7)) ,
tf.float32)
 minlat = tf.constant(38.0)
 maxlat = tf.constant(42.0)
 minlon = tf.constant(-76.0)
 maxlon = tf.constant(-72.0)
 features['pickup_longitude'] = tf.clip_by_value(features['pickup_longitude'], minlon, maxlon)
 features['pickup_latitude'] = tf.clip_by_value(features['pickup_latitude'], minlat, maxlat)
 features['dropoff_longitude'] = tf.clip_by_value(features['dropoff_longitude'], minlon, maxlon)
 features['dropoff_latitude'] = tf.clip_by_value(features['dropoff_latitude'], minlat, maxlat)
 minpass = tf.constant(1.0)
 maxpass = tf.constant(6.0)
 features['passenger_count'] = tf.clip_by_value(tf.cast(features['passenger_count'], tf.float32), minpass, maxpass)
 if label != None:
minfare = tf.constant(3.0)
 maxfare = tf.constant(300.0)
 label = tf.clip_by_value(label, minfare, maxfare)
 features['longitude_dist'] = tf.abs(features['pickup_longitude'] - features['dropoff_longitude'])
 features['latitude_dist'] = tf.abs(features['pickup_latitude'] - features['dropoff_latitude'])
 features['distance'] = tf.sqrt(features['longitude_dist']**2 + features['latitude_dist']**2)
 features['pick_dist_center'] = distance_from_loc(nyccenter[0], nyccenter[1],
features['pickup_longitude'], features['pickup_latitude'])
 features['drop_dist_center'] = distance_from_loc(nyccenter[0], nyccenter[1],
features['dropoff_longitude'], features['dropoff_latitude'])
 features['pick_angle'] = angle_from_nyc(features['pickup_longitude'],features['pickup_latitude'])
 features['drop_angle'] = angle_from_nyc(features['dropoff_longitude'],features['dropoff_latitude'])
 features['angle'] = features['pick_angle'] - features['drop_angle']
if label == None:
return features
return(features, label )<define_variables>
df[obj_cols] = df[obj_cols].astype(float )
def read_dataset(filename, mode, batch_size = 512):
def _input_fn() :
if mode == tf.estimator.ModeKeys.TRAIN:
num_epochs = None
shuffle = False
else:
num_epochs = 1
shuffle = False
if mode == tf.estimator.ModeKeys.PREDICT:
label_name=None
select_columns=[1, 2, 3, 4, 5, 6]
else:
label_name ='fare_amount'
select_columns = [1, 2, 3, 4, 5, 6, 7]
file_list = tf.gfile.Glob(filename)
 dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=file_list,
batch_size=batch_size,
column_names=None,
column_defaults=None,
label_name=label_name,
select_columns=select_columns,
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=num_epochs,
shuffle=shuffle,
shuffle_buffer_size=128*batch_size,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 skip_train_lines = TRAIN_LINES // batch_size // 100 * 10
if mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.skip(skip_train_lines)
 elif mode == tf.estimator.ModeKeys.EVAL:
dataset = dataset.take(skip_train_lines)
 dataset = dataset.map(feat_eng_func)
 return dataset.make_one_shot_iterator().get_next()
return _input_fn 
<load_from_csv>
data = df.copy()
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = 8)
<train_model>
X = data.drop(['ID','Class'],axis=1)
 y = data['Class']
with timer('Evaluating'):
with tf.Session() as sess:
features, label = sess.run(train_input_fn())
 print("Features:
", features, "
Label:
", label )<define_variables>
X.fillna(X.mean() ,inplace = True )
NUMERIC_COLUMNS = ['passenger_count', 'pickup_dense_year',
'longitude_dist', 'latitude_dist',
'distance', 'pick_dist_center', 'drop_dist_center', 'angle',
'pick_angle','drop_angle',
]
BOOL_COLUMNS = ['is_JFK_pickup', 'is_JFK_dropoff', 'is_EWR_pickup', 'is_EWR_dropoff',
'is_LGU_pickup', 'is_LGU_dropoff', 'is_NYC_airport'
]
def create_feature_cols() :
hour_cat = tf.feature_column.categorical_column_with_identity('pickup_hour', 24)
 weekday_cat = tf.feature_column.categorical_column_with_identity('pickup_weekday', 7)
 hour_X_weekday = tf.feature_column.crossed_column([hour_cat, weekday_cat], 500)
 NUM_BUCKETS = 27
long_list = list(np.linspace(-74.2, -73.7, NUM_BUCKETS))
lat_list = list(np.linspace(40.55, 41.0, NUM_BUCKETS))
p_lon = tf.feature_column.numeric_column('pickup_longitude')
 p_lat = tf.feature_column.numeric_column('pickup_latitude')
 d_lon = tf.feature_column.numeric_column('dropoff_longitude')
 d_lat = tf.feature_column.numeric_column('dropoff_latitude')
 buck_p_lon = tf.feature_column.bucketized_column(p_lon, long_list)
 buck_p_lat = tf.feature_column.bucketized_column(p_lat, lat_list)
 buck_d_lon = tf.feature_column.bucketized_column(d_lon, long_list)
 buck_d_lat = tf.feature_column.bucketized_column(d_lat, lat_list)
 X_p = tf.feature_column.crossed_column([buck_p_lon, buck_p_lat], 3*NUM_BUCKETS**2)
 X_d = tf.feature_column.crossed_column([buck_d_lon, buck_d_lat], 3*NUM_BUCKETS**2)
 X_loc = tf.feature_column.crossed_column([X_p, X_d],(3*NUM_BUCKETS**2)**2)
 numeric_columns = [ tf.feature_column.numeric_column(key)
 for key in NUMERIC_COLUMNS]
other_columns = [tf.feature_column.indicator_column(hour_cat),
tf.feature_column.embedding_column(hour_X_weekday, 2),
tf.feature_column.embedding_column(
 tf.feature_column.categorical_column_with_vocabulary_list('pickup_month',(0, 1, 2, 3, 4, 5, 6,
7, 8, 9, 10, 11, 12)) ,
2),
tf.feature_column.embedding_column(X_loc, NUM_BUCKETS)
 ]
return numeric_columns + other_columns
def create_sparse_feature_cols() :
bool_columns = [ tf.feature_column.indicator_column(
 tf.feature_column.categorical_column_with_identity(key, 2))
for key in BOOL_COLUMNS]
return bool_columns 
<load_from_csv>
X_data = X
BATCH_SIZE = 512
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 def train_and_evaluate(output_dir, num_train_steps):
runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=25000, log_step_count_steps=25000,
save_checkpoints_steps=50000,
tf_random_seed = 42 
)
 optimizer = tf.contrib.opt.LazyAdamOptimizer(
)
 estimator = tf.estimator.DNNRegressor(model_dir = output_dir, feature_columns = create_feature_cols() ,
hidden_units=[128, 128, 128],
optimizer=optimizer,
config=runconfig,
batch_norm=True,
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE)
 train_spec = tf.estimator.TrainSpec(input_fn = train_input_fn,
max_steps = num_train_steps)
 eval_spec = tf.estimator.EvalSpec(input_fn = eval_input_fn,
steps = None,
start_delay_secs = 0,
throttle_secs = 60)
 evaluation, result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
 return estimator, evaluation, result
OUTDIR = './trained_model'
shutil.rmtree(OUTDIR, ignore_errors = True)
 estimator, evaluation, result = train_and_evaluate(OUTDIR, 180000 )<load_from_csv>
X = StandardScaler().fit_transform(X )
if False:
BATCH_SIZE = 512
OUTDIR = './trained_model'
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 shutil.rmtree(OUTDIR, ignore_errors = True)
 runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=2500, log_step_count_steps=2500,
save_checkpoints_steps=500000,
tf_random_seed = 42 
)
 optimizer = tf.train.FtrlOptimizer(learning_rate=0.05,
learning_rate_power=-0.5,
initial_accumulator_value=0.1,
l1_regularization_strength=0.0,
l2_regularization_strength=0.0 
)
 estimator = tf.estimator.DNNRegressor(model_dir = OUTDIR, feature_columns = create_feature_cols() ,
hidden_units=[128, 128, 128],
batch_norm=True,
config=runconfig,
dropout=0.0)
 with timer('Training...'):
estimator.train(train_input_fn, max_steps=130000)
 with timer('Evaluating'):
evaluation = estimator.evaluate(eval_input_fn, name='train_eval')
<save_to_csv>
X = pd.DataFrame(X,columns = X_data.columns)
 X.head()
print(evaluation)
 avg_loss = evaluation['average_loss']
predict_input_fn = read_dataset(f'{PATH}/test.csv', tf.estimator.ModeKeys.PREDICT, batch_size=1)
 predictions = estimator.predict(predict_input_fn)
 test_df = pd.read_csv(f'{PATH}/test.csv', nrows=10000)
 s = pd.Series()
for i, p in enumerate(predictions):
if i < 9915:
s.at[i] = p['predictions'][0]
else:
break
test_df['fare_amount'] = s
sub = test_df[['key', 'fare_amount']]
sub.to_csv(f'DNNregr-{avg_loss:4.4}.csv', index=False, float_format='%.1f')
<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv', nrows = 13_000_000)
 train_df.dtypes
train_df.head(20)
 train_df.info()
test_df = pd.read_csv('.. /input/test.csv')
 test_df.dtypes
test_df.head(20)
 test_df.info() 
<feature_engineering>
X2 = X_data
def prepare_time_features(df):
df['pickup_datetime_dt'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S UTC')
 df['hour_of_day'] = df.pickup_datetime_dt.dt.hour
df['month'] = df.pickup_datetime_dt.dt.month
df["year"] = df.pickup_datetime_dt.dt.year
df['day_of_year'] = df.pickup_datetime_dt.dt.dayofyear
df["day_of_week"] = df.pickup_datetime_dt.dt.dayofweek
return df
train_df = prepare_time_features(train_df)
 train_df["is_weekend"] = 0
train_df.loc[train_df['day_of_week']==6, ['is_weekend']] = 1
train_df.loc[train_df['day_of_week']==5, ['is_weekend']] = 1
train_df["passenger_count_class"] = 0
train_df.loc[train_df['passenger_count']>=5, ['passenger_count_class']] = 1
train_df[train_df.passenger_count_class==1].head(20)
 test_df = prepare_time_features(test_df)
 test_df["is_weekend"] = 0
test_df.loc[test_df['day_of_week']==6, ['is_weekend']] = 1
test_df.loc[test_df['day_of_week']==5, ['is_weekend']] = 1
test_df["passenger_count_class"] = 0
test_df.loc[test_df['passenger_count']>=5, ['passenger_count_class']] = 1
test_df[test_df.passenger_count_class==1].head(20)
<feature_engineering>
X2 = StandardScaler().fit_transform(X2)
 X2 = normalize(X2,norm='l1')
 X2 = pd.DataFrame(X2,columns = X_data.columns )
def add_travel_vector_features(df):
degree_len_long = 84.1
degree_len_lat = 111.1
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
df['d'] = np.sqrt(( df.abs_diff_longitude*degree_len_long)**2 +(df.abs_diff_latitude*degree_len_lat)**2)
 df['d_long'] = df.abs_diff_longitude*degree_len_long
df['d_lat'] = df.abs_diff_latitude*degree_len_lat
add_travel_vector_features(train_df)
 add_travel_vector_features(test_df)
<drop_column>
from sklearn.decomposition import PCA
print('Old size: %d' % len(train_df))
train_df = train_df[(train_df.abs_diff_longitude < 5.0)&(train_df.abs_diff_latitude < 5.0)]
train_df = train_df.dropna(how = 'any', axis = 'rows')
 train_df = train_df[train_df.pickup_longitude<0]
train_df = train_df[train_df.pickup_latitude >0]
train_df = train_df[train_df.dropoff_longitude<0]
train_df = train_df[train_df.dropoff_latitude>0]
train_df = train_df[train_df.d>=0.01]
print('New size: %d' % len(train_df))
train_fare_amount_mean = train_df['fare_amount'].mean()
def search_fare(test_df,row_num,train_df,delta=0.0,delta_step=0.001,delta_hour=2,row_count_min=1,fare_amount_max=30):
degree_len_long = 84.1
degree_len_lat = 111.1
row_count = 0
df_row = test_df.iloc[row_num]
p_long = df_row.pickup_longitude
p_lat = df_row.pickup_latitude
d_long = df_row.dropoff_longitude
d_lat = df_row.dropoff_latitude
hour_of_day = df_row.hour_of_day
year = df_row.year
month = df_row.month
is_weekend = df_row.is_weekend
day_of_week = df_row.day_of_week
test_key = df_row.key
passenger_count = df_row.passenger_count
passenger_count_class = df_row.passenger_count_class
d = df_row.d
i_step = 0
while(( row_count < row_count_min)&(i_step<10)) :
delta = delta+delta_step
i_step+=1
df_result = train_df[(abs(train_df.pickup_longitude - p_long)*degree_len_long<=delta)&(abs(train_df.pickup_latitude - p_lat)*degree_len_lat<=delta)
 &(abs(train_df.dropoff_longitude - d_long)*degree_len_long<=delta)&(abs(train_df.dropoff_latitude - d_lat)*degree_len_lat<=delta)&(train_df.key !=test_key)
 &(train_df.year ==year)&(train_df.month == month)]
row_count = df_result.key.count()
if(row_count > 0):
fare_amount_mean = df_result.fare_amount.mean()
fare_amount_pred = fare_amount_mean
else:
fare_amount_mean = train_fare_amount_mean
fare_amount_pred = fare_amount_mean
return fare_amount_pred, delta, row_count, fare_amount_mean, df_result
def get_my_score(y_pred,y_real):
my_sum =0.0
for j in range(0,y_pred.size):
my_sum +=(y_pred.iat[j] - y_real.iat[j])**2
my_sum = np.sqrt(my_sum/y_pred.size)
 return my_sum
def get_prediction(test_df, train_df, row_num = 0, delta_step = 0.006, delta_hour=2, row_count_min=1,fare_amount_max=30):
if(row_num ==0):
row_num = test_df.shape[0]
print(row_num)
 print(delta_step)
 sub_df = pd.DataFrame()
sub_df['key'] = test_df.iloc[0:row_num]['key']
sub_df['fare_amount'] = 0
if(test_df.columns[1]=='fare_amount'):
sub_df['fare_amount_train'] = test_df['fare_amount']
sub_df['row_count'] = 0
sub_df['delta'] = 0
print(datetime.datetime.now())
 d1 = datetime.datetime.now()
for i in range(test_df.iloc[0:row_num].shape[0]):
if(i%100 ==0):
print(i)
 fare_amount_pred, delta, row_count,fare_amount_mean, df_result = search_fare(test_df,i,train_df,delta=0.0,
delta_step=delta_step,delta_hour=delta_hour,row_count_min=row_count_min,
fare_amount_max = fare_amount_max)
 sub_df['fare_amount'].iloc[i]=round(fare_amount_pred,2)
 if(test_df.columns[1]=='fare_amount'):
sub_df['row_count'].iloc[i]=row_count
sub_df['delta'].iloc[i]=delta
print(sub_df.shape)
 sub_df.to_csv('submission.csv', index = False)
 d2 = datetime.datetime.now()
print(d2-d1)
 print('row_num=' + str(row_num))
print('delta_step=' + str(delta_step))
print('delta_hour=' + str(delta_hour))
print('row_count_min=' + str(row_count_min))
print('fare_amount_max=' + str(fare_amount_max))
if(test_df.columns[1]=='fare_amount'):
print(get_my_score(test_df.iloc[0:row_num]['fare_amount'],sub_df['fare_amount']))
return sub_df
i_count = 0
mean_score = 0
for i in range(100,100):
print(i)
 i_count+= 1
y_train = train_df['fare_amount']
X_train_d, X_test_d, y_train_d, y_test_d = train_test_split(train_df, y_train, test_size=0.00002, random_state=i)
 X_train_d = pd.DataFrame()
print(X_test_d.shape)
 sub_df= get_prediction(test_df=X_test_d, train_df=train_df, row_num = 0, delta_step = 0.6,
row_count_min=10, fare_amount_max=1000)
 run_score = get_my_score(X_test_d['fare_amount'],sub_df['fare_amount'])
 print('run_score=' + str(run_score))
mean_score+= run_score
if(i_count>0):
print('mean_total_score=' + str(mean_score/i_count))
sub_df['fare_error'] = abs(sub_df['fare_amount'] - sub_df['fare_amount_train'])
 sub_df.sort_values(by=['fare_error'] ).tail(20)
 sub_df= get_prediction(test_df=test_df, train_df=train_df, row_num = 0, delta_step = 0.6, row_count_min=1)
 sub_df.to_csv('submission01.csv', index = False )<import_modules>
pca = PCA(n_components=70)
 principalComponents = pca.fit_transform(X2)
 features = range(pca.n_components_)
 plt.bar(features, pca.explained_variance_ratio_, color='blue')
 plt.xlabel('PCA features')
 plt.ylabel(range(10))
plt.xticks(features)
 PCA_components = pd.DataFrame(principalComponents )
print(dt.datetime.now() , 'python version:')
 print(sys.version)
 print(dt.datetime.now() , 'pandas version:', pd.__version__ )<load_from_csv>
def check_acc(old_y,new_y,k,mp):
for i in range(1,k+1):
mx = -1;
mxInd = 1;
for j in range(1,6):
count = 0
num_count = 0
for k in range(len(new_y)) :
if(old_y[k] == j):
num_count += 1
if(old_y[k]==j and new_y[k]==i):
count += 1
frac = count/num_count
if(mx<frac):
mx = frac
mxInd = j
mp.append(mxInd)
print(dt.datetime.now() , 'start reading data...')
 dirc = '.. /input/'
train = pd.read_csv(dirc + 'train.csv')
 test = pd.read_csv(dirc + 'test.csv')
 submission = pd.read_csv(dirc + 'sample_submission.csv')
 x = train.drop(['ID_code', 'target'], axis=1)
 y = train.target
print('train shape', train.shape)
 print('test shape', test.shape)
 print(dt.datetime.now() , 'finish reading data')
 test.drop('ID_code', axis=1, inplace=True )<train_model>
for k in range(5,20):
kmeans = KMeans(n_clusters = k,random_state=100 ).fit(principalComponents)
 y2 = kmeans.labels_
y2 = y2 + 1
mp = []
mp = []
check_acc(y[:1300],y2[:1300],k,mp)
 edited_y = []
for i in range(len(y2)) :
for j in range(1,k+1):
if(y2[i]==j):
edited_y.append(mp[j-1])
 print(k)
 accuracy_score(edited_y[:1300],y[:1300])
 unique_elements, counts_elements = np.unique(edited_y[1300:], return_counts=True)
 print(np.asarray(( unique_elements, counts_elements)) )
n_sp = 5
sfk = StratifiedKFold(n_splits=n_sp)
 pred_train_mtx = np.zeros(len(train))
pred_test_mtx = np.zeros(( len(test), n_sp))
i = 0
for train_idx, val_idx in sfk.split(x, y):
train_x = x.iloc[train_idx, :]
val_x = x.iloc[val_idx, :]
train_y = y.iloc[train_idx]
val_y = y.iloc[val_idx]
print(dt.datetime.now() , 'cycle', i, 'start training...')
 model = XGBClassifier(n_estimators=100000, n_jobs=32, tree_method='gpu_hist',
learning_rate=0.01,
max_delta_step=2, colsample_bylevel=0.6, colsample_bytree=0.1,
gamma=0.004, max_bin=256, max_depth=8, max_leaves=27, min_child_weight=96,
reg_alpha=0.003, reg_lambda=0.060, subsample=0.4)
 model.fit(train_x, train_y, eval_set=[(val_x, val_y)],
verbose=500,
eval_metric='auc', early_stopping_rounds=2500)
 pred_temp = model.predict_proba(val_x)[:, 1]
print('roc_auc_score:', roc_auc_score(val_y, pred_temp))
pred_train_mtx[val_idx] = pred_temp
pred_test_mtx[:, i] = model.predict_proba(test)[:, 1]
print(dt.datetime.now() , 'finish cycle', i, 'training')
 print()
i += 1
print('final auc score:', roc_auc_score(y, pred_train_mtx))
pred = pred_test_mtx.mean(axis = 1 )<save_to_csv>
k = 16
submission['target'] = pred
submission.to_csv('submission.csv', index=False )<load_from_csv>
kmeans = KMeans(n_clusters = k, random_state=100 ).fit(principalComponents )
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )<count_values>
y2 = kmeans.labels_
y2 = y2 + 1
train["target"].value_counts()<count_values>
unique_elements, counts_elements = np.unique(y2[:1300], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )
count_class_0, count_class_1 = train.target.value_counts()
df_class_0 = train[train['target'] == 0]
df_class_1 = train[train['target'] == 1]<compute_test_metric>
unique_elements, counts_elements = np.unique(y[:1300], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall 
<choose_model_class>
mp = []
check_acc(y[:1300],y2[:1300],k,mp )
def _Model1() :
inp = Input(shape=(200, 1))
d1 = Dense(128, activation='sigmoid' )(inp)
 d2 = Dense(64, activation='relu' )(d1)
 d3 = Dense(32, activation='sigmoid' )(d2)
 d4 = Dense(16, activation='relu' )(d3)
 f2 = Flatten()(d4)
 preds = Dense(1, activation='sigmoid' )(f2)
 model = Model(inputs=inp, outputs=preds)
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',recall])
 return model 
<choose_model_class>
edited_y = []
for i in range(len(y2)) :
for j in range(1,k+1):
if(y2[i]==j):
edited_y.append(mp[j-1] )
def _Model2() :
inp = Input(shape=(200, 1))
d1 = Dense(64, activation='sigmoid' )(inp)
 d2 = Dense(128, activation='relu' )(d1)
 d3 = Dense(32, activation='sigmoid' )(d2)
 d4 = Dense(16, activation='relu' )(d3)
 f2 = Flatten()(d4)
 preds = Dense(1, activation='sigmoid' )(f2)
 model = Model(inputs=inp, outputs=preds)
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',recall])
 return model<choose_model_class>
accuracy_score(edited_y[:1300],y[:1300] )
def _Model4() :
inp = Input(shape=(200, 1))
d1 = Dense(128, activation='sigmoid' )(inp)
 d2 = Conv1D(64, 2,activation='relu' )(d1)
 d3 = Dense(32, activation='sigmoid' )(d2)
 d4 = Dense(16, activation='relu' )(d3)
 f2 = Flatten()(d4)
 preds = Dense(1, activation='sigmoid' )(f2)
 model = Model(inputs=inp, outputs=preds)
 model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',recall])
 return model<prepare_x_and_y>
unique_elements, counts_elements = np.unique(edited_y[1300:], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
 df_train=train.drop(["ID_code","target"],axis=1)
 y_train=train["target"]
df_test=test.drop(["ID_code"],axis=1)
 scaler=StandardScaler()
df_train=scaler.fit_transform(df_train)
 df_test=scaler.fit_transform(df_test)
<create_dataframe>
ids = df['ID']
ids = ids[1300:]
ids = ids.tolist()
df_train= pd.DataFrame(df_train)
 df_test= pd.DataFrame(df_test)
<train_model>
submission = pd.concat([pd.Series(ids),pd.Series(edited_y[1300:])],axis=1)
 submission.columns = ['ID','Class']
submission.to_csv('submission.csv', index=False)
 submission.head()
preds = []
c = 0
oof_preds = np.zeros(( len(df_train), 1))
for train, valid in cv.split(df_train, y_train):
print("VAL %s" % c)
 X_train = np.reshape(df_train.iloc[train].values,(-1, 200, 1))
y_train_ = y_train.iloc[train].values
X_valid = np.reshape(df_train.iloc[valid].values,(-1, 200, 1))
y_valid = y_train.iloc[valid].values
model = _Model2()
history=model.fit(X_train, y_train_, validation_data=(X_valid, y_valid), epochs=20, verbose=2, batch_size=256)
 X_test = np.reshape(df_test.values,(200000, 200, 1))
curr_preds = model.predict(X_test, batch_size=2048)
 oof_preds[valid] = model.predict(X_valid)
 preds.append(curr_preds)
 c += 1
auc = roc_auc_score(y_train, oof_preds)
 print("CV_AUC: {}".format(auc))<prepare_output>
maps = {}
for i in range(len(y)) :
key = i
maps.setdefault(key, [] )
prediction=(preds[0]+preds[1]+preds[2]+preds[3]+preds[4])/5<save_to_csv>
for i in range(len(y)) :
maps[i].append(edited_y[i] )
sub = pd.DataFrame()
sub["ID_code"] = test["ID_code"]
sub["target"] = prediction
sub.to_csv("submission-cnn-ksplit.csv", index=False )<save_to_csv>
for i in range(10):
print(maps[i], y[i] )
sub.to_csv("submission-cnn-ksplit.csv", index=False )<prepare_x_and_y>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index=False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}"target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html )
preds = []
c = 0
oof_preds = np.zeros(( len(df_train), 1))
df_under_x=df_test_under.drop(["ID_code","target"],axis=1)
 df_under_y=df_test_under["target"]
for train, valid in cv.split(df_under_x,df_under_y):
print("VAL %s" % c)
 X_train = np.reshape(df_under_x.iloc[train].values,(-1, 200, 1))
y_train_ = df_under_y.iloc[train].values
X_valid = np.reshape(df_under_x.iloc[valid].values,(-1, 200, 1))
y_valid = df_under_y.iloc[valid].values
model = _Model1()
history=model.fit(X_train, y_train_, validation_data=(X_valid, y_valid), epochs=20, verbose=2, batch_size=256)
 X_test = np.reshape(df_test.values,(200000, 200, 1))
curr_preds = model.predict(X_test, batch_size=2048)
 oof_preds[valid] = model.predict(X_valid)
 preds.append(curr_preds)
 c += 1
auc = roc_auc_score(y_train, oof_preds)
 print("CV_AUC: {}".format(auc))<split>
create_download_link(submission )