Kaggle competition: New York City Taxi Fare Prediction
X = X_train
y = y_train
X_traine, X_val, y_traine, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=42 )<define_variables>
TRAIN_PATH = '.. /input/train.csv'
traintypes = {'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
cols = list(traintypes.keys() )
input_shape=(28,28,1 )<choose_model_class>
with open(TRAIN_PATH)as file:
n_rows = len(file.readlines())
 print(f'Exact number of rows: {n_rows}' )
model = Sequential([
Conv2D(32,(3,3), input_shape=input_shape, padding='same', activation='relu'),
Conv2D(32,(3,3), padding='same', activation='relu'),
BatchNormalization(momentum=0.15),
MaxPool2D(( 2,2)) ,
BatchNormalization(momentum=0.15),
Conv2D(64,(5,5), padding='same', activation='relu'),
Dropout(0.3),
Conv2D(32,(3,3), padding='same', activation='relu'),
Conv2D(32,(3,3), padding='same', activation='relu'),
BatchNormalization(momentum=0.15),
MaxPool2D(( 2,2)) ,
BatchNormalization(momentum=0.15),
Conv2D(64,(5,5), padding='same', activation='relu'),
Dropout(0.3),
Flatten() ,
Dense(128, activation='relu'),
Dropout(0.4),
Dense(10, activation='softmax')
 ] )<choose_model_class>
chunksize = 5_000_000
total_chunk = n_rows // chunksize + 1
print(f'Chunk size: {chunksize:,}
Total chunks required: {total_chunk}' )
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
<choose_model_class>
df_list = []
i=0
for df_chunk in pd.read_csv(TRAIN_PATH, usecols=cols, dtype=traintypes, chunksize=chunksize):
i = i+1
print(f'DataFrame Chunk {i:02d}/{total_chunk}')
 df_chunk['pickup_datetime'] = df_chunk['pickup_datetime'].str.slice(0, 16)
 df_chunk['pickup_datetime'] = pd.to_datetime(df_chunk['pickup_datetime'], utc=True, format='%Y-%m-%d %H:%M')
 df_list.append(df_chunk )
earlystopper = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=5, verbose=0, mode='max', baseline=0.995, restore_best_weights=False)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 callback = [learning_rate_reduction]<train_model>
train_df = pd.read_csv(".. /input/train.csv", nrows = 1000000)
 test_df = pd.read_csv(".. /input/test.csv" )
history = model.fit(
 X_traine, y_traine,
batch_size=512,
epochs=50,
validation_data=(X_val, y_val),
callbacks=callback
)<predict_on_test>
print(train_df.isnull().sum() )
layer_outputs = [layer.output for layer in model.layers[:len(model.layers)]]
activation_model = Model(input=model.input, output=layer_outputs)
 activations = activation_model.predict(test_im.reshape(1,28,28,1))
first_layer_activation = activations[0]
plt.matshow(first_layer_activation[0, :, :, 4], cmap='viridis' )<choose_model_class>
print('Old size: %d' % len(train_df))
train_df = train_df.dropna(how = 'any', axis = 'rows')
 print('New size: %d' % len(train_df))
model.layers[:-1]<define_variables>
train_df['fare_amount'].value_counts()
layer_names = []
for layer in model.layers[:-1]:
layer_names.append(layer.name)
 images_per_row = 16
zipped_layers = zip(layer_names, activations)
 for layer_name, layer_activation in zipped_layers:
if layer_name.startswith('conv'):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,:, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )<define_variables>
train_df = train_df.drop(train_df[train_df['fare_amount'] < 0].index, axis=0)
 train_df.shape
layer_names = []
for layer in model.layers[:-1]:
layer_names.append(layer.name)
 images_per_row = 16
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name.startswith('max'):
n_features = layer_activation.shape[-1]
size = layer_activation.shape[1]
n_cols = n_features // images_per_row
display_grid = np.zeros(( size * n_cols, images_per_row * size))
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,:, :, col * images_per_row + row]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255 ).astype('uint8')
 display_grid[col * size :(col + 1)* size,
row * size :(row + 1)* size] = channel_image
scale = 1./ size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
 plt.grid(False)
 plt.imshow(display_grid, aspect='auto', cmap='viridis' )<compute_test_metric>
train_df = train_df.drop(train_df[train_df['passenger_count'] == 208].index, axis=0 )
final_loss, final_acc = model.evaluate(X_val, y_val, verbose=0)
 print("Final loss: {0:.6f}, final accuracy: {1:.6f}".format(final_loss, final_acc))<save_to_csv>
train_df['key'] = pd.to_datetime(train_df['key'])
 train_df['pickup_datetime'] = pd.to_datetime(train_df['pickup_datetime'] )
predictions = model.predict_classes(X_test, verbose=0)
 predictions = model.predict_classes(X_test, verbose=0)
 submission = pd.read_csv(path_sample_sub)
 submission['label'] = predictions
submission.to_csv('submission.csv', index=False )<import_modules>
test_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'] )
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
import matplotlib.image as mpimg
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense,Conv2D,Flatten,MaxPool2D,Dropout,BatchNormalization
from keras.optimizers import Adam
from keras.callbacks import ReduceLROnPlateau
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix<define_variables>
data = [train_df, test_df]
for i in data:
i['Year'] = i['pickup_datetime'].dt.year
i['Month'] = i['pickup_datetime'].dt.month
i['Date'] = i['pickup_datetime'].dt.day
i['Day of Week'] = i['pickup_datetime'].dt.dayofweek
i['Hour'] = i['pickup_datetime'].dt.hour
IMG_SIZE = 28
N_CHANNELS = 1<load_from_csv>
train_df = train_df.drop(['key','pickup_datetime'], axis = 1)
 test_df = test_df.drop(['key', 'pickup_datetime'], axis = 1 )
train_df = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test_df = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 pred_df = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv' )<concatenate>
x_train = train_df.iloc[:, train_df.columns != 'fare_amount']
y_train = train_df['fare_amount'].values
x_test = test_df
train_df = train_df.append(test_df )<train_model>
print(f'Training set: {train_df.shape}')
 print(f'To be Predicted: {pred_df.shape}' )<prepare_x_and_y>
import lightgbm as lgbm
X_train = train_df.drop(['label'], axis = 1)
 Y_train = train_df['label']
X_pred = pred_df.drop(['id'], axis = 1 )<split>
lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse'
}
X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.05 )<prepare_output>
pred_test_y = np.zeros(x_test.shape[0])
 train_set = lgbm.Dataset(x_train, y_train, silent=True)
 model = lgbm.train(lgbm_params, train_set = train_set, num_boost_round=300)
 pred_test_y = model.predict(x_test, num_iteration = model.best_iteration )
<categorify><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = pred_test_y
submission.to_csv('lgbm_submission.csv', index=False)
 submission.head(20 )