Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<define_variables>
%matplotlib inline
warnings.filterwarnings("ignore" )
SUBMISSIONS_PATH = '/kaggle/input/submissions/'<sort_values>
train = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/train.zip')
 test = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/test.zip')
 sample_submission = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/sample_submission.zip' )
submissions_all = []
for dirname, _, filenames in os.walk(SUBMISSIONS_PATH):
for filename in filenames:
submissions_all.append(os.path.join(dirname, filename))
submissions_all.sort()
print(submissions_all )<load_from_csv>
train.isnull().sum()
def ensemble(submissions_all, sub_idx,weights=[]):
submission_with_weight = []
for i in range(len(sub_idx)) :
print(f"I'm taking submission {submissions_all[sub_idx[i]]} with weight {weights[i]}")
 submission = pd.read_csv(submissions_all[sub_idx[i]])
 submission = submission.loc[:, ['healthy', 'multiple_diseases', 'rust', 'scab']].values
submission_with_weight.append(submission*weights[i])
 submission_avg = sum(submission_with_weight)
 return submission_avg<save_to_csv>
train.isnull().sum()
def make_submission_file(submission_avg, submissions_all):
submission_df = pd.read_csv(submissions_all[0])
 submission_df.iloc[:, 1:] = 0
submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = submission_avg
submission_df.to_csv('submission.csv', index=False )<define_search_space>
from scipy import stats
from scipy.stats import norm
submission_avg = ensemble(submissions_all,[0,2,5], [0.1, 0.8,0.1])
 make_submission_file(submission_avg,submissions_all )<import_modules>
train['trip_duration'] = np.log(train['trip_duration'].values )
print(tf.__version__)
 print(tf.keras.__version__ )<install_modules>
feature_names=list(test)
 df_train=train[feature_names]
df=pd.concat(( df_train, test))
!pip install efficientnet 
<set_options>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'] )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['weekday'] = df['pickup_datetime'].dt.weekday
df['hour'] = df['pickup_datetime'].dt.hour
df['dayofweek'] = df['pickup_datetime'].dt.dayofweek
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_2.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
df.drop(['pickup_datetime'], axis=1, inplace=True )
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
df['dist_long'] = df['pickup_longitude'] - df['dropoff_longitude']
df['dist_lat'] = df['pickup_latitude'] - df['dropoff_latitude']
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 31<normalization>
df['dist'] = np.sqrt(np.square(df['dist_long'])+ np.square(df['dist_lat']))
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
def ft_haversine_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
df['distance'] = ft_haversine_distance(df['pickup_latitude'].values,
df['pickup_longitude'].values,
df['dropoff_latitude'].values,
df['dropoff_longitude'].values )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
df = pd.concat([df, pd.get_dummies(df['store_and_fwd_flag'],prefix = 'store')], axis=1)
 df.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 df = pd.concat([df, pd.get_dummies(df['vendor_id'],prefix = 'vendor')], axis=1)
 df.drop(['vendor_id'], axis=1, inplace=True )
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_log_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import LinearRegression
LR_START = 0.00001
LR_MAX = 0.0001 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 15
LR_SUSTAIN_EPOCHS = 3
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
df.drop(["id"], axis=1, inplace=True )
def get_model() :
base_model = efn.EfficientNetB4(weights='imagenet', include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(nb_classes, activation="softmax" )(x)
 return Model(inputs=base_model.input, outputs=predictions )<choose_model_class>
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]
with strategy.scope() :
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'] )<train_model>
target = train['trip_duration']
%%time
model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[lr_callback],
epochs=EPOCHS
)<predict_on_test>
X_train, X_val, y_train, y_val = train_test_split(new_train, target, test_size=0.2, shuffle=True )
%%time
probs = model.predict(test_dataset )<save_to_csv>
def rmsle_score(preds, true):
rmsle_score =(np.sum(( np.log1p(preds)-np.log1p(true)) **2)/len(true)) **0.5
return rmsle_score
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
RMSLE = make_scorer(rmsle_score )
import numpy as np
import pandas as pd
import os 
<install_modules>
import statsmodels.api as sm
sys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')
 ! pip install -e /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle<import_modules>
model = sm.OLS(target.values, new_train.astype(float))
print("Tensorflow version " + tf.__version__)
<init_hyperparams>
re = model.fit()
re.summary()
cfg = {"smoothing":0.00,
"arch_fn":efn,
"name": [f"EfficientNetB{i}" for i in [4,5,6]],
"num_class":5,
"epochs":20,
"kfold":[5,5,5],
"seed":42,
"verbose":1,
"resize":512,
"crop_size":480,
"rotation":0.0,
"shear":0.0,
"h-zoom":5.0,
"w-zoom":5.0,
"h-shift":5.0,
"w-shift":5.0,
"path_models":".. /input/cassavatpumodelsbaseline/"
}
AUTOTUNE = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
FILENAMES = tf.io.gfile.glob(".. /input/cassava-leaf-disease-classification" + '/test_tfrecords/ld_test*.tfrec')
 BATCH_SIZE = 128 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]<feature_engineering>
import lightgbm as lgbm
os.environ['PYTHONHASHSEED']=str(cfg["seed"])
 os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
random.seed(cfg["seed"])
 np.random.seed(cfg["seed"])
 tf.random.set_seed(cfg["seed"] )<categorify>
lgb_params = {
'metric' : 'rmse',
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000 }
def read_tfrecord(example, labeled, return_image_name=False):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"image_name": tf.io.FixedLenFeature([], tf.string)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 if return_image_name:
return image, tf.reshape(tf.one_hot([label], depth=cfg["num_class"], axis=-1), [-1]), example["image_name"]
return image, tf.reshape(tf.one_hot([label], depth=cfg["num_class"], axis=-1), [-1])
 idnum = example['image_name']
return image, idnum
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<create_dataframe>
lgb_df = lgbm.Dataset(new_train,target )
def load_dataset(filenames, labeled=True, ordered=False, return_image_name=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled, return_image_name=return_image_name), num_parallel_calls=AUTOTUNE)
 return dataset<create_dataframe>
lgb_model = lgbm.train(lgb_params, lgb_df, num_boost_round=1500 )
def get_test_dataset(filenames, ordered=True, tta=False):
dataset = load_dataset(filenames, labeled=False, ordered=ordered)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<normalization>
pred = lgb_model.predict(new_test )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["resize"]
ROT_ = cfg["rotation"]
SHR_ = cfg["shear"]
HZOOM_ = cfg["h-zoom"]
WZOOM_ = cfg["w-zoom"]
HSHIFT_ = cfg["h-shift"]
WSHIFT_ = cfg["w-shift"]
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<choose_model_class>
pred_lgb = np.exp(pred )
def get_model(cfg, name):
model_input = tf.keras.Input(shape=(cfg['resize'], cfg['resize'], 3), name='inputs')
 constructor = getattr(cfg["arch_fn"], name)
 x = constructor(include_top=False, weights=None,
input_shape=(cfg['resize'], cfg['resize'], 3),
pooling=None )(model_input)
 x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 x = tf.keras.layers.Dropout(0.25 )(x)
 outputs = tf.keras.layers.Dense(cfg["num_class"], activation='softmax', name="outputs" )(x)
 model = tf.keras.Model(model_input, outputs, name=name+'_{0}'.format(cfg["resize"]))
return model
def compile_new_model(cfg, name):
with strategy.scope() :
model = get_model(cfg, name)
 losses = tf.keras.losses.CategoricalCrossentropy(label_smoothing = cfg['smoothing'])
 model.compile(
 optimizer = tf.keras.optimizers.Adam(lr=1e-3),
loss = losses,
metrics = tf.keras.metrics.CategoricalAccuracy() 
)
 return model<load_from_csv>
import xgboost as xgb
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 tr = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv' )<create_dataframe>
params = {
'booster': 'gbtree',
'objective': 'reg:linear',
'learning_rate': 0.1,
'max_depth': 14,
'subsample': 0.8,
'colsample_bytree': 0.7,
'colsample_bylevel': 0.7,
'silent': 1
}
preds_all = []
for num_model in range(len(cfg["name"])) :
preds_model = []
for fold in range(cfg["kfold"][num_model]):
print(f"
ds_test = generator(JPEG_PATH,submission.image_id.values)
 K.clear_session()
with strategy.scope() :
model = compile_new_model(cfg, cfg['name'][num_model])
 print('Loading best model...')
 model.load_weights(cfg["path_models"]+ cfg['name'][num_model] + "/" + cfg['name'][num_model]+'-fold-%i.h5'%fold)
 preds = model.predict(ds_test, verbose=True)
 preds_model.append(preds)
 preds_model = np.stack(preds_model ).mean(0)
 preds_all.append(preds_model)
 preds_all = np.stack(preds_all )<prepare_x_and_y>
dtrain = xgb.DMatrix(new_train, target )
<save_to_csv>
gbm = xgb.train(params,
dtrain,
num_boost_round = 200 )
submission["label"] = preds_all.mean(0 ).argmax(1)
 submission.to_csv("submission.csv", index=False )<define_variables>
pred_test = np.exp(gbm.predict(xgb.DMatrix(new_test)) )
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext50-32x4d-starter-training/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<define_variables>
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )
class CFG:
debug=False
num_workers=4
model_name='resnext50_32x4d'
size=256
batch_size=32
seed=42
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
train=False
inference=True<load_from_csv>
sub = pd.DataFrame()
sub['id'] = test.id
sub['trip_duration'] = ensemble
sub.head(3 )
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test.head()<normalization>
sub.to_csv('submission.csv', index=False )
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['image_id'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image<normalization>
sub.to_csv('submission.csv', index=False )