Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<load_from_csv>
!pip install efficientnet
print("Tensorflow version " + tf.__version__ )
def init() :
test_df = pd.read_csv(".. /input/titanic/test.csv")
 train_df = pd.read_csv(".. /input/titanic/train.csv")
 test_df["Survived"]=-8888
Main_df=pd.concat(( train_df, test_df), axis=0, sort=False)
 print("Columns with null values {}".format(Main_df.columns[Main_df.isnull().any() ]))
global title_map
title_map={
"mr": "Mr","mrs": "Mrs","miss": "Miss","master": "Master","don": "Royal", "rev": "Officer", "dr": "Officer",
"mme": "Mrs", "dona": "Royal", "jonkheer" : "Royal", "the countess": "Royal", "capt": "Officer", "col": "Officer",
"ms": "Mrs", "mlle": "Miss", "major": "Officer", "lady" : "Royal", "sir" : "Royal"
}
return(Main_df )<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("TPU REPLICAS: ", strategy.num_replicas_in_sync )
def get_title(x):
x=x.split(",")[1].split(".")[0].strip().lower()
return(title_map[x])
 def is_in_group(df):
gdf=pd.DataFrame({"GroupSize": df.groupby("Ticket")["Ticket"].count() } ).reset_index()
df=pd.merge(df, gdf, how="left", on="Ticket" ).set_index(df.index)
 df["Fare"].fillna(8.0, inplace=True)
 df["ActualFare"]=df["Fare"].astype(int)/df["GroupSize"]
df["IsMother"]= np.where(( df["Sex"]=="female")&(df["Parch"] > 0)&(df["Title"] != "Miss"), 1, 0)
 df['ST_third_class'] = np.where(( df['Ticket'].str.match(pat='(SOTON)|(W./C.) |(A/5)')) &(df["Pclass"]==3)&(df["Cabin"].isnull()), 1, 0)
 df["PC"]= np.where(( df["Pclass"]==1)&(df['Ticket'].str.match(pat='(PC)')) &(df["Cabin"].isnull()), 1, 0)
 df['ST_WC'] = np.where(( df['Ticket'].str.match(pat='(W./C.) ')) &(df["Cabin"].isnull()), 1, 0)
 return(df)
 def adjust_columns(df):
columns=["Title", "Sex","Pclass", "Embarked"]
df=pd.get_dummies(df, columns=columns)
 df.drop(["Name", "Ticket", "Cabin", "Fare", "Cabin"], axis=1, inplace=True)
 df.drop(["Title_Royal"], axis=1, inplace=True)
 return(df)
 def age(df):
cond = df["IsMother"]==1
median = df.loc[cond, "Age"].median()
df.loc[cond, "Age"]=df.loc[cond, "Age"].fillna(median)
 cond=df["Parch"]==0
median = df.loc[cond, "Age"].median()
df.loc[cond, "Age"]=df.loc[cond, "Age"].fillna(median)
 df["Age"].fillna(df.groupby(['Sex','Title'])["Age"].transform("median"), inplace=True)
 fig,(axis1)= plt.subplots(1,1, figsize=(10,5))
sn.distplot(df["Age"], ax=axis1)
 return(df )<set_options>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings("ignore" )<prepare_x_and_y>
IMAGE_SIZE = [224, 224]
EPOCHS = 10
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
train=df.loc[df["Survived"]!=-8888]
test=df.loc[df["Survived"]==-8888]
cols=list(train.columns)
 cols.remove("Survived")
 cols.remove("PassengerId")
 print(cols)
 X_train=train.drop(["Survived","PassengerId"], axis=1 ).values
Y_train=train["Survived"].values
X_test=test.drop(["Survived","PassengerId"], axis=1 ).values
print(df.shape, X_train.shape, Y_train.shape, X_test.shape )<define_search_model>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def grid_search_param(algo):
if algo == "Logistic":
clf=LogisticRegression()
c_space = np.logspace(-5, 8, 15)
 penalty=["l1", "l2"]
param_grid = {'C': c_space, 'penalty' : penalty}
elif algo=="RandomForest":
clf=RandomForestClassifier()
param_grid = { 'n_estimators': [700],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [6],
'criterion' :['gini', 'entropy'],
}
elif algo=="KNN":
clf=KNeighborsClassifier()
param_grid ={'n_neighbors' : [6,7,8,9,10,11,12,14,16,18,20,22],
'algorithm' : ['auto'],
'weights' : ['uniform', 'distance'],
'leaf_size' : list(range(1,50,5))
}
return(param_grid, clf)
 def get_best_param(clf, param):
grid = GridSearchCV(clf, param, cv = 10)
 grid.fit(X_train, Y_train)
 print("Tuned Parameters: {}".format(grid.best_params_))
print("Best score is {}".format(grid.best_score_))
return(grid)
 def run_clf(grid, clf, algo):
if algo == "Logistic":
c_value=grid.best_params_["C"]
penalty=grid.best_params_["penalty"]
print("Best parameter: C is {} and Penalty is {}".format(c_value, penalty))
clf=LogisticRegression(C=c_value, penalty=penalty)
 elif algo=="RandomForest":
n_estimators=grid.best_params_["n_estimators"]
max_features=grid.best_params_["max_features"]
max_depth=grid.best_params_["max_depth"]
criterion=grid.best_params_["criterion"]
clf=RandomForestClassifier(n_estimators=n_estimators, max_features=max_features, max_depth=max_depth, criterion=criterion)
 elif algo=="KNN":
n_neighbors=grid.best_params_["n_neighbors"]
algorithm=grid.best_params_["algorithm"]
weights=grid.best_params_["weights"]
leaf_size=grid.best_params_["leaf_size"]
clf=KNeighborsClassifier(n_neighbors=n_neighbors, algorithm=algorithm,weights=weights,leaf_size=leaf_size)
 clf.fit(X_train, Y_train)
 print(confusion_matrix(Y_train, clf.predict(X_train)))
 print("{} score is {}".format(algo, clf.score(X_train,Y_train)))
 return(clf)
<save_to_csv>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
print("Running Logistic regression")
 param, clf=grid_search_param("Logistic")
(grid)= get_best_param(clf, param)
 clfLR=run_clf(grid, clf, "Logistic")
 pred_values=clfLR.predict(X_test)
 test["Survived"]=pd.Series(pred_values, index=test.index)
 test[["PassengerId", "Survived"]].to_csv("LRPrediction.csv", index=False)
 print("Running Random Forest")
 param, clf=grid_search_param("RandomForest")
(grid)= get_best_param(clf, param)
 clfRF=run_clf(grid, clf, "RandomForest")
 importances = clfRF.feature_importances_
DF=pd.DataFrame({"Features": cols, "Importance": importances})
 print(DF.sort_values(by ='Importance', ascending = False))
pred_values=clfRF.predict(X_test)
 test["Survived"]=pd.Series(pred_values, index=test.index)
 test[["PassengerId", "Survived"]].to_csv("RFPrediction.csv", index=False)
<install_modules>
def load_dataset(filenames, labeled=True, ordered=False, is_train=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/train/*.tfrec'), labeled=True, is_train=True)
 dataset = dataset.map(transform)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/val/*.tfrec'), labeled=True, ordered=False, is_train=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/test/*.tfrec'), labeled=False, ordered=ordered, is_train=False)
 dataset = dataset.batch(BATCH_SIZE)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()
!pip install h2o<set_options>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
learning_rate_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
%matplotlib inline 
<train_model>
with strategy.scope() :
effb3_model = efn.EfficientNetB3(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb3_model.trainable = True
model1 = tf.keras.Sequential([
effb3_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model1.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical1 = model1.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset )
h2o.init()<load_from_csv>
with strategy.scope() :
effb4_model = efn.EfficientNetB4(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb4_model.trainable = True
model2 = tf.keras.Sequential([
effb4_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model2.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical2 = model2.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset)
train = h2o.import_file('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = h2o.import_file('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train.head()<feature_engineering>
with strategy.scope() :
effb5_model = efn.EfficientNetB5(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb5_model.trainable = True
model3 = tf.keras.Sequential([
effb5_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model3.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical3 = model3.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset )
train = train[train['GrLivArea'] < 4500]
train['SalePrice'] = train['SalePrice'].log1p()<prepare_x_and_y>
with strategy.scope() :
effb6_model = efn.EfficientNetB6(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb6_model.trainable = True
model4 = tf.keras.Sequential([
effb6_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model4.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical4 = model4.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset )
<train_model><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities1 = model1.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 probabilities3 = model3.predict(test_images_ds)
 probabilities4 = model4.predict(test_images_ds)
 probabilities =(probabilities1 + probabilities2 + probabilities3 + probabilities4)/ 4
predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 print('Done !' )