Kaggle competition: Digit Recognizer
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset<feature_engineering>
data_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
HPA_MODELS = False
COMPETITION_NAME = "hpa-single-cell-image-classification"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE =(224, 240, 260, 300, 380, 456, 528, 600, 720)
 load_dir = f"/kaggle/input/{COMPETITION_NAME}/"
sub_df = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv')
 sub_df = sub_df.drop(sub_df.columns[1:],axis=1)
 for i in range(19):
sub_df[f'{i}'] = pd.Series(np.zeros(sub_df.shape[0]))
if HPA_MODELS:
colours = ['blue', 'green', 'red', 'yellow']
sub_dfs = []
for colour in colours:
test_paths = load_dir + "/test/" + sub_df['ID'] + '_' + colour + '.png'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[-1], IMSIZE[-1]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model(
 f'.. /input/hpa-models/effb7_individual_model_{colour}.h5' 
)
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 sub_dfs.append(sub_df)
 if colour == 'yellow':
sub_df = pd.concat(( sub_df.ID,(
 sub_dfs[0].iloc[:, 1:] + sub_dfs[1].iloc[:, 1:] + sub_dfs[2].iloc[:, 1:] + sub_dfs[3].iloc[:, 1:] 
)/4), axis=1)
 else:
test_paths = load_dir + "/test/" + sub_df['ID'] + '_' + 'green' + '.png'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[-1], IMSIZE[-1]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model('.. /input/hpa-models/HPA classification efnb7 train 13cc0d 20/model_green.h5')
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 ss_df = pd.merge(ss_df, sub_df, on = 'ID', how = 'left')
 for row in range(ss_df.shape[0]):
pred = ss_df.loc[row,'PredictionString']
pred_split = pred.split()
for j in range(int(len(pred_split)/3)) :
for k in range(19):
if int(pred_split[ 3*j ])== k:
p = pred_split[ 3*j + 1 ]
pred_split[ 3*j + 1 ] = str(ss_df.loc[row, f'{k}']*0.6 + float(p)*0.4)
 ss_df.loc[row,'PredictionString'] = ' '.join(pred_split)
 ss_df = ss_df[['ID','ImageWidth','ImageHeight','PredictionString']]
ss_df.to_csv('submission.csv',index = False)
 print(f'HPA_MODELS = {HPA_MODELS}' )<install_modules>
X, y = data_train.drop(labels = ["label"],axis = 1)/255.,data_train["label"]
X_test = X_test/255 .
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps<install_modules>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.00001)
 early_stopping = EarlyStopping(
 min_delta=0.000001,
patience=20,
restore_best_weights=True,
)
print(" 
...INSTALLING AND IMPORTING CELL-PROFILER TOOL(HPACELLSEG )...
 ")
 try:
except:
!pip install -q "/kaggle/input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl"
!pip install -q "/kaggle/input/hpapytorchzoozip/pytorch_zoo-master"
!pip install -q "/kaggle/input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master"
print(" 
...OTHER IMPORTS STARTING...
 ")
 print("
\tVERSION INFORMATION")
 LBL_NAMES = ["Nucleoplasm", "Nuclear Membrane", "Nucleoli", "Nucleoli Fibrillar Center", "Nuclear Speckles", "Nuclear Bodies", "Endoplasmic Reticulum", "Golgi Apparatus", "Intermediate Filaments", "Actin Filaments", "Microtubules", "Mitotic Spindle", "Centrosome", "Plasma Membrane", "Mitochondria", "Aggresome", "Cytosol", "Vesicles", "Negative"]
INT_2_STR = {x:LBL_NAMES[x] for x in np.arange(19)}
INT_2_STR_LOWER = {k:v.lower().replace(" ", "_")for k,v in INT_2_STR.items() }
STR_2_INT_LOWER = {v:k for k,v in INT_2_STR_LOWER.items() }
STR_2_INT = {v:k for k,v in INT_2_STR.items() }
FIG_FONT = dict(family="Helvetica, Arial", size=14, color="
LABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette("Spectral", len(LBL_NAMES)) ]
LABEL_COL_MAP = {str(i):x for i,x in enumerate(LABEL_COLORS)}
print(" 
...IMPORTS COMPLETE...
 ")
 ONLY_PUBLIC = True
if ONLY_PUBLIC:
print(" 
...ONLY INFERRING ON PUBLIC TEST DATA(USING PRE-PROCESSED DF )...
 ")
 else:
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "...Physical GPUs,", len(logical_gpus), "Logical GPUs...
 ")
 except RuntimeError as e:
print(e )<install_modules>
skf = StratifiedKFold(n_splits=3,random_state=42,shuffle=True)
 sub = pd.DataFrame(data=None, index=(range(1,28001)) , columns=None, dtype=None, copy=False)
 for train_index, val_index in skf.split(X, y):
model = keras.Sequential([
keras.layers.Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)) ,
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Flatten() ,
keras.layers.Dense(28*28, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Dense(28*28, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Dense(28*28, activation='relu'),
keras.layers.Dense(10, activation='softmax')
 ])
 X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 history = model.fit(
 x=X_train, y=y_train, batch_size=100, epochs=250, verbose=1, callbacks=[early_stopping,learning_rate_reduction],
validation_data=(X_val,y_val), shuffle=True)
 results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 sub = pd.concat([sub, results],axis=1)
!cp -r.. /input/focallosstensorflowstablefromartemmavrin/focal-loss-master/*./
!pip install./focal-loss-master/<choose_model_class>
sub["result"] = sub.mode(dropna=True,axis=1)[0]
result = pd.Series(sub["result"],name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),result],axis = 1)
 submission = submission.dropna().astype('int32')
 submission.to_csv("mnist_ansamble_of_cnn.csv",index=False )