Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<concatenate>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
all_data_sprs = scipy.sparse.hstack([scipy.sparse.csr_matrix(all_data),
encoded_nom_matrix,
encoded_date_matrix],
format='csr' )<prepare_x_and_y>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from fastai.vision import *
from PIL import Image
num_train = train.shape[0]
X_train = all_data_sprs[:num_train]
X_test = all_data_sprs[num_train:]
y_train = train['target']<train_model>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
clf = LogisticRegression(C=0.125, solver='lbfgs', max_iter=800, verbose=0, n_jobs=-1)
 clf.fit(X_train, y_train )<save_to_csv>
PATH.ls()
y_preds = clf.predict_proba(X_test)[:,1]
submission['target'] = y_preds
submission.to_csv('submission.csv' )<install_modules>
train_img = train.iloc[:,1:785]
test_img = test.iloc[:,:]
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
def save_img(data,fpath,isTest=False):
if isTest == False:
for index, row in data.iterrows() :
label,digit = row[0], row[1:]
filepath = fpath
filename = "train_{}.jpg".format(index)
 digit = digit.values
digit = digit.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath/filename)
 else:
for index, row in data.iterrows() :
digit = row[:]
filepath = fpath
filename = "test_{}.jpg".format(index)
 digit = digit.values
digit = digit.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath/filename )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
save_img(train,TRAIN,False )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
sorted(os.listdir(TRAIN)) [0]
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 5<load_from_csv>
save_img(test,TEST,True )
database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'
submission = pd.read_csv(f'{database_base_path}sample_submission.csv')
 display(submission.head())
 TEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print(f'GCS: test: {NUM_TEST_IMAGES}' )<train_model>
sorted(os.listdir(TEST)) [0]
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-tpu-tensorflow-training/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<train_model>
train['filename'] = ['train/train_{}.jpg'.format(x)for x,_ in train.iterrows() ]
model_path_list_2 = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list_2.sort()
print('Models to predict:')
 print(*model_path_list_2, sep='
' )<choose_model_class>
test['filename'] = ['test/test_{}.jpg'.format(x)for x,_ in test.iterrows() ]
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='inputs')
 base_model = efn.EfficientNetB3(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 model = tf.keras.Sequential([
base_model,
L.Dropout (.25),
L.Dense(N_CLASSES, activation='softmax', name='output')
 ])
 return model
def model_fn_2(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='inputs')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model_2 = model_fn_2(( None, None, CHANNELS), N_CLASSES)
 model.summary() 
<predict_on_test>
img = open_image(TRAIN/'train_0.jpg')
 img
files_path = f'{database_base_path}test_images/'
test_preds = np.zeros(( len(os.listdir(files_path)) , N_CLASSES))
print('First model')
 for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True)
 for step in range(TTA_STEPS):
print(f'TTA step {step+1}/{TTA_STEPS}')
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/(TTA_STEPS * len(model_path_list))
else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 print('
Second model')
 for model_path in model_path_list_2:
print(model_path)
 K.clear_session()
model_2.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True)
 for step in range(TTA_STEPS):
print(f'TTA step {step+1}/{TTA_STEPS}')
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model_2.predict(x_test)/(TTA_STEPS * len(model_path_list))
else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model_2.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_ds.unbatch())]<save_to_csv>
tfms =([*rand_pad(padding=3,size=28,mode='reflection'),zoom(scale=1.005),],[] )
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<define_variables>
src = ImageList.from_df(train,PATH,cols='filename' ).split_by_rand_pct(0.2 ).label_from_df(cols='label' ).add_test_folder(PATH/'test' )
sys.path.append('.. /input/timm-pytorch-image-models/pytorch-image-models-master')
 TESTSAMPLEPATH = '.. /input/cassava-leaf-disease-classification/sample_submission.csv'
CLASSESJSONPATH = '.. /input/cassava-leaf-disease-classification/label_num_to_disease_map.json'
TESTDATAPATH = '.. /input/cassava-leaf-disease-classification/test_images'
TRAINDATAPATH = '.. /input/cassava-leaf-disease-classification/train_images'
cfg = {}
cfg['image_size'] = 512
cfg['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 cfg['batch_size'] = 16
classes = {}
num_class = 0
with open(CLASSESJSONPATH)as f:
classes = json.load(f)
 num_class = len(classes)
 def readImage(ID, path):
ID = ID.split('.')[0]
filepath = os.path.join(path, ID + '.jpg')
 img = cv2.imread(filepath)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 return img<categorify>
data = src.transform(tfms, size=28 ).databunch().normalize(imagenet_stats )
class CLCdataset(Dataset):
def __init__(self, df,isTrain = True):
self.df = df
self.isTrain = isTrain
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
ID = self.df['image_id'].iloc[idx].split('.')[0]
img = readImage(ID, TRAINDATAPATH if self.isTrain else TESTDATAPATH)
 if self.isTrain:
img = train_transform(image = img)['image']
else:
img = test_transform(image = img)['image']
if self.isTrain:
label = self.df['label'].iloc[idx]
return ID, img, label
return ID, img 
<normalization>
data.show_batch(2,2 )
test_transform = A.Compose([
A.Resize(cfg['image_size'], cfg['image_size'],p=1),
A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2()
] )<choose_model_class>
data.classes,data.c
class CFCmodel(nn.Module):
def __init__(self, out_dim):
super(CFCmodel, self ).__init__()
self.model = timm.create_model('efficientnet_b3', pretrained=False)
 self.model.classifier = nn.Linear(in_features=1536,out_features=out_dim, bias=True)
 self.model.eval()
def forward(self, input):
return self.model(input )<install_modules>
learn = cnn_learner(data,models.resnet50,metrics=accuracy,model_dir='/kaggle/working/' )
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
learn.lr_find(num_it=600 )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 1234
seed_everything(seed)
 warnings.filterwarnings('ignore' )<define_variables>
learn.recorder.plot(skip_end=25 )
BATCH_SIZE = 32 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5

 TTA_STEPS = 10<define_search_space>
lr=4e-3
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label<normalization>
learn.fit_one_cycle(10,slice(lr))
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
return image
def center_crop(image):
image = tf.reshape(image, [600, 800, CHANNELS])
 h, w = image.shape[0], image.shape[1]
if h > w:
image = tf.image.crop_to_bounding_box(image,(h - w)// 2, 0, w, w)
 else:
image = tf.image.crop_to_bounding_box(image, 0,(w - h)// 2, h, h)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<train_model>
learn.save('stage1-resnet50-mnist' )
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<choose_model_class>
learn.load('stage1-resnet50-mnist' )
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout(0.4 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='tanh', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary()<define_variables>
learn.lr_find(num_it=600 )
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
learn.fit_one_cycle(10,slice(1e-5,1e-4/5))
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<load_from_zip>
learn.save('stage2-resnet50-mnist' )
!git clone --depth 1 https://github.com/location-competition/indoor-location-competition-20 indoor_location_competition_20
!rm -rf indoor_location_competition_20/data<import_modules>
PATH.ls()
import multiprocessing
import numpy as np
import pandas as pd
import scipy.interpolate
import scipy.sparse
from tqdm import tqdm
from indoor_location_competition_20.io_f import read_data_file
import indoor_location_competition_20.compute_f as compute_f<define_variables>
interp = ClassificationInterpretation.from_learner(learn )
INPUT_PATH = '.. /input/indoor-location-navigation'<feature_engineering>
test = ImageList.from_folder(TEST)
 test
def compute_rel_positions(acce_datas, ahrs_datas):
step_timestamps, step_indexs, step_acce_max_mins = compute_f.compute_steps(acce_datas)
 headings = compute_f.compute_headings(ahrs_datas)
 stride_lengths = compute_f.compute_stride_length(step_acce_max_mins)
 step_headings = compute_f.compute_step_heading(step_timestamps, headings)
 rel_positions = compute_f.compute_rel_positions(stride_lengths, step_headings)
 return rel_positions<load_from_csv>
learn = load_learner(PATH, test=test )
def correct_path(args):
path, path_df = args
T_ref = path_df['timestamp'].values
xy_hat = path_df[['x', 'y']].values
example = read_data_file(f'{INPUT_PATH}/test/{path}.txt')
 rel_positions = compute_rel_positions(example.acce, example.ahrs)
 if T_ref[-1] > rel_positions[-1, 0]:
rel_positions = [np.array([[0, 0, 0]]), rel_positions, np.array([[T_ref[-1], 0, 0]])]
else:
rel_positions = [np.array([[0, 0, 0]]), rel_positions]
rel_positions = np.concatenate(rel_positions)
 T_rel = rel_positions[:, 0]
delta_xy_hat = np.diff(scipy.interpolate.interp1d(T_rel, np.cumsum(rel_positions[:, 1:3], axis=0), axis=0 )(T_ref), axis=0)
 N = xy_hat.shape[0]
delta_t = np.diff(T_ref)
 alpha =(8.1)**(-2)* np.ones(N)
 beta =(0.3 + 0.3 * 1e-3 * delta_t)**(-2)
 A = scipy.sparse.spdiags(alpha, [0], N, N)
 B = scipy.sparse.spdiags(beta, [0], N-1, N-1)
 D = scipy.sparse.spdiags(np.stack([-np.ones(N), np.ones(N)]), [0, 1], N-1, N)
 Q = A +(D.T @ B @ D)
 c =(A @ xy_hat)+(D.T @(B @ delta_xy_hat))
xy_star = scipy.sparse.linalg.spsolve(Q, c)
 return pd.DataFrame({
'site_path_timestamp' : path_df['site_path_timestamp'],
'floor' : path_df['floor'],
'x' : xy_star[:, 0],
'y' : xy_star[:, 1],
} )<save_to_csv>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
sub = pd.read_csv('.. /input/simple-99-accurate-floor-model/submission.csv')
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp')
 sub.to_csv('submission.csv', index=False )<define_variables>
probabilities = preds[0].tolist()
[f"{index}: {probabilities[index]}" for index in range(len(probabilities)) ]
EPOCH = 50
BATCH_SIZE = 256
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 MEMORY_LENGTH = 5
WIFI_NUM = 50
FOLDS = 10
inpath = '.. /input/indoor-location-navigation/'
metapath = inpath + 'metadata'
trainpath = inpath + 'train'
testpath = inpath + 'test'
def set_seed(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
set_seed()<load_from_csv>
class_score = np.argmax(preds, axis=1 )
sample_submission = pd.read_csv('.. /input/indoor-location-navigation/sample_submission.csv' )<load_from_csv>
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 sample_submission.head()
floors = pd.read_csv('/kaggle/input/indoor-xy-floor/result_floor_feb22.csv',index_col=0)
 floors.head()<load_from_csv>
ImageId = [os.path.splitext(p)[0].split('test_')[1] for p in os.listdir(TEST)]
ImageId = [int(path)for path in ImageId]
ImageId = [ID+1 for ID in ImageId]
train_data = pd.read_csv('.. /input/time-series-unified-wifi/train_all.csv',index_col=0)
 test_data = pd.read_csv('.. /input/time-series-unified-wifi/test_all.csv',index_col=0)
 train_data<feature_engineering>
sorted(ImageId)[-1]
del_path = []
new_trains = []
for i,(p, g)in enumerate(train_data.groupby('path_id')) :
if len(g)> MEMORY_LENGTH:
del_path.append(p)
 for j in range(( len(g)// MEMORY_LENGTH)+ 1):
if j ==(len(g)// MEMORY_LENGTH):
tmp = g.iloc[j*MEMORY_LENGTH:]
else:
tmp = g.iloc[j*MEMORY_LENGTH:(j+1)*MEMORY_LENGTH]
tmp.loc[:, 'path_id'] = p + '_' + str(j)
 new_trains.append(tmp)
 train_data.drop(train_data[train_data['path_id'].isin(del_path)].index, inplace=True)
 train_data = pd.concat([train_data, pd.concat(new_trains)] ).reset_index(drop=True)
 print(f"previous path len:{i+1}, current path len:{len(train_data.groupby('path_id')) }")
 del new_trains
gc.collect()<prepare_x_and_y>
submission = pd.DataFrame({
"ImageId": ImageId,
"Label": class_score
})
 submission.to_csv("nona-submission.csv", index=False )