Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<define_variables>
import numpy as np
import pandas as pd
import os
from tqdm import tqdm
GCS_DS_PATH = KaggleDatasets().get_gcs_path("tpu-getting-started" )<define_variables>
svr = pd.read_csv('/kaggle/input/rapids-svm-on-trends-neuroimaging/submission.csv')
 svr
IMAGE_SIZE = [331, 331]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SEED = 2020<data_type_conversions>
nn = pd.read_csv('/kaggle/input/trends-augmented-tabular/submission_10fold.csv')
 nn
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=True)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print('Dataset: {} validation images, {} unlabeled test images'.format(NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
submission = nn.append(svr ).groupby('Id' ).mean().reset_index()
submission
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
 return model
with strategy.scope() :
model1 = get_model(ResNet50)
 model1.load_weights("/kaggle/input/fork-of-start-with-pre-train-da6a79/my_ef_net_b7.h5" )<choose_model_class>
submission = submission.set_index('Id')
 submission
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
 return model
with strategy.scope() :
model2 = get_model(NASNetLarge)
 model2.load_weights("/kaggle/input/start-with-pre-train/my_ef_net_b7.h5" )<predict_on_test>
df_1590 = pd.read_csv("/kaggle/input/fork-of-fork-of-kernel19ee18bcfe/submission.csv" ).set_index('Id' )
test_ds = get_test_dataset(ordered=True)
 best_alpha = 0.55
print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities1 = model1.predict(test_images_ds)
 probabilities2 = model2.predict(test_images_ds)
 probabilities = best_alpha * probabilities1 +(1 - best_alpha)* probabilities2
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
df_1587 = pd.read_csv("/kaggle/input/trends-ensemble/submission_svr_nn.csv" ).set_index('Id')
 df_pri_1588 = pd.read_csv(".. /input/trends-priyanshu-meta/submission.csv" ).set_index('Id' )
!pip install -q efficientnet<set_options>
submission =.5*submission +.5*df_1590
submission
%matplotlib inline
print("Tensorflow version " + tf.__version__ )<define_variables>
submission =.4*submission +.6*df_1587
submission =.6*submission +.4*df_pri_1588
submission
GCS_DS_PATH = KaggleDatasets().get_gcs_path("tpu-getting-started" )<define_variables>
submission['temp'] = submission.index
age = [ val for val in submission.temp.values if 'age' in val]
counter = 0
for index, row in tqdm(submission.iterrows() , total = submission.shape[0]):
counter +=1
if index in age:
submission.loc[submission.temp == row.temp, 'Predicted'] = row.Predicted * 1.01
IMAGE_SIZE = [331, 331]
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SEED = 2020<data_type_conversions>
submission = submission.drop(['temp'], axis=1)
 submission.head()
<choose_model_class><EOS> 
submission.to_csv("submission_svr_nn.csv" )