Kaggle competition: RANZCR CLiP - Catheter and Line Position Challenge
<SOS> metric: MCAUC Kaggle data source: ranzcr-clip-catheter-and-line-position-challenge<define_variables>
%env SM_FRAMEWORK=tf.keras
!pip install.. /input/segmentation-models-keras/Keras_Applications-1.0.8-py3-none-any.whl --quiet
!pip install.. /input/segmentation-models-keras/image_classifiers-1.0.0-py3-none-any.whl --quiet
!pip install.. /input/segmentation-models-keras/efficientnet-1.0.0-py3-none-any.whl --quiet
!pip install.. /input/segmentation-models-keras/segmentation_models-1.0.1-py3-none-any.whl --quiet
print("Segmentation Models installed." )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}fasttext-crawl-300d-2m/crawl-300d-2M.vec'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'<define_variables>
DEBUG = False
embed_size = 300
max_features = 20000
maxlen = 100<load_from_csv>
print(tf.__version__ )
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values<string_transform>
data_dir = '.. /input/ranzcr-clip-catheter-line-classification'
model_dir = '.. /input/ranzcr-1st-place-solution-by-tf-models'
seg_image_size = 1024
cls_image_size = 512
batch_size = 16
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<compute_test_metric>
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))<feature_engineering>
seg_model_names = [
'seg_model_V10_0.hdf5'
]
cls_model_names = [
'cls_model_V14_0.hdf5',
'cls_model_V15_1.hdf5',
'cls_model_V15_2.hdf5',
'cls_model_V16_3.hdf5',
'cls_model_V16_4.hdf5'
]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
tfrec_path = data_dir + '/test_tfrecords/*.tfrec'
tfrec_file_names = sorted(tf.io.gfile.glob(tfrec_path))
tfrec_file_names = \
[ tfrec_file_names[0] ] if DEBUG else tfrec_file_names
tfrec_file_names
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
AUTOTUNE = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 return image
def read_tfrecord(example):
TFREC_FORMAT = {
'image': tf.io.FixedLenFeature([], tf.string),
'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, TFREC_FORMAT)
 image = decode_image(example['image'])
 study_inst_id = example['StudyInstanceUID']
return image, study_inst_id
def load_dataset(filenames):
ds = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 ds = ds.map(read_tfrecord, num_parallel_calls=AUTOTUNE)
 return ds
model.fit(X_t, y, batch_size=32, epochs=2, validation_split=0.1);<save_to_csv>
raw_test_ds = load_dataset(tfrec_file_names)
 raw_test_ds
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('submission.csv', index=False )<import_modules>
study_inst_id_list = [
study_inst_id.numpy().decode('utf-8')for image, study_inst_id in raw_test_ds
]
print(study_inst_id_list[ :10 ])
 print(study_inst_id_list[ -10: ] )
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<load_from_csv>
def drop_study_inst_id(image, study_inst_id):
return image
def preprocess_image(image):
image_seg = tf.image.resize(image,(seg_image_size, seg_image_size))
image_seg = image_seg / 255.0
image_cls = tf.image.resize(image,(cls_image_size, cls_image_size))
return(( image_seg, image_cls),)
 def make_test_dataset() :
ds = load_dataset(tfrec_file_names)
 ds = ds.map(drop_study_inst_id, num_parallel_calls=AUTOTUNE)
 ds = ds.map(preprocess_image, num_parallel_calls=AUTOTUNE)
 ds = ds.batch(batch_size)
 ds = ds.prefetch(AUTOTUNE)
 return ds
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )<define_variables>
test_ds = make_test_dataset()
test_ds
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
data_folder = ".. /input/jigsaw-toxic-comment-classification-challenge/"
pretrained_folder = ".. /input/"
train_filepath = data_folder + "train.csv"
test_filepath = data_folder + "test.csv"
submission_path = data_folder + "submission.csv"
facebook_filepath = pretrained_folder + "fasttext-crawl-300d-2m/crawl-300d-2M.vec"
google_filepath = pretrained_folder + "glove840b300dtxt/glove.840B.300d.txt"
embedding_dim_facebook = 300
embedding_dim_google = 300<load_pretrained>
def load_model(weight_file_name):
weight_file_path = os.path.join(model_dir, weight_file_name)
 model = tf.keras.models.load_model(weight_file_path)
 return model
def make_seg_masks(x):
fold_seg_masks = tf.stack(x, axis=0)
 average_seg_masks = \
tf.math.reduce_mean(fold_seg_masks, axis=0)
 return average_seg_masks
def make_cls_inputs(x):
cls_images = x[0]
seg_masks = x[1]
seg_masks = tf.image.resize(
 seg_masks,(cls_image_size, cls_image_size))
seg_masks = seg_masks * 255.0
cls_inputs = tf.concat([cls_images, seg_masks], axis=-1)
 return cls_inputs
def make_model(cls_model_name):
seg_images = tf.keras.Input(
 shape=(seg_image_size, seg_image_size, 3),
name="seg_images")
 seg_outputs = []
for seg_model_name in seg_model_names:
seg_model = load_model(seg_model_name)
 seg_output = seg_model(seg_images)
 seg_outputs.append(seg_output)
 seg_masks = L.Lambda(
 make_seg_masks, name="seg_masks" )(seg_outputs)
 cls_images = tf.keras.Input(
 shape=(cls_image_size, cls_image_size, 3),
name="cls_images")
 cls_inputs = L.Lambda(
 make_cls_inputs, name="cls_inputs" )(
 [cls_images, seg_masks])
 cls_model = load_model(cls_model_name)
 ett, others, pred = cls_model(cls_inputs)
 model = tf.keras.Model(
 inputs=[seg_images, cls_images],
outputs=pred,
name="infer_model")
 return model
hyphens_filepath = ".. /input/cleaning-dictionaries/hyphens_dictionary.bin"
misspellings_filepath = ".. /input/cleaning-dictionaries/misspellings_all_dictionary.bin"
merged_filepath = ".. /input/cleaning-dictionaries/merged_all_dictionary.bin"
hyphens_dict = misspellings_dict = merged_dict = {}
with open(hyphens_filepath, mode='rb')as file: hyphens_dict = pickle.load(file)
 with open(misspellings_filepath, mode='rb')as file: misspellings_dict = pickle.load(file)
 with open(merged_filepath, mode='rb')as file: merged_dict = pickle.load(file)
 print(len(hyphens_dict))
print(len(misspellings_dict))
print(len(merged_dict))<define_variables>
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}<define_variables>
df_sub['StudyInstanceUID'] = study_inst_id_list
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
target_cols = [
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result 
<load_from_csv>
df_subs = [df_sub.copy() for _ in range(PROBS.shape[0])]
for i, this_sub in enumerate(df_subs):
this_sub[target_cols] = PROBS[i]
this_sub[target_cols] = \
this_sub[target_cols].rank(pct=True )
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
rank_values = \
[this_sub[target_cols].values for this_sub in df_subs]
df_sub[target_cols] = \
np.stack(rank_values, 0 ).mean(0 )
<string_transform><EOS> 
df_sub.to_csv('submission.csv', index=False)
 !head submission.csv