Kaggle competition: Quora Insincere Questions Classification
results = model.predict(testv)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
<save_to_csv>
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Dense, Input, Embedding, Dropout, Activation, Conv1D
from tensorflow.keras.layers import GRU,LSTM
from tensorflow.keras.layers import Bidirectional, GlobalMaxPooling1D, MaxPool1D
from tensorflow.keras.models import Model , Sequential
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.callbacks import EarlyStopping
sample_submission = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 sample_submission["Label"]=results
sample_submission.to_csv("submission.csv", index=False )<import_modules>
df_train=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv' )
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm
import cv2 as cv
import random
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
import tensorflow.keras
from tensorflow.keras.models import Sequential, load_model, Model
from tensorflow.keras import layers
from tensorflow.keras.optimizers import Adam, SGD, RMSprop
from tensorflow.keras.metrics import Precision, Recall, AUC
from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping 
<load_from_csv>
df_test=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv' )
train_df = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_df = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_df.head()<prepare_x_and_y>
from zipfile import ZipFile
%%time
samples, columns = train_df.shape
X = np.zeros(( samples,28,28,1))
y_true = np.zeros(( samples,1))
for sample in tqdm(range(samples)) :
X[sample,:,:,:] = train_df.iloc[sample,1:columns].values.reshape(28,28,1 ).astype('float32')
 y_true[sample,0] = train_df.iloc[sample,0]<categorify>
dim=300
embeddings1_index={}
with zipfile.ZipFile(".. /input/quora-insincere-questions-classification/embeddings.zip")as zf:
with io.TextIOWrapper(zf.open("glove.840B.300d/glove.840B.300d.txt"), encoding="utf-8")as f:
for line in tqdm(f):
values=line.split(' ')
 word=values[0]
vectors=np.asarray(values[1:],'float32')
 embeddings1_index[word]=vectors
values = train_df.label
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
 print('The original output labels', values)
 onehot_encoder = OneHotEncoder(sparse=False)
 integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
 onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
 samples, classes = onehot_encoded.shape
print("Number of vectors:", samples, "
Number of neurons / length of vector:", classes )<prepare_x_and_y>
print('Found %s word vectors.' % len(embeddings1_index))
y = onehot_encoded<normalization>
del zipfile
%%time
def standard_norm(img):
return(img - np.mean(img)) /np.std(img)
 norm_X = np.zeros(( samples,28,28,1))
for sample in tqdm(range(samples)) :
norm_X[sample,:,:,:] = standard_norm(X[sample,:,:,:] ).reshape(28,28,1)
<choose_model_class>
gc.collect()
def METRICS() :
metrics = ['accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='AUC')]
return metrics
model = Sequential()
model.add(layers.Input(shape=(28, 28, 1)))
 model.add(layers.Conv2D(32,(3,3), padding = 'same', activation='relu'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(pool_size =(2, 2)))
 model.add(layers.Conv2D(64,(3,3), padding = 'same', activation='relu'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(pool_size =(2, 2)))
 model.add(layers.Conv2D(128,(3,3), padding = 'same', activation='relu'))
model.add(layers.BatchNormalization())
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dense(classes,activation='softmax', name = 'output_layer'))
model.compile(Adam(lr = 0.00100005134), metrics= METRICS() , loss = 'categorical_crossentropy')
 model.summary()<train_model>
def build_vocab(sentences,verbose=True):
vocab={}
for sentence in tqdm(sentences,disable=(not verbose)) :
for word in sentence:
try:
vocab[word] +=1
except:
vocab[word] =1
return vocab
def split_data(X,Y):
return train_test_split(X, Y, test_size=0.2, random_state=42)
 def train_model(model, X, Y, epochs, bs):
X_train, X_val, y_train, y_val = split_data(X,Y)
 STEP_SIZE_TRAIN = X_train.shape[0]//bs + 1
STEP_SIZE_VAL = X_val.shape[0]//bs + 1
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
 train_history = model.fit(X_train, y_train,
steps_per_epoch = STEP_SIZE_TRAIN,
validation_data =(X_val,y_val),
validation_steps = STEP_SIZE_VAL,
epochs = epochs, shuffle = True, 
)
 return train_history, model 
<train_model>
sentences=df_train['question_text'].progress_apply(lambda x : x.split() ).values
vocab=build_vocab(sentences)
 print({k : vocab[k] for k in list(vocab)[:5]} )
epochs, bs = 20, 32
train_hist, final_model = train_model(model, norm_X, y, epochs, bs )<data_type_conversions>
def check_coverage(vocab, embeddings_index):
oov={}
a={}
i,k=0,0
for word in tqdm(vocab):
try:
a[word]=embeddings_index[word]
k+= vocab[word]
except:
oov[word]=vocab[word]
i+=vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
%%time
samples, columns = test_df.shape
X_test = np.zeros(( samples,28,28,1))
X_norm_test = np.zeros(( samples,28,28,1))
for sample in tqdm(range(samples)) :
X_test[sample,:,:,:] = test_df.iloc[sample,:].values.reshape(28,28,1 ).astype('float32')
 X_norm_test[sample,:,:,:] = standard_norm(X_test[sample,:,:].reshape(28,28,1))<init_hyperparams>
oov = check_coverage(vocab,embeddings1_index )
class MNIST_CAM:
def __init__(self, img):
self.resize_width, self.resize_height, _ = img.shape
def standard_norm(self, img):
return(( img - np.mean(img)) /np.std(img))
def feature_model(self, model):
return Model(inputs = model.layers[0].input, outputs = model.layers[-3].output)
 def weight_tensor(self, model):
final_outputs = model.layers[-1]
return final_outputs.get_weights() [0]
def predict_class(self, model, X):
prob_vec = model.predict(X)
 return np.argmax(prob_vec[0])
 def generate_CAM(self, model, img):
norm_img = self.standard_norm(img)
 Fmap_model = self.feature_model(model)
 Wtensor = self.weight_tensor(model)
 feature_map = Fmap_model.predict(norm_img.reshape(1,28,28,1))
label = self.predict_class(model, norm_img.reshape(1,28,28,1))
CAM = feature_map.dot(Wtensor[:,label])[0,:,:]
return cv.resize(CAM, 
(self.resize_width, self.resize_height),
interpolation = cv.INTER_CUBIC), label
def generate_probvec(self, model, img):
X = self.standard_norm(img)
 prob_vec = model.predict(X.reshape(1,28,28,1))
return prob_vec<predict_on_test>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
final_model.predict(X_norm_test[0,:,:,:].reshape(1,28,28,1))<predict_on_test>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
y_test,test_Ids = np.zeros(( samples,1)) , np.zeros(( samples,1))
for sample in tqdm(range(samples)) :
y_test[sample,0] = np.argmax(final_model.predict(X_norm_test[sample,:,:,:].reshape(1,28,28,1)))
 test_Ids[sample,0] = int(sample+1 )<prepare_output>
df_train['question_text']=df_train['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))
df_test['question_text']=df_test['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))
sentences= df_train['question_text'].apply(lambda x : x.split())
 vocab = build_vocab(sentences )
label_df, pred_df = pd.DataFrame(test_Ids), pd.DataFrame(y_test)
 sub_df = pd.concat([label_df, pred_df], axis = 1)
 sub_df.iloc[:,:] = sub_df.iloc[:,:].astype('int')
 sub_df.columns = ['ImageId', 'Label']
sub_df.head()<save_to_csv>
oov = check_coverage(vocab,embeddings1_index )
sub_df.to_csv('sample_submission.csv', index=False )<import_modules>
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
from torchvision import transforms, models
from torch.utils.data import DataLoader, Dataset
from torch.autograd import Variable
from torch import nn, optim
from torch.optim import lr_scheduler
import torch.nn.functional as F
import os<set_options>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
WORKING_FOLDER = "/kaggle/input/digit-recognizer"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )<load_from_csv>
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
train_csv_df = pd.read_csv(os.path.join(WORKING_FOLDER, "train.csv"))
print(train_csv_df.label.head())
 print(train_csv_df.info() )<categorify>
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
sentences= df_train['question_text'].apply(lambda x : x.split())
 vocab = build_vocab(sentences)
n_feat_cols = 28 * 28
class MNIST_Dataset(Dataset):

 def __init__(self, dataframe, transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees=20, translate=(0.1, 0.1)) ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,))
])) :
df = dataframe
if len(df.columns)== n_feat_cols:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
if self.y is not None:
return self.transform(self.X[idx]), self.y[idx]
else:
return self.transform(self.X[idx])
<choose_model_class>
oov = check_coverage(vocab,embeddings1_index )
class MNIST_ResNet(ResNet):
def __init__(self):
super(MNIST_ResNet, self ).__init__(BasicBlock, [2, 2, 2, 2], num_classes=10)
 self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=1, padding=3,bias=False)
 model = MNIST_ResNet()
model = model.to(device)
<create_dataframe>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
train_csv_labels = train_csv_df.iloc[:,0].values
batch_size = 256
train_full_dataset = MNIST_Dataset(train_csv_df)
 train_full_loader = torch.utils.data.DataLoader(dataset=train_full_dataset, batch_size=batch_size, shuffle=True)
<choose_model_class>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
epochs = 50
steps = 0
print_every = 132
train_losses, val_losses = [], []
step_size = 10
base_lr = 0.01
optimizer = optim.Adam(model.parameters() , lr=base_lr)
 criterion = nn.CrossEntropyLoss()
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=0.1)
 print("Training...")
 for e in range(epochs):
print("Epoch ", e)
 running_loss = 0
for images, labels in train_full_loader:
images = images.to(device)
 labels = labels.to(device)
 steps += 1
optimizer.zero_grad()
log_ps = model(images)
 loss = criterion(log_ps, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
if steps % print_every == 0:
train_losses.append(running_loss/len(train_full_loader))
print("Epoch: {}/{}.. ".format(e+1, epochs),
"Training Loss: {:.3f}.. ".format(train_losses[-1])
)
 exp_lr_scheduler.step() 
<load_from_csv>
df_train['question_text'] = df_train['question_text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
df_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
sentences= df_train['question_text'].progress_apply(lambda x : x.split())
 sentences = [[word for word in sentence] for sentence in tqdm(sentences)]
vocab = build_vocab(sentences )
test_csv_df = pd.read_csv(os.path.join(WORKING_FOLDER, "test.csv"))
batch_size = 256
test_dataset = MNIST_Dataset(test_csv_df)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False )<train_model>
oov = check_coverage(vocab,embeddings1_index )
submission = [['ImageId', 'Label']]
with torch.no_grad() :
model.eval()
image_id = 1
for images in test_loader:
images = images.to(device)
 log_ps = model(images)
 ps = torch.exp(log_ps)
 top_p, top_class = ps.topk(1, dim=1)
 for prediction in top_class:
submission.append([image_id, prediction.item() ])
 image_id += 1 
<save_to_csv>
gc.collect()
submission_df = pd.DataFrame(submission)
 submission_df.columns = submission_df.iloc[0]
submission_df = submission_df.drop(0, axis=0)
 submission_df.to_csv("submission.csv", index=False )<set_options>
len_voc = 95000
max_len =60
%matplotlib inline<load_from_csv>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
X = pd.read_csv('.. /input/digit-recognizer/train.csv')
 X_test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 X.shape, X_test.shape<prepare_x_and_y>
t = Tokenizer(num_words=len_voc, filters='')
 t.fit_on_texts(df_train['question_text'])
 X = t.texts_to_sequences(df_train['question_text'])
 X_test=t.texts_to_sequences(df_test['question_text'])
 X = pad_sequences(X, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 word_index=t.word_index
X_train = X.drop(['label'],1)
 Y_train = X['label']
X_train.shape<data_type_conversions>
y = df_train['target'].values
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train = X_train/255
X_test = X_test/255
X_train = X_train.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1,28 ,28, 1)
 X_train.shape, X_test.shape 
<categorify>
from sklearn.model_selection import train_test_split
Y_train= tf.keras.utils.to_categorical(Y_train, 10)
 Y_train.shape<split>
from sklearn.model_selection import train_test_split
x_train, val_x, y_train, val_y = train_test_split(X_train, Y_train, test_size=0.20 )<choose_model_class>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=420 )
datagen = ImageDataGenerator(zoom_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1,
rotation_range = 15 )<choose_model_class>
del df_train
es = EarlyStopping(monitor='loss', patience=12)
 filepath="/kaggle/working/bestmodel.h5"
md = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min' )<define_search_space>
gc.collect()
epochs = 30
num_classes = 10
batch_size = 128
input_shape =(28, 28, 1)
 adam = tf.keras.optimizers.Adam(0.001 )<choose_model_class>
def make_embed_matrix(embeddings_index, word_index, len_voc):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = word_index
embedding_matrix = np.random.normal(emb_mean, emb_std,(len_voc, embed_size))
for word, i in word_index.items() :
if i >= len_voc:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
i = Input(shape=input_shape)
 x = Conv2D(32,(3, 3), strides=2, activation='relu' )(i)
 x = Conv2D(64,(3, 3), strides=2, activation='relu' )(x)
 x = Flatten()(x)
 x = Dropout(0.2 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.2 )(x)
 x = Dense(K, activation='softmax' )(x)
 model = Model(i, x)
 model.summary()<choose_model_class>
embedding = make_embed_matrix(embeddings1_index, word_index, len_voc)
 del word_index
gc.collect()
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
early = EarlyStopping(monitor='val_loss', mode="min", patience=2 )
History = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
epochs = epochs,
validation_data =(val_x, val_y),
callbacks = [es,md],
shuffle = True
)<predict_on_test>
lstm = Sequential()
lstm.add(Embedding(len_voc, 300, weights=[embedding], trainable=False))
lstm.add(Bidirectional(LSTM(units = 256, return_sequences= True)))
 lstm.add(Dropout(rate = 0.2))
lstm.add(Conv1D(128, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform"))
lstm.add(GlobalMaxPooling1D())
 lstm.add(Dropout(rate = 0.2))
lstm.add(Dense(units = 1, activation = 'sigmoid'))
lstm.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'] )
pred = model1.predict(X_test)
 pred_class = np.argmax(pred,axis=1)
<save_to_csv>
epochs = 3
batch_size = 128
submissions=pd.DataFrame({"ImageId": list(range(1,len(pred_class)+1)) ,
"Label": pred_class})
 submissions.to_csv("submissions.csv", index=False, header=True)
 submissions 
<import_modules>
hist = lstm.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_val,y_val),callbacks=[early] )
import numpy as np
import pandas as pd
import os
import re
import tensorflow.keras.backend as K<set_options>
from sklearn.metrics import confusion_matrix,classification_report
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<load_from_csv>
pred_val_y = lstm.predict([X_val], batch_size=1024, verbose=0 )
df_train = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/train.csv")
 df_test = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/test.csv")
 sample_sub = pd.read_csv("/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv" )<define_variables>
pred_val_y =(pred_val_y > 0.50 ).astype(int )
FOLDS = 5
IMG_SIZE = 512
BATCH_SIZE = 32
EPOCHS = 12<define_variables>
print(classification_report(y_val,pred_val_y))
GCS_PATH2 = KaggleDatasets().get_gcs_path("isic2019-512x512")
 GCS_PATH1 = KaggleDatasets().get_gcs_path("melanoma-512x512" )<define_variables>
print(confusion_matrix(y_val,pred_val_y))
train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')
 train_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec')
 data_filenames = train_filenames1 + train_filenames2<split>
pred_test=lstm.predict([X_test], batch_size=1024, verbose=0 )
train_filenames, valid_filenames = train_test_split(data_filenames, test_size=0.2, random_state=0, shuffle=True )<prepare_x_and_y>
pred_test =(pred_test > 0.50 ).astype(int )
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = example['image']
label = example['target']
return image, label
def read_unlabeled_tfrecord(example, return_image_name=True):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = example['image']
idnum = example['image_name']
return image, idnum if return_image_name else 0
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset<prepare_x_and_y>
out_df = pd.DataFrame({"qid":df_test["qid"].values} )
def decode_image(image_data, augment=False):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if augment:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 image = tf.image.random_saturation(image, 0, 2)
 image = tf.image.rot90(image)
 image = tf.reshape(image, [IMG_SIZE,IMG_SIZE, 3])
 return image
def get_training_dataset() :
dataset = tf.data.TFRecordDataset(train_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(1024*8)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=True), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_val_dataset() :
dataset = tf.data.TFRecordDataset(valid_filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = tf.data.TFRecordDataset(test_filenames, num_parallel_reads=AUTO)
 dataset = dataset.repeat()
dataset = dataset.map(lambda example: read_unlabeled_tfrecord(example, False), num_parallel_calls=AUTO)
 dataset = dataset.map(lambda img, imgname_or_label:(decode_image(img, augment=False), imgname_or_label), num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE * REPLICAS)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<init_hyperparams>
out_df['prediction'] = pred_test
<choose_model_class><EOS> 
out_df.to_csv("submission.csv", index=False )