Kaggle competition: OCRV Test Task
<SOS> metric: meanfscore Kaggle data source: ocrv-test-task<load_from_csv>
!pip install pymystem3
df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv' )<choose_model_class>
RAND_STATE = 37
N_JOBS = -1
VERB_LEVEL = 2
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights(".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5")
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<load_from_csv>
df_train = pd.read_csv('/kaggle/input/ocrv-intent-classification/train.csv', index_col='id')
 df_train.head()
submit = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 model = load_model('.. /input/densenet_v2_0817_2/densenet_v2_0817_2.h5')
 preds4 = []<set_options>
df_train[df_train.text.isna() ]
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
df_train = df_train.drop_duplicates()
df_train.info()
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
from fastai.callbacks import *
import PIL
import cv2<set_options>
df_test = pd.read_csv('/kaggle/input/ocrv-intent-classification/test.csv', index_col='id' )
def seed_everything(seed=1358):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_from_csv>
df_test = df_test.fillna(' ' )
PATH = Path('.. /input/aptos2019-blindness-detection')
 df = pd.read_csv(PATH/'train.csv' )<feature_engineering>
lemmatizator = Mystem()
mystem_preprocessor = lambda x: ''.join(lemmatizator.lemmatize(x)[:-1])
 X_train = df_train.text.apply(mystem_preprocessor )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2 )<compute_test_metric>
y_train = df_train.label
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<load_from_csv>
X_test = df_test.text.apply(mystem_preprocessor )
sample_df = pd.read_csv(PATH/'sample_submission.csv')
 sample_df.head()<normalization>
nltk.download('stopwords')
 russian_stopwords = stopwords.words('russian' )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0')
 data =(
 src.transform(tfms,size=224)
.databunch() 
.normalize(imagenet_stats)
)
 learn = cnn_learner(data, base_arch=models.resnet50 ,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True)
 learn.data.add_test(ImageList.from_df(sample_df,PATH,folder='test_images',suffix='.png'))
learn.load('resnet50_0823' )<predict_on_test>
mnnb_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('mnnb', MultinomialNB()),])
 parameters = {
'vect__ngram_range': [(1,1),(1,2),(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'mnnb__alpha': [.1,.01,.001]}
gs_mnnb_clf= GridSearchCV(mnnb_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_mnnb_clf = gs_mnnb_clf.fit(X_train, y_train)
 print(gs_mnnb_clf.best_score_, gs_mnnb_clf.best_params_ )
preds,y = learn.get_preds(DatasetType.Test )<choose_model_class>
def predict_submit(gs, df, name_suf):
y_pred = gs.predict(df)
 submission = pd.DataFrame(y_pred, columns=['label'])
 submission.index.name = 'id'
submission.to_csv(f'submission_{name_suf}.csv' )
optR = OptimizedRounder()
coefficients = [0.563203, 1.400153, 2.43412, 3.439786]
test_predictions = optR.predict(preds, coefficients )<data_type_conversions>
predict_submit(gs_mnnb_clf, X_test, name_suf='mnnb' )
preds5 = test_predictions.astype(int )<categorify>
sgd_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('sgdc', SGDClassifier(random_state=RAND_STATE))
])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'sgdc__loss': ['hinge', 'perceptron'],
'sgdc__penalty': ['l1', 'l2', 'elasticnet'],
'sgdc__alpha': [1.25e-5, 2.5e-5, 5e-5]}
gs_sgd_clf = GridSearchCV(sgd_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_sgd_clf = gs_sgd_clf.fit(X_train, y_train)
 print(gs_sgd_clf.best_score_, gs_sgd_clf.best_params_ )
preds1 = preds1.flatten()
preds2 = preds2.flatten()
preds3 = preds3.flatten()
preds4 = preds4.flatten()
preds5 = preds5.flatten()<categorify>
predict_submit(gs_sgd_clf, X_test, name_suf='sgd' )
score = [0.783, 0.750, 0.783, 0.741, 0.738]
weight = [0.23, 0.195, 0.23, 0.175, 0.170]
predsData = weight[0]*preds1 + weight[1]*preds2 + weight[2]*preds3 + weight[3]*preds4 + weight[4]*preds5
coef = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(predsData):
if pred < coef[0]:
predsData[i] = 0
elif pred >= coef[0] and pred < coef[1]:
predsData[i] = 1
elif pred >= coef[1] and pred < coef[2]:
predsData[i] = 2
elif pred >= coef[2] and pred < coef[3]:
predsData[i] = 3
else:
predsData[i] = 4
submission = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 submission.diagnosis = predsData.astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
logreg_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('logreg', LogisticRegression(random_state=RAND_STATE)) ,])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None, russian_stopwords],
'tfidf__use_idf': [True, False],
'logreg__C': [11,17,23],
'logreg__multi_class': ['ovr', 'multinomial'],
'logreg__solver': ['lbfgs', 'newton-cg']}
gs_logreg_clf = GridSearchCV(logreg_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_logreg_clf = gs_logreg_clf.fit(X_train, y_train)
 print(gs_logreg_clf.best_score_, gs_logreg_clf.best_params_ )
KAGGLE_DIR = '.. /input/aptos2019-blindness-detection/'
TRAIN_DF_PATH = KAGGLE_DIR + "train.csv"
TEST_DF_PATH = KAGGLE_DIR + 'test.csv'
TRAIN_IMG_PATH = KAGGLE_DIR + "train_images/"
TEST_IMG_PATH = KAGGLE_DIR + 'test_images/'
WEIGHTS_PATH = os.path.join(KAGGLE_DIR, '.. ','trained-models', 'effnet_modelB5.h5')
 SAVED_MODEL_NAME = 'effnet_modelB5.h5'
seed = 42
np.random.seed(seed)
 tf.set_random_seed(seed)
 t_start = time.time()
print('
for file in os.listdir(KAGGLE_DIR):
print('{}| {} MB'.format(file.ljust(30),
str(round(os.path.getsize(KAGGLE_DIR + file)/ 1000000, 2))))<feature_engineering>
predict_submit(gs_logreg_clf, X_test, name_suf='logreg' )
print("Image IDs and Labels(TRAIN)")
 train_df = pd.read_csv(TRAIN_DF_PATH)
 train_df['id_code'] = train_df['id_code'] + ".png"
print(f"Training images: {train_df.shape[0]}")
 display(train_df.head())
 print("Image IDs(TEST)")
 test_df = pd.read_csv(TEST_DF_PATH)
 test_df['id_code'] = test_df['id_code'] + ".png"
print(f"Testing Images: {test_df.shape[0]}")
 display(test_df.head() )<define_variables>
word2vec_path = '.. /input/web-mystem-skipgram-500-2-2015/web.bin'
word2vec_size = 500
word2vec = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)
 words = word2vec.index2word
w_rank = {}
for i,word in enumerate(words):
word = word.split('_')[0]
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or known(edits2(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
IMG_WIDTH = 456
IMG_HEIGHT = 456
IMAGE_SIZE = 456
CHANNELS = 3<predict_on_test>
!pip install pymorphy2
def get_preds_and_labels(model, generator):

 preds = []
labels = []
for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):
x, y = next(generator)
 preds.append(model.predict(x))
labels.append(y)
 return np.concatenate(preds ).ravel() , np.concatenate(labels ).ravel()<train_model>
def spell_norm(ser):
tokenizer = RegexpTokenizer(r'[а-яА-Я]+')
 morph = pymorphy2.MorphAnalyzer()
normolize = lambda t: morph.parse(t)[0].normal_form
def spelling_correct(token):
methods_stack = morph.parse(token)[0].methods_stack
if str(morph.parse(token)[0].methods_stack[0][0])!= '<DictionaryAnalyzer>':
return False
for method in methods_stack[1:]:
if 'unknown' in str(method[0] ).lower() :
return False
return True
def norm_spelling(tokens):
lemmas = []
for token in tokens:
if not spelling_correct(token):
token = correction(token)
 lemma = normolize(token)
 lemmas.append(lemma)
 return lemmas
return ser.apply(tokenizer.tokenize ).apply(norm_spelling)
 spell_norm(df_train.text[10:20] )
class Metrics(Callback):

 def on_train_begin(self, logs={}):

 self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):

 y_pred, labels = get_preds_and_labels(model, val_generator)
 y_pred = np.rint(y_pred ).astype(np.uint8 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {round(_val_kappa, 4)}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save(SAVED_MODEL_NAME)
 return<prepare_x_and_y>
%%time
df_train['spell_norm_text'] = spell_norm(df_train.text ).apply(lambda x: ' '.join(x))
def info_image(im):
cy = im.shape[0]//2
midline = im[cy,:]
midline = np.where(midline>midline.mean() /3)[0]
if len(midline)>im.shape[1]//2:
x_start, x_end = np.min(midline), np.max(midline)
 else:
x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10
cx =(x_start + x_end)/2
r =(x_end - x_start)/2
return cx, cy, r
def resize_image(im, augmentation=False):
cx, cy, r = info_image(im)
 scaling = IMAGE_SIZE/(2*r)
 rotation = 0
if augmentation:
scaling *= 1 + 0.3 *(np.random.rand() -0.5)
 rotation = 360 * np.random.rand()
M = cv2.getRotationMatrix2D(( cx,cy), rotation, scaling)
 M[0,2] -= cx - IMAGE_SIZE/2
M[1,2] -= cy - IMAGE_SIZE/2
return cv2.warpAffine(im,M,(IMAGE_SIZE,IMAGE_SIZE))
def subtract_median_bg_image(im):
k = 45
bg = cv2.medianBlur(im, k)
 return cv2.addWeighted(im, 4, bg, -4, 128)
 def subtract_gaussian_bg_image(im):
k = np.max(im.shape)/15
bg = cv2.GaussianBlur(im ,(0,0),k)
 return cv2.addWeighted(im, 4, bg, -4, 128)
 def contrast_stretching(img):
rr, gg, bb = cv2.split(img)
 imgray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
 im = imgray
ih, iw = imgray.shape 
(minVal, maxVal, minLoc, maxLoc)= cv2.minMaxLoc(imgray)
 for i in range(ih):
for j in range(iw):
im[i, j] = 255 *(( gg[i, j] - minVal)/(maxVal - minVal))
limg = cv2.merge(( rr, im, bb))
return limg
def histogram_normalization(image):
hist,bins = np.histogram(image.flatten() ,256,[0,256])
 cdf = hist.cumsum()
cdf_m = np.ma.masked_equal(cdf,0)
 cdf_m =(cdf_m - cdf_m.min())*255/(cdf_m.max() -cdf_m.min())
 cdf = np.ma.filled(cdf_m,0 ).astype('uint8')
 img2 = cdf[image]
return img2
PARAM = 96
def Radius_Reduction(img,PARAM):
h,w,c=img.shape
Frame=np.zeros(( h,w,c),dtype=np.uint8)
 cv2.circle(Frame,(int(math.floor(w/2)) ,int(math.floor(h/2))),int(math.floor(( h*PARAM)/float(2*100))),(255,255,255), -1)
 Frame1=cv2.cvtColor(Frame, cv2.COLOR_BGR2GRAY)
 img1 =cv2.bitwise_and(img,img,mask=Frame1)
 return img1
def preprocess_imagev2(image, sigmaX=10):

 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = resize_image(image)
 image = subtract_gaussian_bg_image(image)
 image = Radius_Reduction(image, PARAM)
 return image<normalization>
X_train, y_train = df_train.spell_norm_text, df_train.label
def info_image(im):
cy = im.shape[0]//2
midline = im[cy,:]
midline = np.where(midline>midline.mean() /3)[0]
if len(midline)>im.shape[1]//2:
x_start, x_end = np.min(midline), np.max(midline)
 else:
x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10
cx =(x_start + x_end)/2
r =(x_end - x_start)/2
return cx, cy, r
def resize_image(im, augmentation=False):
cx, cy, r = info_image(im)
 scaling = IMAGE_SIZE/(2*r)
 rotation = 0
if augmentation:
scaling *= 1 + 0.3 *(np.random.rand() -0.5)
 rotation = 360 * np.random.rand()
M = cv2.getRotationMatrix2D(( cx,cy), rotation, scaling)
 M[0,2] -= cx - IMAGE_SIZE/2
M[1,2] -= cy - IMAGE_SIZE/2
return cv2.warpAffine(im,M,(IMAGE_SIZE,IMAGE_SIZE))
def adaptive_histo_equalization(res_image):
lab= cv2.cvtColor(res_image, cv2.COLOR_BGR2LAB)
 l, a, b = cv2.split(lab)
 clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8,8))
cl = clahe.apply(l)
 limg = cv2.merge(( cl,a,b))
final = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR)
 return final
def preprocess_imagev3(image, sigmaX=10):

 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = resize_image(image)
 image = adaptive_histo_equalization(image)
 return image<define_search_space>
%%time
df_test['spell_norm_text'] = spell_norm(df_test.text ).apply(lambda x: ' '.join(x))
def Kirsch(crop):
Input=crop[:,:,1]
a,b=Input.shape
Kernel=np.zeros(( 3,3,8))
Kernel[:,:,0]=np.array([[5,5,5],[-3,0,-3],[-3,-3,-3]])
 Kernel[:,:,1]=np.array([[-3,5,5],[-3,0,5],[-3,-3,-3]])
 Kernel[:,:,2]=np.array([[-3,-3,5],[-3,0,5],[-3,-3,5]])
 Kernel[:,:,3]=np.array([[-3,-3,-3],[-3,0,5],[-3,5,5]])
 Kernel[:,:,4]=np.array([[-3,-3,-3],[-3,0,-3],[5,5,5]])
 Kernel[:,:,5]=np.array([[-3,-3,-3],[5,0,-3],[5,5,-3]])
 Kernel[:,:,6]=np.array([[5,-3,-3],[5,0,-3],[5,-3,-3]])
 Kernel[:,:,7]=np.array([[5,5,-3],[5,0,-3],[-3,-3,-3]])
 dst=np.zeros(( a,b,8))
for x in range(0,8):
dst[:,:,x] = cv2.filter2D(Input,-1,Kernel[:,:,x])
 Out=np.zeros(( a,b))
for y in range(0,a-1):
for z in range(0,b-1):
Out[y,z]=max(dst[y,z,:])
 Out=np.uint8(Out)
 return Out
def preprocess_imagev4(image):

 image = preprocess_image(image, sigmaX=10)
 image_edge = Kirsch(image)
 return image_edge<normalization>
X_test = df_test.spell_norm_text
def info_image(im):
cy = im.shape[0]//2
midline = im[cy,:]
midline = np.where(midline>midline.mean() /3)[0]
if len(midline)>im.shape[1]//2:
x_start, x_end = np.min(midline), np.max(midline)
 else:
x_start, x_end = im.shape[1]//10, 9*im.shape[1]//10
cx =(x_start + x_end)/2
r =(x_end - x_start)/2
return cx, cy, r
def resize_image(im, augmentation=False):
cx, cy, r = info_image(im)
 scaling = IMAGE_SIZE/(2*r)
 rotation = 0
if augmentation:
scaling *= 1 + 0.3 *(np.random.rand() -0.5)
 rotation = 360 * np.random.rand()
M = cv2.getRotationMatrix2D(( cx,cy), rotation, scaling)
 M[0,2] -= cx - IMAGE_SIZE/2
M[1,2] -= cy - IMAGE_SIZE/2
return cv2.warpAffine(im,M,(IMAGE_SIZE,IMAGE_SIZE))
def adaptive_histo_equalization(res_image):
lab= cv2.cvtColor(res_image, cv2.COLOR_BGR2LAB)
 l, a, b = cv2.split(lab)
 l = l.astype(np.uint8)
 a = a.astype(np.uint8)
 b = b.astype(np.uint8)
 clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8,8))
cl = clahe.apply(l)
 limg = cv2.merge(( cl,a,b))
final = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR)
 return final
def preprocess_imagev5(image, sigmaX=5):

 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = crop_image_from_gray(image)
 image = circle_crop_v2(image)
 image = cv2.resize(image,(IMG_WIDTH, IMG_HEIGHT))
image = adaptive_histo_equalization(image)
 image = cv2.addWeighted(image,4, cv2.GaussianBlur(image,(0,0),sigmaX), -4, 128)
 return image.astype(np.uint8 )<prepare_x_and_y>
sgd_spnorm_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('sgdc', SGDClassifier(random_state=RAND_STATE))
])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__stop_words': [None,],
'tfidf__use_idf': [False, ],
'sgdc__loss': ['hinge', 'perceptron'],
'sgdc__penalty': ['l1', 'l2', 'elasticnet'],
'sgdc__alpha': [5e-6, 1.25e-5, 2.5e-5, 5e-5],
'sgdc__class_weight': [None, 'balanced']}
gs_sgd_spnorm_clf = GridSearchCV(sgd_spnorm_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_sgd_spnorm_clf = gs_sgd_spnorm_clf.fit(X_train, y_train)
 print(gs_sgd_spnorm_clf.best_score_, gs_sgd_spnorm_clf.best_params_ )
y_labels = train_df['diagnosis'].values<train_model>
predict_submit(gs_sgd_spnorm_clf, X_test, name_suf='sgd_spnorm' )
BATCH_SIZE = 4
train_datagen = ImageDataGenerator(rotation_range=360,
horizontal_flip=True,
vertical_flip=True,
validation_split=0.10,
preprocessing_function=preprocess_image,
rescale=1 / 128.
)
 train_generator = train_datagen.flow_from_dataframe(train_df,
x_col='id_code',
y_col='diagnosis',
directory = TRAIN_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
subset='training')
 val_generator = train_datagen.flow_from_dataframe(train_df,
x_col='id_code',
y_col='diagnosis',
directory = TRAIN_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
subset='validation' )<choose_model_class>
logreg_spnorm_clf = Pipeline(
 [('vect', CountVectorizer()), 
('tfidf', TfidfTransformer()), 
('logreg', LogisticRegression(random_state=RAND_STATE)) ,])
 parameters = {
'vect__ngram_range': [(1,3),],
'vect__min_df': [1,],
'vect__max_df': [1.],
'vect__stop_words': [None,],
'tfidf__use_idf': [False,],
'logreg__C': [8,11,17],
'logreg__multi_class': ['ovr', 'multinomial'],
'logreg__solver': ['lbfgs', 'newton-cg'],
'logreg__class_weight': [None, 'balanced']}
gs_logreg_spnorm_clf = GridSearchCV(logreg_spnorm_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_logreg_spnorm_clf = gs_logreg_spnorm_clf.fit(X_train, y_train)
 print(gs_logreg_spnorm_clf.best_score_, gs_logreg_spnorm_clf.best_params_ )
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items())) 
<choose_model_class>
predict_submit(gs_logreg_spnorm_clf, X_test, name_suf='logreg_spnorm' )
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape<load_pretrained>
vectorizer = TfidfVectorizer(lowercase=True, analyzer='word', ngram_range=(1,3), use_idf=False)
 train_vectors = vectorizer.fit_transform(df_train.spell_norm_text.apply(lambda tr_vect: np.str_(tr_vect)) )
effnet = EfficientNetB5(weights=None,
include_top=False,
input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS))
effnet.load_weights('.. /input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5' )<choose_model_class>
def tagged_tokens(ser, tokenizer=RegexpTokenizer(r'\w+'), lemmatizator=Mystem()):
def lemteg(tokens, tokenizer=tokenizer, lemmatizator=lemmatizator):
lemtegs = []
for token in tokens:
try:
tag = lemmatizator.analyze(token)[0]['analysis'][0]['gr'].split(',')[0].split('=')[0]
except:
tag = 'XXX'
lemtegs.append(f'{token}_{tag}')
 return lemtegs
return ser.apply(tokenizer.tokenize ).apply(lemteg)
 tagged_tokens(df_train.spell_norm_text[:10] )
for i, layer in enumerate(effnet.layers):
if "batch_normalization" in layer.name:
effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001 )<choose_model_class>
df_train['tagged_tokens']= tagged_tokens(df_train.spell_norm_text )
def build_model() :

 model = Sequential()
model.add(effnet)
 model.add(GlobalAveragePooling2D())
 model.add(Dropout(0.5))
model.add(Dense(5, activation=elu))
model.add(Dense(1, activation="linear"))
model.compile(loss='mse',
optimizer=RAdam(lr=0.00005),
metrics=['mse', 'acc'])
 print(model.summary())
 return model
model = build_model()<init_hyperparams>
def get_average_word2vec(tokens_list, vector, vec_size, generate_missing=False):
if len(tokens_list)<1:
return np.zeros(vec_size)
 if generate_missing:
vectorized = [vector[word] if word in vector else np.random.rand(vec_size)for word in tokens_list]
else:
vectorized = [vector[word] if word in vector else np.zeros(vec_size)for word in tokens_list]
length = len(vectorized)
 summed = np.sum(vectorized, axis=0)
 averaged = np.divide(summed, length)
 return averaged
def get_word2vec_embeddings(vectors, ser, vec_size, generate_missing=False):
embeddings = ser.apply(
 lambda x: get_average_word2vec(x, vectors, vec_size=vec_size, generate_missing=generate_missing))
return sparse.csr_matrix(list(embeddings))
class_weight = {0: 1.,
1: 5.,
2: 1.75,
3: 5.,
4: 5.}<train_on_grid>
train_embeddings = get_word2vec_embeddings(word2vec, df_train.tagged_tokens, vec_size=word2vec_size )
kappa_metrics = Metrics()
es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)
 rlr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
verbose=1,
mode='auto',
epsilon=0.0001)
 model.fit_generator(train_generator,
steps_per_epoch=train_generator.samples // BATCH_SIZE,
epochs=25,
validation_data=val_generator,
validation_steps = val_generator.samples // BATCH_SIZE,
class_weight = class_weight,
callbacks=[kappa_metrics, es, rlr] )<load_pretrained>
X_train, y_train = sparse.hstack(( train_embeddings, train_vectors)) , df_train.label
model.load_weights(SAVED_MODEL_NAME)
<compute_test_metric>
test_vectors = vectorizer.transform(df_test.spell_norm_text.apply(lambda tr_vect: np.str_(tr_vect)))
 df_test['tagged_tokens']= tagged_tokens(df_test.spell_norm_text)
 test_embeddings = get_word2vec_embeddings(word2vec, df_test.tagged_tokens, vec_size=word2vec_size)
 X_test = sparse.hstack(( test_embeddings, test_vectors))
y_train_preds, train_labels = get_preds_and_labels(model, train_generator)
 y_train_preds = np.rint(y_train_preds ).astype(np.uint8 ).clip(0, 4)
 train_score = cohen_kappa_score(train_labels, y_train_preds, weights="quadratic")
 y_val_preds, val_labels = get_preds_and_labels(model, val_generator)
 y_val_preds = np.rint(y_val_preds ).astype(np.uint8 ).clip(0, 4)
 val_score = cohen_kappa_score(val_labels, y_val_preds, weights="quadratic" )<compute_test_metric>
sgd_w2v_clf = Pipeline(
 [('sgdc', SGDClassifier(random_state=RAND_STATE)) ])
 parameters = {
'sgdc__loss': ['hinge'],
'sgdc__penalty': ['l1', 'l2', 'elasticnet'],
'sgdc__alpha': [1.25e-5, 2.5e-5, 5e-5]}
gs_sgd_w2v_clf = GridSearchCV(sgd_w2v_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_sgd_w2v_clf = gs_sgd_w2v_clf.fit(X_train, y_train)
 print(gs_sgd_w2v_clf.best_score_, gs_sgd_w2v_clf.best_params_ )
print(f"The Training Cohen Kappa Score is: {round(train_score, 5)}")
 print(f"The Validation Cohen Kappa Score is: {round(val_score, 5)}" )<compute_train_metric>
predict_submit(gs_sgd_w2v_clf, X_test, name_suf='sgd_w2v' )
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_train_metric>
logreg_w2v_clf = Pipeline(
 [('logreg', LogisticRegression(random_state=RAND_STATE)) ,])
 parameters = {
'logreg__C': [11, 14, 17],
'logreg__multi_class': ['ovr', 'multinomial'],
'logreg__solver': ['lbfgs', 'newton-cg']}
gs_logreg_w2v_clf = GridSearchCV(logreg_w2v_clf, parameters, n_jobs=N_JOBS, verbose=VERB_LEVEL)
 gs_logreg_w2v_clf = gs_logreg_w2v_clf.fit(X_train, y_train)
 print(gs_logreg_w2v_clf.best_score_, gs_logreg_w2v_clf.best_params_ )
<compute_test_metric><EOS> 
predict_submit(gs_logreg_w2v_clf, X_test, name_suf='logreg_w2v' )