Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<define_variables>
from typing import Any, Dict
import umap
import numpy as np
from scipy.sparse.csgraph import connected_components
from sklearn.decomposition import PCA
import os
import pandas as pd
import matplotlib.pyplot as plt
import warnings
import tensorflow as tf
from sklearn.decomposition import PCA
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import Reshape, LayerNormalization, PReLU
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import AveragePooling1D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.constraints import max_norm
from tensorflow.keras import regularizers
import os
import lightgbm as lgb
import matplotlib.pyplot as plt
from datetime import datetime
from sklearn.model_selection import KFold
import xgboost as xgb
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
train = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/train.csv")
 test = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/test.csv")
 ss = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv" )
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
def docking(train: pd.DataFrame, test: pd.DataFrame)-> [pd.DataFrame, pd.Series]:
print("docking")
 dict_a: Dict
dict_a = {"Class_1": 0,
"Class_2": 1,
"Class_3": 2,
"Class_4": 3,
"Class_5": 4,
"Class_6": 5,
"Class_7": 6,
"Class_8": 7,
"Class_9": 8,
}
target = train["target"].map(dict_a)
 con_df = pd.concat([train, test], sort=False)

con_df = con_df.drop(["id", "target"], axis=1)
 con_df.columns = con_df.columns.map(lambda x: int(x[5:]))
con_df = np.log1p(con_df)
 return con_df, target
def split_data(df, df_pca, df_features, target, join_pca: bool):
if join_pca:
df = pd.concat([df.reset_index(drop=True), df_pca.reset_index(drop=True)], axis=1)
 df = pd.concat([df.reset_index(drop=True), df_features.reset_index(drop=True)], axis=1)
 train_df = df[:len(target)]
test_df = df[len(target):]
return train_df, test_df
def make_features(df: pd.DataFrame):
memo = pd.DataFrame()
memo["count_zero"] = df[df == 0].count(axis=1)
 return memo
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.3 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.3 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
def do_pca(df: pd.DataFrame, target: pd.Series):
n = 10
pca = PCA(n_components=n)
 pca.fit(df[:len(target)])
 df_pca = pca.transform(df)
 n_name = [f"pca{i}" for i in range(n)]
df_pca = pd.DataFrame(df_pca, columns=n_name)
 return df_pca
n_splits = 5<choose_model_class>
df, target = docking(train, test)
 df_pca = do_pca(df, target)
 df_features = make_features(df)
 df_train, df_test = split_data(df, df_pca, df_features, target, join_pca=True )
<predict_on_test>
def nn_train_model(
 df: pd.DataFrame, target: pd.DataFrame, test: pd.DataFrame 
):
n_splits = 5
num_class = 9
epochs = 10
lr_init = 0.01
bs = 256
num_features = df.shape[1]
folds = KFold(n_splits=n_splits, random_state=71, shuffle=True)
 model = tf.keras.models.Sequential([
Input(shape=(num_features,)) ,
Dense(128, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(128, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(64, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(num_class, activation="softmax")
 ])
 optimizer = tf.keras.optimizers.Adam(lr=lr_init, decay=0.0001)

callbacks = []
model.compile(optimizer=optimizer, loss=tf.keras.losses.SparseCategoricalCrossentropy() , metrics=['accuracy'])
 preds = np.zeros(( test.shape[0], num_class))
for trn_idx, val_idx in folds.split(df, target):
train_x = df.iloc[trn_idx, :].values
val_x = df.iloc[val_idx, :].values
train_y = target[trn_idx].values
val_y = target[val_idx].values
model.fit(train_x, train_y, validation_data=(val_x, val_y), epochs=epochs, verbose=2, batch_size=bs,
callbacks=callbacks)
 preds += model.predict(test.values)/ n_splits
return preds
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(15):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
pred1 = nn_train_model(df_train, target, df_test )
<save_to_csv><EOS> 
def make_submit_file(pred: np.ndarray, ss: pd.DataFrame)-> None:
save_path = "submission.csv"
ss.iloc[:, 1:] = pred
ss.to_csv(save_path, index=None )