Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<filter>
import numpy as np
import pandas as pd
import random
import math
import argparse
import pickle
import tokenizers
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras import layers as L
from transformers import TFBertMainLayer,TFBertModel,TFBertPreTrainedModel,BertConfig,BertTokenizer
from transformers import TFRobertaModel, RobertaTokenizer,RobertaConfig
from transformers.modeling_tf_utils import get_initializer
from sklearn.model_selection import StratifiedKFold
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
train.iloc[detected_outliers_parch]<train_on_grid>
def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed)
 set_seed(args )
fare=train["Fare"]
fare=fare.values
fare=fare.reshape(-1,1)
 gmm=GaussianMixture(n_components=3,max_iter=100,random_state=0)
 gmm.fit(fare )<define_variables>
test_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 train_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 train_df.dropna(inplace=True)
 train_df=train_df.reset_index()
detected_outliers_fare=np.where(log_likelihood<-10)[0]<concatenate>
def load_data_roberta(df,tokenizer, is_train_eval):
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
ct = df.shape[0]
MAX_LEN_WORD=args.max_seq_length
input_ids = np.ones(( ct,MAX_LEN_WORD),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN_WORD),dtype='int32')
 offsets=[]
tweets=[]
for k in range(ct):
text1 = " "+" ".join(df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 sentiment_token = sentiment_id[df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, sentiment_token] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
offsets.append([(0,0)]* 2 + enc.offsets + [(0,0)])
 tweets.append(text1)
 if is_train_eval:
chars = np.zeros(( len(text1)))
 text2 = " ".join(df.loc[k,'selected_text'].split())
 start_idx = text1.find(text2)
 chars[start_idx:start_idx+len(text2)]=1
if text1[start_idx-1]==' ':
chars[start_idx-1] = 1
selected_toks_index = []
for i,(a,b)in enumerate(enc.offsets):
sm = np.sum(chars[a:b])
 if sm>0: selected_toks_index.append(i)
 if len(selected_toks_index)>0:
start_tokens[k,selected_toks_index[0]+2] = 1
end_tokens[k,selected_toks_index[-1]+2] = 1
if is_train_eval:
dataset={'input_ids':input_ids,
'attention_mask':attention_mask,
'token_type_ids':token_type_ids,
'start_tokens':start_tokens,
'end_tokens':end_tokens,
'tweets':tweets,
'offsets':offsets}
else:
dataset={'input_ids':input_ids,
'attention_mask':attention_mask,
'token_type_ids':token_type_ids,
'tweets':tweets,
'offsets':offsets}
return dataset
outliers=(detected_outliers_parch,detected_outliers_SibSp,detected_outliers_fare,detected_outliers_age)
 outliers=np.concatenate(outliers)
 outliers<drop_column>
model_path=".. /input/robertatransformer/roberta-base-tf_model.h5"
model_class=TFRobertaModel
config = RobertaConfig.from_pretrained(".. /input/robertatransformer/roberta-base-config.json")
 tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file=".. /input/robertatransformer/roberta-base-vocab.json",
merges_file=".. /input/robertatransformer/roberta-base-merges.txt",
lowercase=True,
add_prefix_space=True )
train=train.drop(index=outliers,axis=1)
 train.shape<concatenate>
trainset=load_data_roberta(train_df,tokenizer, is_train_eval=True)
 testset=load_data_roberta(test_df,tokenizer, is_train_eval=False )
data=pd.concat(objs=[train.drop(labels="Survived",axis=1), test], axis=0 ).reset_index(drop=True)
<count_values>
PAD_ID=1
lr=args.learning_rate
def build_model_cnn(model_class,model_path,config):
MAX_LEN_WORD=args.max_seq_length
ids = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN_WORD,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN_WORD - tf.reduce_sum(padding, -1)
 max_len_word = tf.reduce_max(lens)
 ids_ = ids[:, :max_len_word]
att_ = att[:, :max_len_word]
tok_ = tok[:, :max_len_word]
bertmodel = model_class.from_pretrained(model_path,config=config)
 x = bertmodel(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768,2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn,optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN_WORD - max_len_word]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN_WORD - max_len_word]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
dist_Pclass=train["Pclass"].value_counts(sort=False)
 dist_Pclass.rename(index={1:"First class",2:"Second class",3:"Third class"},inplace=True)
 dist_Pclass_Norm=dist_Pclass/dist_Pclass.sum()
print("Abs.values
{0}
".format(dist_Pclass))
print("Rel.values
{0}".format(dist_Pclass_Norm))<count_values>
LABEL_SMOOTHING=0.1
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
dist_sex=train["Sex"].value_counts(sort=False)
 dist_sex_Norm=dist_sex/dist_sex.sum()
print("Abs.values
{0}
".format(dist_sex))
print("Rel.values
{0}".format(dist_sex_Norm))<feature_engineering>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f )
data["title"]=data["Name"].apply(extract)
 train["title"]=train["Name"].apply(extract)
 data["title"].value_counts()<categorify>
def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
data.loc[(data["title"]=="Ms.")|(data["title"]=="Mlle.")|(data["title"]=="Mme."),"title"]="Miss."
data.loc[(data["title"]=="Master.")|(data["title"]=="Dr.")|(data["title"]=="Rev.")|(data["title"]=="Col.")|(data["title"]=="Major.")|(data["title"]=="Dona.")|(data["title"]=="Don.")|(data["title"]=="Capt.")|(data["title"]=="Sir.")|(data["title"]=="Jonkheer.")|(data["title"]=="Countess.")|(data["title"]=="Lady."),"title"]="wellSituated"
train.loc[(train["title"]=="Ms.")|(train["title"]=="Mlle.")|(train["title"]=="Mme."),"title"]="Miss."
train.loc[(train["title"]=="Master.")|(train["title"]=="Dr.")|(train["title"]=="Rev.")|(train["title"]=="Col.")|(train["title"]=="Major.")|(train["title"]=="Dona.")|(train["title"]=="Don.")|(train["title"]=="Capt.")|(train["title"]=="Sir.")|(train["title"]=="Jonkheer.")|(train["title"]=="Countess.")|(train["title"]=="Lady."),"title"]="wellSituated"
data["title"].value_counts()<categorify>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
data["title"]=data["title"].map({"Mr.":0,"Miss.":1,"Mrs.":2,"wellSituated":3})
 train["title"]=train["title"].map({"Mr.":0,"Miss.":1,"Mrs.":2,"wellSituated":3})
 data.head(5 )<categorify>
jac = [];
VER='v0';
DISPLAY=1
n_splits=args.cv_splits
lr=args.learning_rate
MAX_LEN=args.max_seq_length
n_best=3
input_ids=trainset['input_ids']
attention_mask=trainset['attention_mask']
token_type_ids=trainset['token_type_ids']
start_tokens=trainset['start_tokens']
end_tokens=trainset['end_tokens']
input_ids_test=testset['input_ids']
attention_mask_test=testset['attention_mask']
token_type_ids_test=testset['token_type_ids']
ct_train=input_ids.shape[0]
ct_test=input_ids_test.shape[0]
oof_start = np.zeros(( ct_train,MAX_LEN))
oof_end = np.zeros(( ct_train,MAX_LEN))
preds_start = np.zeros(( ct_test,MAX_LEN))
preds_end = np.zeros(( ct_test,MAX_LEN))
jac_valiation=np.zeros(( ct_train,))
out_of_range=[]
abnormal=[]
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=args.seed)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
K.clear_session()
model, padded_model = build_model_cnn(model_class,model_path,config)
 inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = f'roberta-base-{fold+1}.h5'
for epoch in range(1, args.num_train_epochs + 1):
BATCH_SIZE=args.train_batch_size
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY,
callbacks=[],validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model for validation and pred...')
 padded_model.set_weights(model.get_weights())
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_test,attention_mask_test,token_type_ids_test],verbose=DISPLAY)
 preds_start+=preds[0]/n_splits
preds_end+=preds[1]/n_splits
fold_jac_val=[]
for k in idxV:
a= np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st=train_df.loc[k,'text']
else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 jac_val=jaccard(st,train_df.loc[k,'selected_text'])
 fold_jac_val.append(jac_val)
 jac.append(np.mean(fold_jac_val))
print(f'>>>> FOLD {fold+1} Jaccard ={np.mean(fold_jac_val)}' )
train=pd.get_dummies(data=train,columns=["title"],prefix="title")
 data=pd.get_dummies(data=data,columns=["title"],prefix="title")
 train.head(5)
 train=train.drop(labels="Name",axis=1)
 data=data.drop(labels="Name",axis=1 )<categorify>
print(f'The CV jaccard is {np.mean(jac)}' )
labelencoder_X_1=LabelEncoder()
train["Sex"]=labelencoder_X_1.fit_transform(train.iloc[:,3])
 data["Sex"]=labelencoder_X_1.fit_transform(data.iloc[:,7])
 data.head(5)
<count_missing_values>
roberta_models=["roberta-base-1.h5",
"roberta-base-2.h5",
"roberta-base-3.h5",
"roberta-base-4.h5",
"roberta-base-5.h5"]
print(train["Cabin"].isnull().any())
 print(data["Cabin"].isnull().any() )<categorify>
MAX_LEN_WORD=args.max_seq_length
DISPLAY=1
input_ids_test=testset['input_ids']
attention_mask_test=testset['attention_mask']
token_type_ids_test=testset['token_type_ids']
preds_test_start = np.zeros(( ct_test,MAX_LEN_WORD))
preds_test_end = np.zeros(( ct_test,MAX_LEN_WORD))
input_ids_train=trainset['input_ids']
attention_mask_train=trainset['attention_mask']
token_type_ids_train=trainset['token_type_ids']
preds_oof_start = np.zeros(( ct_train,MAX_LEN_WORD))
preds_oof_end = np.zeros(( ct_train,MAX_LEN_WORD))
train=pd.get_dummies(data=train,columns=["Cabin"],prefix="Cabin")
 data=pd.get_dummies(data=data,columns=["Cabin"],prefix="Cabin")
 train.head(5 )<categorify>
model_number=len(roberta_models)
 for i in range(model_number):
print(f"predict roberta model----{i+1}")
 K.clear_session()
weight_fn=roberta_models[i]
_, padded_model = build_model_cnn(model_class,model_path,config)
 load_weights(padded_model, weight_fn)
 preds_test = padded_model.predict([input_ids_test,attention_mask_test,token_type_ids_test],verbose=DISPLAY)
 preds_test_start+=preds_test[0]/model_number
preds_test_end+=preds_test[1]/model_number
preds_oof = padded_model.predict([input_ids_train,attention_mask_train,token_type_ids_train],verbose=DISPLAY)
 preds_oof_start+=preds_oof[0]/model_number
preds_oof_end+=preds_oof[1]/model_number
train.loc[train["Embarked"].isnull() ==True,"Embarked"]="S"
data.loc[data["Embarked"].isnull() ==True,"Embarked"]="S"
train=pd.get_dummies(data=train,columns=["Embarked"],prefix="Embarked")
 data=pd.get_dummies(data=data,columns=["Embarked"],prefix="Embarked" )<drop_column>
def token_level_to_char_level(text, offsets, preds):
probas_char = np.zeros(len(text))
for i, offset in enumerate(offsets):
if offset[0] or offset[1]:
probas_char[offset[0]:offset[1]] = preds[i]
return probas_char
train=train.drop(labels="Ticket",axis=1)
 data=data.drop(labels="Ticket",axis=1 )<categorify>
tweets_train=trainset['tweets']
offsets_train=trainset['offsets']
probas_train_start=[]
probas_train_end=[]
for idx, tweet in enumerate(tweets_train):
probas_train_start.append(token_level_to_char_level(tweet, offsets_train[idx], preds_oof_start[idx]))
probas_train_end.append(token_level_to_char_level(tweet, offsets_train[idx], preds_oof_end[idx]))
train=pd.get_dummies(data=train,columns=["Pclass"],prefix="Pclass")
 data=pd.get_dummies(data=data,columns=["Pclass"],prefix="Pclass" )<count_missing_values>
tweets_test=testset['tweets']
offsets_test=testset['offsets']
probas_test_start=[]
probas_test_end=[]
for idx, tweet in enumerate(tweets_test):
probas_test_start.append(token_level_to_char_level(tweet, offsets_test[idx], preds_test_start[idx]))
probas_test_end.append(token_level_to_char_level(tweet, offsets_test[idx], preds_test_end[idx]))
print("NaN-Vals in SibSp in train set present? {0}".format(train["SibSp"].isnull().any()))
print("NaN-Vals in SibSp in data set present? {0}".format(data["SibSp"].isnull().any()))
print("NaN-Vals in Parch in train present? {0}".format(train["Parch"].isnull().any()))
print("NaN-Vals in Parch in data set present? {0}".format(data["Parch"].isnull().any()))<feature_engineering>
with open('char_pred_train_start.pkl', 'wb')as handle:
pickle.dump(probas_train_start, handle)
 with open('char_pred_train_end.pkl', 'wb')as handle:
pickle.dump(probas_train_end, handle )
train["familySize"]=train["SibSp"]+train["Parch"]+1
data["familySize"]=data["SibSp"]+data["Parch"]+1
train.head(5 )<categorify>
tokenizer_char = Tokenizer(num_words=None, char_level=True, oov_token='UNK', lower=True)
 tokenizer_char.fit_on_texts(train_df['text'].values)
 len_voc = len(tokenizer_char.word_index)+ 1
X_train = tokenizer_char.texts_to_sequences(train_df['text'].values)
 X_test = tokenizer_char.texts_to_sequences(test_df['text'].values )
train["lonely"]=train["familySize"].map(lambda x: 1 if x==1 else 0)
 data["lonely"]=data["familySize"].map(lambda x: 1 if x==1 else 0)
 train["smallFamily"]=train["familySize"].map(lambda x:1 if x==2 else 0)
 data["smallFamily"]=data["familySize"].map(lambda x:1 if x==2 else 0)
 train["biggerFamily"]=train["familySize"].map(lambda x:1 if x>2 & x<=4 else 0)
 data["biggerFamily"]=data["familySize"].map(lambda x:1 if x>2 & x<=4 else 0)
 train["biggerFamily"]=train["familySize"].map(lambda x:1 if x>4 else 0)
 data["biggerFamily"]=data["familySize"].map(lambda x:1 if x>4 else 0)
 train=train.drop(labels=["SibSp","Parch","familySize"],axis=1)
 data=data.drop(labels=["SibSp","Parch","familySize"],axis=1)
 train.head(5 )<drop_column>
def get_start_end_string(text, selected_text):
len_selected_text = len(selected_text)
 idx_start, idx_end = 0, 0
candidates_idx = [i for i, e in enumerate(text)if e == selected_text[0]]
for idx in candidates_idx:
if text[idx : idx + len_selected_text] == selected_text:
idx_start = idx
idx_end = idx + len_selected_text-1
break
assert text[idx_start: idx_end+1] == selected_text, f'"{text[idx_start: idx_end+1]}" instead of "{selected_text}" in "{text}"'
char_targets = np.zeros(len(text))
char_targets[idx_start: idx_end+1] = 1
return idx_start, idx_end
data.drop(labels = ["PassengerId"], axis = 1, inplace = True)
 train_set=data[:len(train)]
test=data[len(train):]
train.head(5 )<prepare_x_and_y>
def load_data_second_model(df, X, char_start_probas, char_end_probas, max_len_char, train=True):
ct=len(df)
 X = pad_sequences(X, maxlen=max_len_char, padding='post', truncating='post')
 start_probas = np.zeros(( ct, max_len_char), dtype=float)
 for i, p in enumerate(char_start_probas):
len_ = min(len(p), max_len_char)
 start_probas[i, :len_] = p[:len_]
start_probas=np.expand_dims(start_probas, axis=2)
 end_probas = np.zeros(( ct, max_len_char), dtype=float)
 for i, p in enumerate(char_end_probas):
len_ = min(len(p), max_len_char)
 end_probas[i, :len_] = p[:len_]
end_probas=np.expand_dims(end_probas, axis=2)
 sentiments_list = ['positive', 'neutral', 'negative']
texts = df['text'].values
selected_texts = df['selected_text'].values if train else [''] * len(df)
 sentiments = df['sentiment'].values
sentiments_input = [[sentiments_list.index(s)]*max_len_char for s in sentiments]
if train:
start_idx = np.zeros(( ct,max_len_char),dtype='int32')
 end_idx = np.zeros(( ct,max_len_char),dtype='int32')
 for i,(text, sel_text)in enumerate(zip(df['text'].values, df['selected_text'].values)) :
start, end = get_start_end_string(text, sel_text.strip())
 start_idx[i,start]=1
end_idx[i,end]=1
else:
start_idx = [0] * len(df)
 end_idx = [0] * len(df)
 dataset= {
'ids': X,
'probas_start': start_probas,
'probas_end': end_probas,
'target_start': start_idx,
'target_end': end_idx,
'text': texts,
'selected_text': selected_texts,
'sentiment': sentiments,
'sentiment_input': np.array(sentiments_input),
}
return dataset
y=train["Survived"]<choose_model_class>
char_dataset_train=load_data_second_model(train_df,
X_train,
probas_train_start,
probas_train_end,
max_len_char=args.max_char_length,
train=True)
 char_dataset_test=load_data_second_model(test_df,
X_test,
probas_test_start,
probas_test_end,
max_len_char=args.max_char_length,
train=False )
kfold = StratifiedKFold(n_splits=10 )<choose_model_class>
LABEL_SMOOTHING=0.1
def loss_fn_2(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
classifier=XGBClassifier()
parameters = {'nthread':[4],
'objective':['binary:logistic'],
'learning_rate': [0.05],
'max_depth': [6],
'min_child_weight': [11],
'silent': [1],
'subsample': [0.8],
'colsample_bytree': [0.7],
'n_estimators': [5],
'missing':[-999],
'seed': [1337]}
clf=GridSearchCV(classifier,parameters, n_jobs=5,cv=kfold,scoring='roc_auc',verbose=2, refit=True)
 clf.fit(train_set,y)
 best_parameters = clf.best_score_
test=clf.predict(test)
<prepare_x_and_y>
class ConvBlock(L.Layer):
def __init__(self,out_dim,kernel_size,padding="same"):
super(ConvBlock, self ).__init__()
self.conv=L.Conv1D(out_dim,kernel_size, padding=padding)
 self.bn=L.BatchNormalization()
def call(self, inputs):
x=self.conv(inputs)
 x=self.bn(x)
 x=tf.keras.activations.relu(x)
 return x
y_pred= test<create_dataframe>
class Logits(L.Layer):
def __init__(self,dim1=32,dim2=2):
super(Logits, self ).__init__()
self.dense1=L.Dense(dim1,activation='relu')
 self.dense2=L.Dense(dim2)
 def call(self, inputs):
x=self.dense1(inputs)
 x=self.dense2(x)
 return x
y_pred=pd.DataFrame(data=y_pred )<save_to_csv>
class CharCNN(tf.keras.Model):
def __init__(self,len_voc,cnn_dim=32, char_embed_dim=16, sent_embed_dim=16,
proba_cnn_dim=16, kernel_size=3,max_len_char=args.max_char_length):
super(CharCNN, self ).__init__()
self.CharEmbedding = L.Embedding(input_dim=len_voc, output_dim=char_embed_dim)
 self.SentimentEmbedding = L.Embedding(input_dim=3, output_dim=sent_embed_dim)
 self.ProbasCNN = ConvBlock(proba_cnn_dim,kernel_size)
 self.CNN1=ConvBlock(cnn_dim,kernel_size)
 self.CNN2=ConvBlock(cnn_dim*2,kernel_size)
 self.CNN3=ConvBlock(cnn_dim*4,kernel_size)
 self.CNN4=ConvBlock(cnn_dim*8,kernel_size)
 self.Dropout = L.Dropout(0.5)
 self.dense_logits=Logits(32,2)
 def call(self, inputs):
char_ids,sentiment,start_probas,end_probas=inputs
probas = tf.concat([start_probas, end_probas], -1)
 probas_fts = self.ProbasCNN(probas)
 char_fts = self.CharEmbedding(char_ids)
 sentiment_fts = self.SentimentEmbedding(sentiment)
 x = tf.concat([char_fts, sentiment_fts, probas_fts], -1)
 features = self.CNN1(x)
 features = self.CNN2(features)
 features = self.CNN3(features)
 features = self.CNN4(features)
 logits=self.Dropout(features)
 logits=self.dense_logits(logits)
 start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]
start_logits= tf.keras.activations.softmax(start_logits)
 end_logits= tf.keras.activations.softmax(end_logits)
 return start_logits,end_logits
submission = pd.read_csv(".. /input/gender_submission.csv")
 submission['Survived'] = y_pred
submission.to_csv('.. /working/submit.csv',index=False )<load_from_csv>
inpTest=[char_dataset_test['ids'],
char_dataset_test['sentiment_input'],
char_dataset_test['probas_start'],
char_dataset_test['probas_end']]
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
n_splits=args.cv_splits_2
char_pred_oof_start=np.zeros(( ct_train,args.max_char_length))
char_pred_oof_end=np.zeros(( ct_train,args.max_char_length))
char_pred_test_start=np.zeros(( ct_test,args.max_char_length))
char_pred_test_end=np.zeros(( ct_test,args.max_char_length))
jac_2=[]
splits = list(StratifiedKFold(n_splits=n_splits ).split(X=train_df, y=train_df['sentiment']))
for i,(train_idx, val_idx)in enumerate(splits):
print("-----------------------------------------------------------------------")
 print(f"start train fold---{i+1}")
 inp_train=[char_dataset_train['ids'][train_idx],
char_dataset_train['sentiment_input'][train_idx],
char_dataset_train['probas_start'][train_idx],
char_dataset_train['probas_end'][train_idx]]
target_train=[char_dataset_train['target_start'][train_idx],char_dataset_train['target_end'][train_idx]]
inp_val=[char_dataset_train['ids'][val_idx],
char_dataset_train['sentiment_input'][val_idx],
char_dataset_train['probas_start'][val_idx],
char_dataset_train['probas_end'][val_idx]]
target_val=[char_dataset_train['target_start'][val_idx],char_dataset_train['target_end'][val_idx]]
model=CharCNN(len_voc)
 optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate_2)
 model.compile(loss=loss_fn_2,optimizer=optimizer)
 model.fit(inp_train, target_train, epochs=args.epoches_2, batch_size=128, verbose=1,
validation_data=(inp_val,target_val),
callbacks=[])
 print('preditc the oof')
 pred_val_start,pred_val_end=model.predict(inp_val,batch_size=128, verbose=1, callbacks=[])
 char_pred_oof_start[val_idx]=pred_val_start
char_pred_oof_end[val_idx]=pred_val_end
print("calcuate the jaccard for valset")
 fold_jac=[]
for k in val_idx:
a=np.argmax(char_pred_oof_start[k,])
 b=np.argmax(char_pred_oof_end[k,])
 orig_text=char_dataset_train['text'][k,]
orig_selected=char_dataset_train['selected_text'][k,]
if a>b:
pred_selected=orig_text
else:
pred_selected=orig_text[a:b+1]
jac_val=jaccard(pred_selected,orig_selected)
 fold_jac.append(jac_val)
 jac_2.append(fold_jac)
 print("fold jac is:", np.mean(fold_jac))
print('predict the testset')
 pred_test_start,pred_test_end=model.predict(inpTest,batch_size=128, verbose=1, callbacks=[])
 char_pred_test_start+=pred_test_start/n_splits
char_pred_test_end+=pred_test_end/n_splits
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<concatenate>
print(f'The mean Jaccard value is {np.mean(jac_2)}' )
all_X = pd.concat(( train.loc[:, 'MSSubClass':'SaleCondition'],
test.loc[:, 'MSSubClass':'SaleCondition']))<feature_engineering>
def convert_prob_to_string(dataset, pred_start, pred_end):
ct=len(dataset['text'])
 pred=[]
for k in range(ct):
start_idx=np.argmax(pred_start[k])
 end_idx=np.argmax(pred_end[k])
 if start_idx>end_idx:
pred_selected_text=dataset['text'][k]
else:
pred_selected_text=dataset['text'][k][start_idx:end_idx+1]
pred.append(pred_selected_text.strip())
 return pred
<categorify><EOS> 
pred=convert_prob_to_string(char_dataset_test,char_pred_test_start, char_pred_test_end)
 test_df['selected_text']=pred
test_df.to_csv('submission.csv',columns=['textID','selected_text'], index=False)