Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<count_missing_values>
train = pd.read_csv(".. /input/train.csv", index_col = "PassengerId")
 test = pd.read_csv(".. /input/test.csv", index_col="PassengerId" )
def miss_val(df):
total=df.isnull().sum()
return pd.concat([total],axis=1,keys=['Total'])
 print("Missing values for train dataset
")
 print(miss_val(train))
print("---------------------------------------------------------------------")
 print("Missing values for test dataset
")
 print(miss_val(test))<correct_missing_values>
%matplotlib inline
train=train.dropna()
train.shape<feature_engineering>
low_low_fare = train[train["Fare"] < 100]
train['target']=train['selected_text'].str.lower()<feature_engineering>
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
train[["FamilySize"]].head()
def find_link(string):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return "".join(url)
 train['target_url']=train['target'].apply(lambda x: find_link(x))
df=pd.DataFrame(train.loc[train['target_url']!=""]['sentiment'].value_counts() ).reset_index()
df.rename(columns={"index": "sentiment", "sentiment": "url_count"} )<string_transform>
train.loc[train["FamilySize"]==1, "FamilyType"] = "Single"
train.loc[(train["FamilySize"] > 1)&(train["FamilySize"] < 5), "FamilyType"] = "Nuclear"
train.loc[train["FamilySize"] >=5, "FamilyType"] = "Big"
train[["FamilySize", "FamilyType"]].head()
def find_punct(text):
line = re.findall(r'[!"\$%&'() *+,\-.\/:;=
string="".join(line)
 return list(string )<feature_engineering>
pd.pivot_table(data=train, index="FamilyType", values="Survived" )
train['target_punct']=train['target'].apply(lambda x:find_punct(x))
train['target_punct_len']=train['target'].apply(lambda x:len(find_punct(x)) )<create_dataframe>
train.loc[train["Name"].str.contains("Mr"), "Title"] = "Mr"
train.loc[train["Name"].str.contains("Miss"), "Title"] = "Miss"
train.loc[train["Name"].str.contains("Mrs"), "Title"] = "Mrs"
train.loc[train["Name"].str.contains("Master"), "Title"] = "Master"
train[["Name", "Title"]].head()
punc_df=pd.DataFrame(train,columns=['target_punct','sentiment'])
 punc_df=punc_df[punc_df['target_punct'].map(lambda d: len(d)) > 0]
punc_df=punc_df.explode('target_punct')
 positive_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="positive"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'pos_punct'})
 negative_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="negative"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neg_punct'})
 neutral_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="neutral"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neut_punct'})
<count_values>
train.loc[train["Sex"] == "male", "Sex_encode"] = 0
train.loc[train["Sex"] == "female", "Sex_encode"] = 1
train[["Sex", "Sex_encode"]].head()
train['star']=train['target'].apply(lambda x:find_star(x))
train.loc[train['star']!=0]['sentiment'].value_counts().to_frame()<feature_engineering>
test.loc[test["Sex"] == "male", "Sex_encode"] = 0
test.loc[test["Sex"] == "female", "Sex_encode"] = 1
test[["Sex", "Sex_encode"]].head()
def find_only_star(text):
if len(text.split())==1:
line=re.findall(r'[*]{2,5}',text)
 return len(line)
 else:
return 0<count_values>
train[train["Fare"].isnull() ]
train['only_star']=train['target'].apply(lambda x:find_only_star(x))
train.loc[train['only_star']==1]['sentiment'].value_counts()<feature_engineering>
test[test["Fare"].isnull() ]
train['target']= np.where(train['only_star']==1,"abusive",train['target'] )<string_transform>
train["Fare_fillin"] = train["Fare"]
test["Fare_fillin"] = test["Fare"]
def remove_link(string):
text = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +'," ",string)
 return " ".join(text.split() )<string_transform>
test.loc[test["Fare"].isnull() , "Fare_fillin"] = 0
def remove_punct(text):
line = re.sub(r'[!"\$%&'() *+,\-.\/:;=
return " ".join(line.split() )<feature_engineering>
train["Fare_fillin"] = train["Fare_fillin"] / 10.0
test["Fare_fillin"] = test["Fare_fillin"] / 10.0
train['target']=train['target'].apply(lambda x:remove_link(x))
train['target']=train['target'].apply(lambda x:remove_punct(x))<string_transform>
train["Embarked"].fillna("S" )
def create_corpus(data,feature,sentiment):
corpus=[]
for x in data[data['sentiment']==sentiment][feature].str.split() :
for i in x:
corpus.append(i)
 return corpus<prepare_x_and_y>
train["Embarked_C"] = False
train.loc[train["Embarked"]=='C', "Embarked_C"] = True
train["Embarked_S"] = False
train.loc[train["Embarked"]=='S', "Embarked_S"] = True
train["Embarked_Q"] = False
train.loc[train["Embarked"]=='Q', "Embarked_Q"] = True
train[["Embarked", "Embarked_C", "Embarked_S", "Embarked_Q"]].head()
def corpus_sentiment(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word not in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y<count_values>
test["Embarked_C"] = False
test.loc[test["Embarked"]=='C', "Embarked_C"] = True
test["Embarked_S"] = False
test.loc[test["Embarked"]=='S', "Embarked_S"] = True
test["Embarked_Q"] = False
test.loc[test["Embarked"]=='Q', "Embarked_Q"] = True
test[["Embarked", "Embarked_C", "Embarked_S", "Embarked_Q"]].head()
def count_values(df,feature):
total=df.loc[:,feature].value_counts(dropna=False)
 percent=round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'])
<count_values>
train["Age"].fillna(train["Age"].mean() , inplace=True)
 test["Age"].fillna(test["Age"].mean() , inplace=True )
count_values(train,'sentiment' )<count_values>
train["Child"] = False
train.loc[train["Age"] < 15, "Child"] = True
train[["Age", "Child"]].head(10 )
count_values(train,'sentiment' )<concatenate>
test["Child"] = False
test.loc[test["Age"] < 15, "Child"] = True
test[["Age", "Child"]].head(10 )
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape<feature_engineering>
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1
full_data['text_punct']=full_data['text'].apply(lambda x:find_punct(x))
full_data['text_punct_len']=full_data['text'].apply(lambda x:len(find_punct(x)) )<create_dataframe>
train["Single"] = False
train.loc[train["FamilySize"]==1, "Single"] = True
train["Nuclear"] = False
train.loc[(train["FamilySize"]>1)&(train["FamilySize"]<5), "Nuclear"] = True
train["Big"] = False
train.loc[train["FamilySize"] >=5, "Big"] = True
train[["FamilySize", "Single", "Nuclear", "Big"]].head(10 )
punc_text_df=pd.DataFrame(full_data,columns=['text_punct','sentiment'])
 punc_text_df=punc_text_df[punc_text_df['text_punct'].map(lambda d: len(d)) > 0]
punc_text_df=punc_text_df.explode('text_punct')
 positive_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="positive"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'pos_punct'})
 negative_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="negative"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neg_punct'})
 neutral_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="neutral"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neut_punct'})
<feature_engineering>
test["Single"] = False
test.loc[test["FamilySize"]==1, "Single"] = True
test["Nuclear"] = False
test.loc[(test["FamilySize"]>1)&(test["FamilySize"]<5), "Nuclear"] = True
test["Big"] = False
test.loc[test["FamilySize"] >=5, "Big"] = True
test[["FamilySize", "Single", "Nuclear", "Big"]].head(10 )
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))<filter>
train["Master"] = False
train.loc[train["Name"].str.contains("Master"), "Master"] = True
train[["Name", "Master"]].head(10 )
full_data.loc[full_data['text']=="",['text']]="nothing"<prepare_x_and_y>
test["Master"] = False
test.loc[test["Name"].str.contains("Master"), "Master"] = True
test[["Name", "Master"]].head(10 )
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y 
<define_variables>
feature_names = ["Pclass", "Sex_encode", "Fare_fillin", "Embarked_C", "Embarked_S", "Embarked_Q", "Child", "Single", "Nuclear", "Big", "Master"]
feature_names
train_word=full_data[:train.shape[0]]
test_word=full_data[train.shape[0]:]<feature_engineering>
label_name = "Survived"
label_name
pos_text_list=list(corpus_sentiment(train_word,'text','positive')[0])
 pos_target_list=list(corpus_sentiment(train_word,'target','positive')[0])
 neg_text_list=list(corpus_sentiment(train_word,'text','negative')[0])
 neg_target_list=list(corpus_sentiment(train_word,'target','negative')[0])
 neutral_text_list=list(corpus_sentiment(train_word,'text','neutral')[0])
 neutral_target_list=list(corpus_sentiment(train_word,'target','neutral')[0])
 pos_common_words_list=list(set(pos_text_list ).intersection(pos_target_list))
neg_common_words_list=list(set(neg_text_list ).intersection(neg_target_list))
neutral_common_words_list=list(set(neutral_text_list ).intersection(neutral_target_list))<compute_train_metric>
random_seed=0
Y = train[label_name]
X = train[feature_names]
X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size=0.2, random_state=random_seed )
results_jaccard=[]
for ind,row in train_word.iterrows() :
sentence1 = row.text
sentence2 = row.target
jaccard_score = jaccard(sentence1,sentence2)
 results_jaccard.append([sentence1,sentence2,jaccard_score] )<merge>
X_test = test[feature_names]
jaccard = pd.DataFrame(results_jaccard,columns=["text","target","jaccard_score"])
 train_word = train_word.merge(jaccard,how='outer' )<compute_test_metric>
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier , GradientBoostingClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import Imputer , Normalizer , scale
from sklearn.cross_validation import train_test_split , StratifiedKFold
from sklearn.feature_selection import RFECV
print("Jaccard score equals 1 : {}%".format(round(train_word.loc[train_word['jaccard_score']==1].shape[0]/train_word.shape[0]*100,2)))
 print("Jaccard score less than 0.3 : {}%".format(round(train_word.loc[train_word['jaccard_score']<0.3].shape[0]/train_word.shape[0]*100,2)) )<feature_engineering>
rforest_model = RandomForestClassifier(n_estimators=100 )
train_word['difference']=abs(train_word['text_tweet_length']-train_word['target_tweet_length'] )<define_variables>
svc = SVC()
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<load_from_csv>
knn = KNeighborsClassifier(n_neighbors = 3 )
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )<string_transform>
gbmodel = GradientBoostingClassifier(max_depth=12 )
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1 
<load_from_csv>
gnb = GaussianNB()
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1 
<load_pretrained>
LR = LogisticRegression()
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('relu' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model 
<predict_on_test>
mlp = MLPClassifier(solver='lbfgs', random_state=0 )
start_time=dt.now()
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
end_time=dt.now()
print(" ")
 print(" ")
 print("Time Taken to run above code :",(end_time-start_time ).total_seconds() /60," minutes" )<string_transform>
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
x_train = X_train.values
x_val = X_val.values
test_nn = test[feature_names].values
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )<load_from_csv>
model = Sequential()
batch_size = 32
epochs = 200
model.add(Dense(32, activation="relu", input_dim=11))
model.add(Dense(32, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(16, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(1, activation="sigmoid"))
model.compile(optimizer="rmsprop", loss="binary_crossentropy", metrics=["accuracy"] )
train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv' ).fillna('')
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' ).fillna('' )<string_transform>
model.fit(x_train,y_train, epochs=epochs, batch_size=batch_size )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
test_loss, test_acc = model.evaluate(x_val, y_val)
 print('Test Score:{}'.format(test_acc))
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<categorify>
nn_predict = model.predict(test_nn )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(str(train.loc[k,'text'] ).split())
 text2 = " ".join(str(train.loc[k,'selected_text'] ).split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
rforest_model.fit(X_train, y_train)
 svc.fit(X_train, y_train)
 knn.fit(X_train, y_train)
 gbmodel.fit(X_train, y_train)
 gnb.fit(X_train, y_train)
 LR.fit(X_train, y_train)
 mlp.fit(X_train, y_train )
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
score = []
score.append(rforest_model.score(X_val, y_val))
score.append(svc.score(X_val, y_val))
score.append(knn.score(X_val, y_val))
score.append(gbmodel.score(X_val, y_val))
score.append(gnb.score(X_val, y_val))
score.append(LR.score(X_val, y_val))
score.append(mlp.score(X_val,y_val))
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<define_variables>
score = []
score.append(rforest_model.score(X_train, y_train))
score.append(svc.score(X_train, y_train))
score.append(knn.score(X_train, y_train))
score.append(gbmodel.score(X_train, y_train))
score.append(gnb.score(X_train, y_train))
score.append(LR.score(X_train, y_train))
score.append(mlp.score(X_train,y_train))
jac = []; VER='v0'; DISPLAY=2
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
FOLDS = 5
skf = StratifiedKFold(n_splits=FOLDS,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
print('Loading model...')
 model.load_weights('/kaggle/input/robertaweights1/%s-roberta-%i.h5'%(VER,fold))
print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits 
<feature_engineering>
x = X.values
y = Y.values
<string_transform>
model.fit(x,y, epochs=epochs, batch_size=batch_size )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
gbmodel.fit(X, Y )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
predictions = gbmodel.predict(X_test )
import os
import torch
import pandas as pd
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
import transformers
import tokenizers
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from tqdm.autonotebook import tqdm
import utils<define_variables>
submission = pd.read_csv('.. /input/gender_submission.csv')
 submission['Survived'] = predictions
<string_transform><EOS> 
submission.to_csv('./simpletitanic.csv', index=False )