Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<statistical_test>
%matplotlib inline
py.init_notebook_mode(connected=True)
 GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier)
 warnings.filterwarnings('ignore' )
def get_top_ngram(corpus, n):
vec = CountVectorizer(ngram_range=(n,n), analyzer='word', max_features=5000)
 bag_of_words = vec.fit_transform(train_df[train_df['sentiment']==corpus]['c_s_text'])
 sum_words=bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0,idx])for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key = lambda x:x[1], reverse=True)
 return words_freq[:20]
top_pos=get_top_ngram('positive',3)
 top_neu=get_top_ngram('neutral',3)
 top_neg=get_top_ngram('negative',3)
 print('Top Positive words: ',top_pos)
 print('Top Neutral words: ', top_neu)
 print('Top Negative words: ', top_neg )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(50 )
max_len = 150
vocab_path = '/kaggle/input/roberta-base/vocab.json'
merge_path = '/kaggle/input/roberta-base/merges.txt'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = vocab_path,
merges_file = merge_path,
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0], 'negative':tokenizer.encode('negative' ).ids[0], 'neutral':tokenizer.encode('neutral' ).ids[0]}
train_df.reset_index(inplace=True )<define_variables>
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col],25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,["Age","SibSp","Parch","Fare"])
 train.loc[Outliers_to_drop]
tot_tw = train_df.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_df.loc[i,'text'].split())
 set2 = " ".join(train_df.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_df.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1<categorify>
tot_test_tw = test_df.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_df.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_df.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1<load_pretrained>
for dataset in full_data:
dataset.loc[ dataset['Name_length'] <= 23, 'Name_length'] 						 = 0
dataset.loc[(dataset['Name_length'] > 23)&(dataset['Name_length'] <= 28), 'Name_length'] = 1
dataset.loc[(dataset['Name_length'] > 28)&(dataset['Name_length'] <= 40), 'Name_length'] = 2
dataset.loc[ dataset['Name_length'] > 40, 'Name_length'] 							 = 3
train['Name_length'].value_counts()
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/prerobertabase/config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained('/kaggle/input/prerobertabase/pretrained-roberta-base.h5', config=config_path)
 x = bert_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.LSTM(1024, return_sequences=True )(x1)
 x1 = tf.keras.layers.Conv1D(128,2, padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(16,2, padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.LSTM(1024, return_sequences=True )(x2)
 x2 = tf.keras.layers.Conv1D(128,2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(16,2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids,att,tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 dataset.loc[ dataset['Age'] <= 14, 'Age'] 						 = 0
dataset.loc[(dataset['Age'] > 14)&(dataset['Age'] <= 30), 'Age'] = 5
dataset.loc[(dataset['Age'] > 30)&(dataset['Age'] <= 40), 'Age'] = 1
dataset.loc[(dataset['Age'] > 40)&(dataset['Age'] <= 50), 'Age'] = 3
dataset.loc[(dataset['Age'] > 50)&(dataset['Age'] <= 60), 'Age'] = 2
dataset.loc[ dataset['Age'] > 60, 'Age'] 							 = 4
train['Age'].value_counts()
jac =[]; DISPLAY=1
preds_start= np.zeros(( input_ids_t.shape[0],max_len))
preds_end= np.zeros(( input_ids_t.shape[0],max_len))
model = build_model()
print('Loading model.... ')
 model.load_weights('/kaggle/input/preroberta8/pre-roberta-v10.h5')
 print('-'*15)
 print('- RESULT -')
 print('-'*15)
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start = preds[0]
preds_end = preds[1]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_df['selected_text']=all
test_df[['textID','selected_text']].to_csv('submission.csv', index=False )<import_modules>
train[["Age", "Survived"]].groupby(['Age'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold
from transformers import *<import_modules>
for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 2.7, 'Fare'] 						 = 0
dataset.loc[ dataset['Fare'] > 2.7, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train['Fare'].value_counts()
print('TF version',tf.__version__ )<define_variables>
for dataset in full_data:
dataset['Has_Cabin'] = dataset["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 train[["Has_Cabin", "Survived"]].groupby(['Has_Cabin'], as_index=False ).sum().sort_values(by='Survived', ascending=False )
SEED = 4262
np.random.seed(SEED)
 tf.random.set_seed(SEED )<set_options>
train[["Has_Cabin", "Survived"]].groupby(['Has_Cabin'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
USE_TPU = False
if USE_TPU:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.OneDeviceStrategy(device = "/gpu:0" )<define_variables>
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_pivot = pd.pivot_table(train, values= 'Survived',index=['Embarked'],columns='Pclass',aggfunc=np.mean, margins=True)
 def color_negative_red(val):
color = 'red' if val < 0.4 else 'black'
return 'color: %s' % color
train_pivot = train_pivot.style.applymap(color_negative_red)
 train_pivot
MAX_LEN = 128
FOLDS = 5
EPOCHS = 3
VERBOSE = 1
ROBERTA_BASE_PATH = '.. /input/tf-roberta/'
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
print('BatchSize: {}'.format(BATCH_SIZE))
LR = 1e-5 * strategy.num_replicas_in_sync
print('LearningRate: {}'.format(LR))
INFERENCE = True<load_from_csv>
dataset['Embarked'] = dataset['Embarked'].replace(['0', '2'], '0')
 train['Fare'].value_counts()
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv', nrows = 27200 ).fillna('')
 train.head()<load_from_csv>
for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Mrs', 'Miss'], 'MM')
 dataset['Title'] = dataset['Title'].replace(['Dr', 'Major', 'Col'], 'DMC')
 dataset['Title'] = dataset['Title'].replace(['Don', 'Rev', 'Capt', 'Jonkheer'],'DRCJ')
 dataset['Title'] = dataset['Title'].replace(['Mme', 'Ms', 'Lady', 'Sir', 'Mlle', 'Countess'],'MMLSMC')
 title_mapping = {"MM": 1, "Master":2, "Mr": 5, "DMC": 4, "DRCJ": 3, "MMLSMC": 0}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(3)
 g = sns.FacetGrid(train, col='Survived')
 g = g.map(sns.distplot, "Age" )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 test.head()<define_variables>
train[["Title", "Survived"]].groupby(['Title'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
tokenizer = RobertaTokenizer(vocab_file = ROBERTA_BASE_PATH + 'vocab-roberta-base.json',
merges_file = ROBERTA_BASE_PATH + 'merges-roberta-base.txt',
add_prefix_space = True,
do_lower_case = False )<define_variables>
deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
for dataset in full_data:
dataset['Cabin'] = dataset['Cabin'].fillna("U0")
 dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 dataset['Deck'] = dataset['Deck'].map(deck)
 dataset['Deck'] = dataset['Deck'].fillna(0)
 dataset['Deck'] = dataset['Deck'].astype(int)
 train['Deck'].value_counts()
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN), dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN), dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN), dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN), dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN), dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 if text1 != '':
input_encoded = tokenizer.encode_plus(text1, train.loc[k,'sentiment'], add_special_tokens = True, max_length = MAX_LEN, padding = True)
 input_ids_sample = input_encoded["input_ids"]
attention_mask_sample = input_encoded["attention_mask"]
attention_mask[k,:len(attention_mask_sample)] = attention_mask_sample
input_ids[k,:len(input_ids_sample)] = input_ids_sample
idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx + len(text2)] = 1
k_ids = tokenizer.encode(text1, add_special_tokens = False)
 offsets = []
idx = 0
for t in k_ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 if len(toks)> 0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
for dataset in full_data:
dataset.loc[ dataset['Deck'] <= 1, 'Deck'] = 1
dataset.loc[(dataset['Deck'] > 1)&(dataset['Deck'] <= 6), 'Deck'] = 3
dataset.loc[ dataset['Deck'] > 6, 'Deck'] = 0
train[["Deck", "Survived"]].groupby(['Deck'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(test.loc[k,'text'].split())
 if text1 != '':
input_encoded = tokenizer.encode_plus(text1, test.loc[k, 'sentiment'], add_special_tokens = True, max_length = MAX_LEN, padding = True)
 input_ids_sample = input_encoded["input_ids"]
attention_mask_sample = input_encoded["attention_mask"]
attention_mask_t[k,:len(attention_mask_sample)] = attention_mask_sample
input_ids_t[k,:len(input_ids_sample)] = input_ids_sample<string_transform>
train[['Pclass', 'Sex', 'Age', 'Parch', 'Fare', 'Embarked', 'Has_Cabin', 'FamilySize', 'Title', 'Survived']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Pclass', ascending=False )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<load_pretrained>
config = RobertaConfig.from_pretrained(ROBERTA_BASE_PATH+'config-roberta-base.json')
 config.attention_probs_dropout_prob = 0.15
print(config )<choose_model_class>
train[["FamilySize", "Survived"]].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def build_model() :
with strategy.scope() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 roberta_model = TFRobertaForQuestionAnswering.from_pretrained(ROBERTA_BASE_PATH + 'pretrained-roberta-base.h5', config = config)
 x = roberta_model(ids, attention_mask = att, token_type_ids = tok)
 x1 = tf.keras.layers.Activation('softmax' )(x[0])
 x2 = tf.keras.layers.Activation('softmax' )(x[1])
 model = tf.keras.models.Model(inputs = [ids, att, tok], outputs=[x1, x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate = LR)
 loss = tf.keras.losses.CategoricalCrossentropy(from_logits = False, label_smoothing = 0.1)
 model.compile(loss = loss, optimizer = optimizer)
 return model<define_variables>
for dataset in full_data:
dataset['Gender_Embarked'] = 0
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==0), 'Gender_Embarked'] = 0
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==2), 'Gender_Embarked'] = 1
dataset.loc[(dataset['Sex']==0)&(dataset['Embarked']==1), 'Gender_Embarked'] = 2
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==2), 'Gender_Embarked'] = 3
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==0), 'Gender_Embarked'] = 4
dataset.loc[(dataset['Sex']==1)&(dataset['Embarked']==1), 'Gender_Embarked'] = 5
train[["Gender_Embarked", "Survived"]].groupby(['Gender_Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
jac, jac1 = [], []
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits = FOLDS, shuffle = True, random_state = SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
model_checkpoint = tf.keras.callbacks.ModelCheckpoint('roberta-%i.h5'%(fold), monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto', save_freq = 'epoch')
 if not INFERENCE:
model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs = EPOCHS,
batch_size = BATCH_SIZE,
verbose = VERBOSE,
callbacks = [model_checkpoint],
validation_data =([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]], [start_tokens[idxV,], end_tokens[idxV,]]),
shuffle = True)
 print('Loading model...')
 model.load_weights(f'roberta-{fold}.h5')
 else:
print('Loading model...')
 model.load_weights(f'/kaggle/input/tensorflow-roberta-qa-model/roberta-{fold}.h5')
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose = VERBOSE)
 print('Predicting Test...')
 preds = model.predict([input_ids_t, attention_mask_t, token_type_ids_t], verbose = VERBOSE)
 preds_start += preds[0] / skf.n_splits
preds_end += preds[1] / skf.n_splits
all = []
all1 = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1, add_special_tokens = False)
 if train.loc[k,'sentiment'] == 'neutral':
st = " ".join(train.loc[k,'text'].split())
 st1 = " ".join(train.loc[k,'text'].split())
 else:
if a > b:
st = text1
st1 = text1
start_sort = np.argsort(oof_start[k,])[::-1]
end_sort = np.argsort(oof_end[k,])[::-1]
a1 = start_sort[1]
b1 = end_sort[1]
a2 = start_sort[2]
b2 = end_sort[2]
if a1 < b1:
st1 = tokenizer.decode(enc[a1-1:b1])
 elif a2 < b2:
st1 = tokenizer.decode(enc[a2-1:b2])
 else:
st = tokenizer.decode(enc[a-1:b])
 st1 = tokenizer.decode(enc[a-1:b])
 all.append(jaccard(st, train.loc[k,'selected_text']))
all1.append(jaccard(st1, train.loc[k,'selected_text']))
jac.append(np.mean(all))
jac1.append(np.mean(all1))
print('>>>> FOLD %i Jaccard ='%(fold+1), np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1), np.mean(all1))
print()<feature_engineering>
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp', 'Boys', 'IsAlone', 'Embarked']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )
print(f'=== OVERALL 5 Fold CV Jaccard = {np.mean(jac)}')
 print(f'=== OVERALL 5 Fold CV Jaccard - Post Processed = {np.mean(jac1)}' )<string_transform>
X_train = train.drop("Survived", axis=1)
 Y_train = train["Survived"]
X_train.shape, Y_train.shape, test.shape
X_train, x_test, Y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=101)
 X_test = test.copy()
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1, add_special_tokens = False)
 if test.loc[k, 'sentiment'] == 'neutral':
st = " ".join(test.loc[k,'text'].split())
 else:
if a > b:
st = text1
start_sort = np.argsort(preds_start[k,])[::-1]
end_sort = np.argsort(preds_end[k,])[::-1]
a1 = start_sort[1]
b1 = end_sort[1]
a2 = start_sort[2]
b2 = end_sort[2]
if a1 < b1:
st = tokenizer.decode(enc[a1-1:b1])
 elif a2 < b2:
st = tokenizer.decode(enc[a2-1:b2])
 else:
st = tokenizer.decode(enc[a-1:b])
 all.append(st )<save_to_csv>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred1 = logreg.predict(x_test)
 acc_log = round(logreg.score(x_test, y_test)* 100, 2)
 acc_log
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
print(classification_report(y_test, Y_pred1))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred1), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
!pip install torchcontrib 
<set_options>
svc=SVC()
svc.fit(X_train, Y_train)
 Y_pred2 = svc.predict(x_test)
 acc_svc = round(svc.score(x_test, y_test)* 100, 2)
 acc_svc
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<define_variables>
print(classification_report(y_test, Y_pred2))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred2), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE=32
EPOCHS=10
BERT_PATH='bert-base-uncased'
TRAINING_FILE = '/kaggle/input/tweet-sentiment-extraction/train.csv'
SAMPLE_FILE ='/kaggle/input/tweet-sentiment-extraction/sample_submission.csv'
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 16
EPOCHS = 5
BERT_PATH = ".. /input/roberta-base"

 TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{BERT_PATH}/vocab.json",
merges_file=f"{BERT_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)<import_modules>
print(classification_report(y_test, knn_predictions))
cm = pd.DataFrame(confusion_matrix(y_test, knn_predictions), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
def param_getter(m): return m.parameters()
def listify(o):
if o is None : return []
if isinstance(o,list): return o
if isinstance(o,str): return [o]
if isinstance(o,Iterable): return list(o)
 return [o]
class DataBunch() :
def __init__(self, train_dl, valid_dl, c=None):
self.train_dl,self.valid_dl,self.c = train_dl,valid_dl,c
@property
def train_ds(self): return self.train_dl.dataset
@property
def valid_ds(self): return self.valid_dl.dataset
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()
class Callback() :
_order=0
def set_runner(self, run): self.run=run
def __getattr__(self, k): return getattr(self.run, k)
 @property
def name(self):
name = re.sub(r'Callback$', '', self.__class__.__name__)
 return camel2snake(name or 'callback')
 def __call__(self, cb_name):
f = getattr(self, cb_name, None)
 if f and f() : return True
return False
class AvgStatsCallback(Callback):
def __init__(self, metrics):
self.train_stats,self.valid_stats = AvgStats(metrics,True),AvgStats(metrics,False)
 def begin_epoch(self):
self.train_stats.reset()
self.valid_stats.reset()
def after_loss(self):
stats = self.valid_stats if not self.in_train else self.train_stats
with torch.no_grad() : stats.accumulate(self.run)
 def after_epoch(self):
print(self.train_stats)
 print(self.valid_stats)
 class Recorder(Callback):
def begin_fit(self):
self.lrs = [[] for _ in self.opt.param_groups]
self.losses = []
def after_batch(self):
if not self.in_train: return
for pg,lr in zip(self.opt.param_groups,self.lrs): lr.append(pg['lr'])
 self.losses.append(self.loss.detach().cpu())
 def plot_lr(self, pgid=-1): plt.plot(self.lrs[pgid])
 def plot_loss(self, skip_last=0): plt.plot(self.losses[:len(self.losses)-skip_last])
 def plot(self, skip_last=0, pgid=-1):
losses = [o.item() for o in self.losses]
lrs = self.lrs[pgid]
n = len(losses)-skip_last
plt.xscale('log')
 plt.plot(lrs[:n], losses[:n])
 class ParamScheduler(Callback):
_order=1
def __init__(self, pname, sched_funcs): self.pname,self.sched_funcs = pname,sched_funcs
def begin_fit(self):
if not isinstance(self.sched_funcs,(list,tuple)) :
self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)
 def set_param(self):
assert len(self.opt.param_groups)==len(self.sched_funcs)
 for pg,f in zip(self.opt.param_groups,self.sched_funcs):
pg[self.pname] = f(self.n_epochs/self.epochs)
 def begin_batch(self):
if self.in_train: self.set_param()
class LR_Find(Callback):
_order=1
def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr
self.best_loss = 1e9
def begin_batch(self):
if not self.in_train: return
pos = self.n_iter/self.max_iter
lr = self.min_lr *(self.max_lr/self.min_lr)** pos
for pg in self.opt.param_groups: pg['lr'] = lr
def after_step(self):
if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
raise CancelTrainException()
if self.loss < self.best_loss: self.best_loss = self.loss
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs=0.
 self.run.n_iter=0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs += 1./self.iters
self.run.n_iter += 1
def begin_epoch(self):
self.run.n_epochs=self.epoch
self.model.train()
self.run.in_train=True
def begin_validate(self):
self.model.eval()
self.run.in_train=False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass
class Learner() :
def __init__(self, model, data, loss_func, optimizer, lr=1e-2, splitter=param_getter,
cbs=None, cb_funcs=None):
self.model,self.data,self.loss_func,self.lr,self.splitter = model,data,loss_func,lr,splitter
self.in_train,self.logger,self.opt = False,print,optimizer
self.cbs = []
self.add_cb(TrainEvalCallback())
 self.add_cbs(cbs)
 self.add_cbs(cbf() for cbf in listify(cb_funcs))
def add_cbs(self, cbs):
for cb in listify(cbs): self.add_cb(cb)
 def add_cb(self, cb):
cb.set_runner(self)
 setattr(self, cb.name, cb)
 self.cbs.append(cb)
 def remove_cbs(self, cbs):
for cb in listify(cbs): self.cbs.remove(cb)
 def one_batch(self, i, xb, yb):
try:
self.iter = i
self.xb,self.yb = xb,yb; self('begin_batch')
 self.pred = self.model(self.xb); self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb); self('after_loss')
 if not self.in_train: return
self.loss.backward() ; self('after_backward')
 self.opt.step() ; self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self):
self.iters = len(self.dl)
 try:
for i,(xb,yb)in enumerate(self.dl): self.one_batch(i, xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def do_begin_fit(self, epochs):
self.epochs,self.loss = epochs,torch.tensor(0.)
self('begin_fit')
 def do_begin_epoch(self, epoch):
self.epoch,self.dl = epoch,self.data.train_dl
return self('begin_epoch')
 def fit(self, epochs, cbs=None, reset_opt=False):
self.add_cbs(cbs)
 try:
self.do_begin_fit(epochs)
 for epoch in range(epochs):
if not self.do_begin_epoch(epoch): self.all_batches()
with torch.no_grad() :
self.dl = self.data.valid_dl
if not self('begin_validate'): self.all_batches()
self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.remove_cbs(cbs)
 ALL_CBS = {'begin_batch', 'after_pred', 'after_loss', 'after_backward', 'after_step',
'after_cancel_batch', 'after_batch', 'after_cancel_epoch', 'begin_fit',
'begin_epoch', 'begin_validate', 'after_epoch',
'after_cancel_train', 'after_fit'}
def __call__(self, cb_name):
res = False
assert cb_name in self.ALL_CBS
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res
def annealer(f):
def _inner(start,end): return partial(f,start,end)
 return _inner
@annealer
def sched_lin(start,end,pos): return start + pos*(end-start)
 @annealer
def sched_cos(start,end,pos):return start+(1+math.cos(math.pi*(1-pos)))*(end-start)/2
@annealer
def sched_no(start,end,pos): return start
@annealer
def sched_exp(start,end,pos): return start*(end/start)**pos
def cos_1cycle_anneal(start,high,end):
return [sched_cos(start,high),sched_cos(high,end)]
def combine_scheds(pcts, scheds):
assert sum(pcts)== 1.
 pcts = torch.tensor([0] + listify(pcts))
assert torch.all(pcts >= 0)
 pcts = torch.cumsum(pcts, 0)
 def _inner(pos):
idx =(pos >= pcts ).nonzero().max()
actual_pos =(pos-pcts[idx])/(pcts[idx+1]-pcts[idx])
 return scheds[idx](actual_pos)
 return _inner
class CudaCallback(Callback):
def begin_fit(self): self.model.cuda()
def begin_batch(self):
if type(self.run.xb)is dict:
for key in self.run.xb.keys() :
self.run.xb[key]=self.run.xb[key].cuda()
if type(self.run.yb)is dict:
for key in self.run.yb.keys() :
if type(self.run.yb[key])is not list:
self.run.yb[key]=self.run.yb[key].cuda()
class AvgStats() :
def __init__(self,metrics,in_train): self.metrics,self.in_train = listify(metrics),in_train
def reset(self):
self.tot_loss,self.count =0.,0.
 self.tot_mets = [0.]*len(self.metrics)
 @property
def all_stats(self):return [self.tot_loss.item() ]+ self.tot_mets
@property
def avg_stats(self): return [o/self.count for o in self.all_stats]
def __repr__(self):
if not self.count: return ''
return f"{'train' if self.in_train else 'valid'}: {self.avg_stats}"
def accumulate(self,run):
bn = run.xb[list(run.xb.keys())[0]].shape[0]
self.tot_loss+=run.loss*bn
self.count+=bn
if not self.in_train:
for i,m in enumerate(self.metrics):
self.tot_mets[i]+= m(run.pred,run.yb)*bn
class ProgressCallback(Callback):
_order=-1
def begin_fit(self):
self.mbar = master_bar(range(self.epochs))
self.mbar.on_iter_begin()
self.run.logger = partial(self.mbar.write, table=True)
 def after_fit(self): self.mbar.on_iter_end()
def after_batch(self): self.pb.update(self.iter)
 def begin_epoch(self): self.set_pb()
def begin_validate(self): self.set_pb()
def after_loss(self): self.pb.comment='loss= ' +str(self.loss.item())[:5]+' lr '+str(self.opt.param_groups[0]['lr'])[:9]
def set_pb(self):
self.pb = progress_bar(self.dl, parent=self.mbar)
 self.mbar.update(self.epoch)
 class EarlyStopingCallback(Callback):
_order=-1
def __init__(self,iters=1,path='bert.pth'):
self.iters=iters
self.bad_metrics =iters
self.path=path

 def begin_fit(self):
self.best_metric=[0]
def after_epoch(self):
mean_metric=self.avg_stats.valid_stats.avg_stats[1]
if mean_metric>self.best_metric:
self.bad_metrics=self.iters
self.best_metric=mean_metric
if self.path:
print('Saving.....',mean_metric)
 torch.save(self.run.model.state_dict() ,self.path)
 else:
self.bad_metrics-=1
if self.bad_metrics==0:
self.run.model.load_state_dict(torch.load(self.path))
raise CancelTrainException() 
<categorify>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred3 = gaussian.predict(x_test)
 acc_gaussian = round(gaussian.score(x_test, y_test)* 100, 2)
 acc_gaussian
class NLPTransform(BasicTransform):

 LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
@property
def targets(self):
return {"data": self.apply}
def update_params(self, params, **kwargs):
if hasattr(self, "interpolation"):
params["interpolation"] = self.interpolation
if hasattr(self, "fill_value"):
params["fill_value"] = self.fill_value
return params
def get_sentences(self, text, lang='en'):
return sent_tokenize(text, self.LANGS.get(lang, 'english'))
class ExcludeNumbersTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeNumbersTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text,sent, lang = data
text = re.sub(r'[0-9]', '', text)
 text = re.sub(r'\s+', ' ', text)
 sent = re.sub(r'[0-9]', '', sent)
 sent = re.sub(r'\s+', ' ', sent)
 return text,sent, lang
class ExcludeHashtagsTransform(NLPTransform):
Exclude @users Exclude urls<categorify>
print(classification_report(y_test, Y_pred3))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred3), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
def get_train_transforms(is_train):
if is_train:
return albumentations.Compose([
ExcludeNumbersTransform(p=0.0),
ExcludeHashtagsTransform(p=0.0),
ExcludeUsersMentionedTransform(p=0.0),
ExcludeUrlsTransform(p=0.0),
])
 else:
return albumentations.Compose([
ExcludeNumbersTransform(p=0),
ExcludeHashtagsTransform(p=0),
ExcludeUsersMentionedTransform(p=0),
ExcludeUrlsTransform(p=0),
] )<string_transform>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred4 = perceptron.predict(x_test)
 acc_perceptron = round(perceptron.score(x_test, y_test)* 100, 2)
 acc_perceptron
class TweetDataset:
def process_data(self,tweet, selected_text, sentiment, tokenizer, max_len):
if self.is_train and np.random.rand() >0.2:
new_df = self.selected_text[self.sentiment==sentiment]
ids=np.random.randint(len(new_df))
tweet=tweet[:tweet.index(selected_text)]+' '+new_df[ids]+" "+tweet[(tweet.index(selected_text)+len(selected_text)) :]
selected_text=new_df[ids]
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {'ids': torch.tensor(input_ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)},{'targets_start': torch.tensor(targets_start, dtype=torch.long),
'targets_end': torch.tensor(targets_end, dtype=torch.long),
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'tweet_offsets': torch.tensor(tweet_offsets, dtype=torch.long)}
def __init__(self,tweet,sentiment,selected_text,MAX_LEN,TOKENIZER,is_train):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.max_len = MAX_LEN
self.tokenizer = TOKENIZER
self.is_train=is_train
self.transforms=get_train_transforms(is_train)
 def __len__(self):
return len(self.tweet)
 def __getitem__(self,item):
sent = self.tweet[item]
selected_text=self.selected_text[item]
if self.sentiment[item]!='neutral':
sent,selected_text,_=self.transforms(data=(sent,selected_text, 'en')) ['data']
return self.process_data(
 sent,
selected_text,
self.sentiment[item],
self.tokenizer,
self.max_len
)<feature_engineering>
print(classification_report(y_test, Y_pred4))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred4), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
df=pd.read_csv(TRAINING_FILE ).dropna().reset_index(drop=True)
 df.loc[2940]['text']=df.loc[2940]['text'][1:][:2]+" "+df.loc[2940]['text'][1:][2:]
df.loc[2940]['selected_text']=df.loc[2940]['selected_text'][1:][:2]+" "+df.loc[2940]['selected_text'][1:][2:]
df.loc[8444]['text']=df.loc[8444]['text'][1:2]+' '+ df.loc[8444]['text'][2:6]+' '+df.loc[8444]['text'][6:8]+' '+df.loc[8444]['text'][8:10]+' '+df.loc[8444]['text'][10:16]+" "+df.loc[8444]['text'][16:18]+' '+df.loc[8444]['text'][18:]
df.loc[8444]['selected_text']=df.loc[8444]['selected_text'][1:2]+' '+ df.loc[8444]['selected_text'][2:6]+' '+df.loc[8444]['selected_text'][6:8]+' '+df.loc[8444]['selected_text'][8:10]+' '+df.loc[8444]['selected_text'][10:16]+" "+df.loc[8444]['selected_text'][16:18]
df.loc[8718]['text']= df.loc[8718]['text'].replace('
df.loc[8718]['selected_text']=df.loc[8718]['selected_text'].replace('
df.loc[10531]['text']= df.loc[10531]['text'].replace('
df.loc[10531]['selected_text']=df.loc[10531]['selected_text'].replace('
df.loc[10576]['text']=df.loc[10576]['text'].replace('
df.loc[10576]['selected_text']=df.loc[10576]['selected_text'].replace('liesboystell','lies boys tell')
 df.loc[11744]['text']=df.loc[11744]['text'].replace('
df.loc[11744]['selected_text']='shortcake fail'
df.loc[14838]['text'] = df.loc[14838]['text'].replace('was boring but had to eat nonetheless',' was boring but had to eat nonetheless')
 df.loc[14934]['text']=df.loc[14934]['text'].replace('
df.loc[14934]['selected_text']='sad'
df.loc[15565]['text'] = df.loc[15565]['text'][1:]
df.loc[17299]['text']=df.loc[17299]['text'].replace('
df.loc[17299]['selected_text']='fail'
df.loc[17418]['text']=df.loc[17418]['text'].replace('
df.loc[17418]['selected_text']='it sucks'
ids=18516
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']='you rock'
ids=18626
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']='Happy Mothers Day'
ids=21559
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']='fail'
ids=21954
df.loc[ids]['text']=df.loc[ids]['text'].replace('
ids=22587
df.loc[ids]['selected_text']='thank you so much'
ids=23672
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']=' no way i couldn`t tune in'
ids=23741
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']='excited'
ids=24681
df.loc[ids]['selected_text']='Wave looks interesting'
ids=26093
df.loc[ids]['text']=df.loc[ids]['text'].replace('
df.loc[ids]['selected_text']='heart break'
df.loc[7926]['selected_text']=df.loc[7926]['selected_text'][2:] +' you'
df.loc[166]['selected_text']=df.loc[166]['selected_text'][2:]
df.loc[251]['selected_text']=df.loc[251]['selected_text'][2:]
df.loc[1986]['selected_text']=df.loc[1986]['selected_text'][2:]
df.loc[706]['selected_text']=df.loc[706]['selected_text'][2:]
df.loc[6112]['selected_text']=df.loc[6112]['selected_text'][6:]
df.loc[11678]['selected_text']=df.loc[11678]['selected_text'][2:]
df.loc[12038]['selected_text']=df.loc[12038]['selected_text'][2:]
df.loc[23923]['selected_text']=df.loc[23923]['selected_text'][2:]
df.drop([26686,25690,24681,25690,26686,2269,697,1032],inplace=True)
 df=df.dropna().reset_index(drop=True)
 df.drop([6056,21591,23425,23669,26088],inplace=True)
 df=df.dropna().reset_index(drop=True)
 dset = TweetDataset(tweet = df.text.values,
sentiment = df.sentiment.values,
selected_text = df.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER,
is_train=True)
<choose_model_class>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred5 = linear_svc.predict(x_test)
 acc_linear_svc = round(linear_svc.score(x_test, y_test)* 100, 2)
 acc_linear_svc
skf = StratifiedKFold(n_splits=5)
 ids=skf.get_n_splits(df['text'], df['sentiment'] )<feature_engineering>
print(classification_report(y_test, Y_pred5))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred5), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
df['kfold']=0
for ids,(train_index, test_index)in enumerate(skf.split(df['text'], df['sentiment'])) :
df['kfold'].loc[test_index]=ids<import_modules>
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred6 = sgd.predict(x_test)
 acc_sgd = round(sgd.score(x_test, y_test)* 100, 2)
 acc_sgd
import torch.nn as nn
import transformers
import torch
from sklearn import model_selection
from transformers.modeling_bert import BertPreTrainedModel
import scipy<load_pretrained>
print(classification_report(y_test, Y_pred6))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred6), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, x):
_, _, hs = self.roberta(x['ids'], x['mask'])
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 perm = None

 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits,perm<split>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred7 = decision_tree.predict(x_test)
 acc_decision_tree = round(decision_tree.score(x_test, y_test)* 100, 2)
 acc_decision_tree
TRAIN_BATCH_SIZE=2
MAX_LEN=128
EPOCHS = 2
dfx=pd.read_csv(TRAINING_FILE ).dropna().reset_index(drop=True)
 sample =pd.read_csv(SAMPLE_FILE)
 df_train,df_valid = model_selection.train_test_split(dfx,random_state=42,test_size=0.2)
 df_train = df_train.reset_index(drop=True)
 df_valid = df_valid.reset_index(drop=True)
 target_cols = list(sample.drop('textID',axis=1 ).columns)
 train_targets = df_train[target_cols].values
valid_targets = df_valid[target_cols].values
train_dataset = TweetDataset(df_train.text.values,df_train.sentiment.values,df_train.selected_text.values,MAX_LEN,config.TOKENIZER,is_train=True)
 train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=TRAIN_BATCH_SIZE,shuffle=True)
 valid_dataset = TweetDataset(df_valid.text.values,df_valid.sentiment.values,df_valid.selected_text.values,MAX_LEN,config.TOKENIZER,is_train=False)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=16,shuffle=False )<load_pretrained>
print(classification_report(y_test, Y_pred7))
cm = pd.DataFrame(confusion_matrix(y_test, Y_pred7), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
data = DataBunch(train_loader,valid_loader)
<prepare_x_and_y>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 random_forest_predictions = random_forest.predict(x_test)
 acc_random_forest = round(random_forest.score(x_test, y_test)* 100, 2)
 test_Survived = pd.Series(random_forest_predictions, name="Survived")
 Submission2 = pd.concat([PassengerId,test_Survived],axis=1)
 acc_random_forest
for x,y in valid_loader:
break<count_values>
print(classification_report(y_test, random_forest_predictions))
cm = pd.DataFrame(confusion_matrix(y_test, random_forest_predictions), ['Actual: NOT', 'Actual: SURVIVED'], ['Predicted: NO', 'Predicted: SURVIVED'])
 print(cm )
df.kfold.value_counts()<load_from_csv>
kfold = StratifiedKFold(n_splits=10)
 random_state = 2
classifiers = []
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(SVC(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(GaussianNB())
 classifiers.append(Perceptron(random_state=random_state))
classifiers.append(LinearSVC(random_state=random_state))
classifiers.append(SGDClassifier(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state = random_state))
classifiers.append(RandomForestClassifier(random_state = random_state))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":['Logistic Regression', 'KNN', 'Gaussian',
'Perceptron', 'linear SVC', 'SGD', 'Decision Tree','SVMC', 'Random Forest']})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )
def get_learner(model,opt,data, loss_func,
cb_funcs=None):
return Learner(model, data, loss_func, opt,cb_funcs=cb_funcs)
 def run_fold(k):
TRAIN_BATCH_SIZE=64
MAX_LEN=128
sample =pd.read_csv(SAMPLE_FILE)
 df_train,df_valid = df[df.kfold!=k],df[df.kfold==k]
df_train=df_train[df_train.sentiment!='neutral']
df_train = df_train.reset_index(drop=True)
 df_valid = df_valid.reset_index(drop=True)
 target_cols = list(sample.drop('textID',axis=1 ).columns)
 train_targets = df_train[target_cols].values
valid_targets = df_valid[target_cols].values
train_dataset = TweetDataset(df_train.text.values,df_train.sentiment.values,df_train.selected_text.values,MAX_LEN,config.TOKENIZER,is_train=True)
 train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=TRAIN_BATCH_SIZE,shuffle=True)
 valid_dataset = TweetDataset(df_valid.text.values,df_valid.sentiment.values,df_valid.selected_text.values,MAX_LEN,config.TOKENIZER,is_train=False)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=16,shuffle=False)
 data = DataBunch(train_loader,valid_loader)
 model = TweetModel()
params = list(model.named_parameters())
 optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
cbfs = [Recorder,
partial(AvgStatsCallback,metric),
CudaCallback,
ProgressCallback,
partial(EarlyStopingCallback,3,'model_'+str(k)) ]
learn = get_learner(model,optimizer,data,loss_fn, cb_funcs=cbfs)
 learn.fit(8)
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test['selected_text']=test['text'].apply(lambda x:x)
 dset = TweetDataset(tweet = test.text.values,
sentiment = test.sentiment.values,
selected_text = test.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER,
is_train=False)
 res_start=[]
res_end=[]
bs=16
for x,y in tqdm_notebook(torch.utils.data.DataLoader(dset,batch_size=bs,shuffle=False)) :
for key in x.keys() :
x[key]=x[key].cuda()
output = learn.model(x)
 res_start.append(output[0].detach().cpu().numpy())
 res_end.append(output[1].detach().cpu().numpy())
 return np.concatenate(res_start),np.concatenate(res_end)
<feature_engineering>
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,2],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,Y_train)
 adaDTC_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_
start1,end1=run_fold(0 )<feature_engineering>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 7],
"min_samples_split": [2, 3, 7],
"min_samples_leaf": [1, 3, 7],
"bootstrap": [False],
"n_estimators" :[300,600],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,Y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_
start2,end2=run_fold(1 )<feature_engineering>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1] }
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_
start3,end3=run_fold(3 )<feature_engineering>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1,10,50,100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_
start4,end4=run_fold(4 )<define_variables>
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 7],
"min_samples_split": [2, 3, 7],
"min_samples_leaf": [1, 3, 7],
"bootstrap": [False],
"n_estimators" :[300,600],
"criterion": ["gini"]}
gsrandom_forest = GridSearchCV(random_forest,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsrandom_forest.fit(X_train,Y_train)
 random_forest_best = gsrandom_forest.best_estimator_
gsrandom_forest.best_score_
start =(start1+start2+start3+start4)/4
end =(end1+end2+end3+end4)/4<load_from_csv>
VotingPredictor = VotingClassifier(estimators=[('ExtC', ExtC_best),('GBC',GBC_best), 
('SVMC', SVMC_best),('random_forest', random_forest_best)], voting='soft', n_jobs=4)
 VotingPredictor = VotingPredictor.fit(X_train, Y_train)
 VotingPredictor_predictions = VotingPredictor.predict(test)
 test_Survived = pd.Series(VotingPredictor_predictions, name="Survived")
 test_Survived = pd.Series(VotingPredictor_predictions, name="Survived")
 Submission3 = pd.concat([PassengerId,test_Survived],axis=1)
 Submission3.head(15 )
test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test['selected_text']=test['text'].apply(lambda x:x)
 dset = TweetDataset(tweet = test.text.values,
sentiment = test.sentiment.values,
selected_text = test.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER,
is_train=False)
 final_output=[]
i=0
for x,y in dset:
ids = x["ids"]
token_type_ids = x["token_type_ids"]
mask =x["mask"]
sentiment = y["sentiment"]
orig_selected = y["orig_selected"]
orig_tweet = y["orig_tweet"]
targets_start = y["targets_start"]
targets_end = y["targets_end"]
offsets = y["tweet_offsets"].numpy()
selected_tweet = orig_selected
tweet_sentiment = sentiment
_, output_sentence = calculate_jaccard_score(
 original_tweet=orig_tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(start[i]),
idx_end=np.argmax(end[i]),
offsets=offsets 
)
 final_output.append(output_sentence)
 i+=1 
<feature_engineering>
Submission3.to_csv("StackingSubmission.csv", index=False)
 print("Completed." )