Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<feature_engineering>
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold
from transformers import *
train['agebucket'] = pd.cut(train['Age'], 5)
 test['agebucket'] = pd.cut(test['Age'], 5)
 train[['agebucket', 'Survived']].groupby(['agebucket'] ).mean().sort_values(by='agebucket', ascending=True ).round(2 )<feature_engineering>
print('TF version',tf.__version__ )
for dataset in [train, test]:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
train.head()<sort_values>
SEED = 4262
np.random.seed(SEED)
 tf.random.set_seed(SEED )
print(train[['Family', 'Survived']].groupby(['Family'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2),
'
',
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2))<sort_values>
USE_TPU = False
if USE_TPU:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.OneDeviceStrategy(device = "/gpu:0" )
train['isalone'] = [1 if x == 1 else 0 for x in train['Family']]
test['isalone'] = [1 if x == 1 else 0 for x in test['Family']]
train[['isalone', 'Survived']].groupby(['isalone'] ).mean().sort_values(by='Survived', ascending=False ).round(2 )<categorify>
MAX_LEN = 128
FOLDS = 5
EPOCHS = 3
VERBOSE = 1
ROBERTA_BASE_PATH = '.. /input/tf-roberta/'
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
print('BatchSize: {}'.format(BATCH_SIZE))
LR = 1e-5 * strategy.num_replicas_in_sync
print('LearningRate: {}'.format(LR))
INFERENCE = True
dummy_features = ['Sex','Title', 'isalone']
drop_features = ['Embarked', 'PassengerId', 'Ticket', 'Name', 'Cabin','Parch','SibSp', 'agebucket']
train = pd.concat([train, pd.get_dummies(train[dummy_features])], axis = 1, sort = False)
 train.drop(columns = train[dummy_features], inplace = True)
 train.drop(columns = train[drop_features], inplace = True)
 test = pd.concat([test, pd.get_dummies(test[dummy_features])], axis = 1, sort = False)
 test.drop(columns = test[dummy_features], inplace = True)
 test.drop(columns = test[drop_features], inplace = True)
 train.tail()<split>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv', nrows = 27200 ).fillna('')
 train.head()
y = train[target]
x = train.drop(columns = target)
 x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state = 42 )<train_on_grid>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 test.head()
RF = ensemble.RandomForestClassifier()
RF_params = {
'n_estimators':[n for n in range(60,140,10)],
'max_depth':[n for n in range(3, 6)],
'max_features' : ['sqrt', 'log2', None],
'random_state' : [42]
}
RF_model = GridSearchCV(RF, param_grid = RF_params, cv = 5, n_jobs = -1 ).fit(x_train, y_train)
 print("Best Hyper Parameters:",RF_model.best_params_)
 RF_probs = RF_model.predict_proba(x_test)
 RF_probs = RF_probs[:, 1]
RF_auc = roc_auc_score(y_test, RF_probs)
 print('AUC: %.3f' % RF_auc)
 RF_predictions = RF_model.predict(x_test)
 RF_accuracy = accuracy_score(y_test, RF_predictions)
 print("RF accuracy: %.3f" % RF_accuracy)
 RF_fpr, RF_tpr, RF_thresholds = roc_curve(y_test, RF_probs)
 plt.plot([0, 1], [0, 1], linestyle='--')
 plt.plot(RF_fpr, RF_tpr, color = 'tab:green')
 plt.show()<train_on_grid>
tokenizer = RobertaTokenizer(vocab_file = ROBERTA_BASE_PATH + 'vocab-roberta-base.json',
merges_file = ROBERTA_BASE_PATH + 'merges-roberta-base.txt',
add_prefix_space = True,
do_lower_case = False )
GBT = ensemble.GradientBoostingClassifier()
GBT_params = {
'n_estimators':[n for n in range(180, 240, 20)],
'max_depth':[n for n in range(3, 6)],
'learning_rate': [0.1, 0.25, 0.5],
'random_state' : [42]
}
GBT_model = GridSearchCV(GBT, param_grid = GBT_params, cv = 5, n_jobs = -1)
 GBT_model.fit(x_train, y_train)
 print("Best Hyper Parameters:",GBT_model.best_params_)
 GBT_probs = GBT_model.predict_proba(x_test)
 GBT_probs = GBT_probs[:, 1]
GBT_auc = roc_auc_score(y_test, GBT_probs)
 print('AUC: %.3f' % GBT_auc)
 GBT_predictions = GBT_model.predict(x_test)
 GBT_accuracy = accuracy_score(y_test, GBT_predictions)
 print("GBT accuracy: %.3f" % GBT_accuracy)
 GBT_fpr, GBT_tpr, GBT_thresholds = roc_curve(y_test, GBT_probs)
 plt.plot([0, 1], [0, 1], linestyle='--')
 plt.plot(GBT_fpr, GBT_tpr, color = 'tab:orange')
 plt.show()<compute_test_metric>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN), dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN), dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN), dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN), dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN), dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 if text1 != '':
input_encoded = tokenizer.encode_plus(text1, train.loc[k,'sentiment'], add_special_tokens = True, max_length = MAX_LEN, padding = True)
 input_ids_sample = input_encoded["input_ids"]
attention_mask_sample = input_encoded["attention_mask"]
attention_mask[k,:len(attention_mask_sample)] = attention_mask_sample
input_ids[k,:len(input_ids_sample)] = input_ids_sample
idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx + len(text2)] = 1
k_ids = tokenizer.encode(text1, add_special_tokens = False)
 offsets = []
idx = 0
for t in k_ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 if len(toks)> 0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
print("GBT cohen_kappa_score: %.3f" % cohen_kappa_score(y_test, GBT_predictions))
print("RF cohen_kappa_score: %.3f" % cohen_kappa_score(y_test, RF_predictions)) 
<compute_test_metric>
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(test.loc[k,'text'].split())
 if text1 != '':
input_encoded = tokenizer.encode_plus(text1, test.loc[k, 'sentiment'], add_special_tokens = True, max_length = MAX_LEN, padding = True)
 input_ids_sample = input_encoded["input_ids"]
attention_mask_sample = input_encoded["attention_mask"]
attention_mask_t[k,:len(attention_mask_sample)] = attention_mask_sample
input_ids_t[k,:len(input_ids_sample)] = input_ids_sample
print("GBT", classification_report(y_test, GBT_predictions))
print("-"*100)
 print("RF", classification_report(y_test, RF_predictions))<save_to_csv>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
predict_RF = RF_model.predict(test)
 predict_GBT = GBT_model.predict(test)
 submit_RF = pd.DataFrame({'PassengerId':testing['PassengerId'],'Survived':predict_RF})
 submit_GBT = pd.DataFrame({'PassengerId':testing['PassengerId'],'Survived':predict_GBT})
 filename_RF = 'Titanic Prediction RF.csv'
submit_RF.to_csv(filename_RF,index=False)
 print('Saved file: ' + filename_RF)
 filename_GBT = 'Titanic Prediction GBT.csv'
submit_GBT.to_csv(filename_GBT,index=False)
 print('Saved file: ' + filename_GBT )<import_modules>
config = RobertaConfig.from_pretrained(ROBERTA_BASE_PATH+'config-roberta-base.json')
 config.attention_probs_dropout_prob = 0.15
print(config )
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None)
 print("Primary Directory:")
 print(os.listdir(".. /input/"))
print("
Files:")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
def build_model() :
with strategy.scope() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 roberta_model = TFRobertaForQuestionAnswering.from_pretrained(ROBERTA_BASE_PATH + 'pretrained-roberta-base.h5', config = config)
 x = roberta_model(ids, attention_mask = att, token_type_ids = tok)
 x1 = tf.keras.layers.Activation('softmax' )(x[0])
 x2 = tf.keras.layers.Activation('softmax' )(x[1])
 model = tf.keras.models.Model(inputs = [ids, att, tok], outputs=[x1, x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate = LR)
 loss = tf.keras.losses.CategoricalCrossentropy(from_logits = False, label_smoothing = 0.1)
 model.compile(loss = loss, optimizer = optimizer)
 return model
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 def quick_look(df, df_name):
print('{} data contains {} observations across {} columns
'.format(df_name, df.shape[0], df.shape[1]))
display(df.head(5))
quick_look(train, 'Train')
 quick_look(test, 'Test' )<drop_column>
jac, jac1 = [], []
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits = FOLDS, shuffle = True, random_state = SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
model_checkpoint = tf.keras.callbacks.ModelCheckpoint('roberta-%i.h5'%(fold), monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto', save_freq = 'epoch')
 if not INFERENCE:
model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs = EPOCHS,
batch_size = BATCH_SIZE,
verbose = VERBOSE,
callbacks = [model_checkpoint],
validation_data =([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]], [start_tokens[idxV,], end_tokens[idxV,]]),
shuffle = True)
 print('Loading model...')
 model.load_weights(f'roberta-{fold}.h5')
 else:
print('Loading model...')
 model.load_weights(f'/kaggle/input/tensorflow-roberta-qa-model/roberta-{fold}.h5')
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose = VERBOSE)
 print('Predicting Test...')
 preds = model.predict([input_ids_t, attention_mask_t, token_type_ids_t], verbose = VERBOSE)
 preds_start += preds[0] / skf.n_splits
preds_end += preds[1] / skf.n_splits
all = []
all1 = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1, add_special_tokens = False)
 if train.loc[k,'sentiment'] == 'neutral':
st = " ".join(train.loc[k,'text'].split())
 st1 = " ".join(train.loc[k,'text'].split())
 else:
if a > b:
st = text1
st1 = text1
start_sort = np.argsort(oof_start[k,])[::-1]
end_sort = np.argsort(oof_end[k,])[::-1]
a1 = start_sort[1]
b1 = end_sort[1]
a2 = start_sort[2]
b2 = end_sort[2]
if a1 < b1:
st1 = tokenizer.decode(enc[a1-1:b1])
 elif a2 < b2:
st1 = tokenizer.decode(enc[a2-1:b2])
 else:
st = tokenizer.decode(enc[a-1:b])
 st1 = tokenizer.decode(enc[a-1:b])
 all.append(jaccard(st, train.loc[k,'selected_text']))
all1.append(jaccard(st1, train.loc[k,'selected_text']))
jac.append(np.mean(all))
jac1.append(np.mean(all1))
print('>>>> FOLD %i Jaccard ='%(fold+1), np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1), np.mean(all1))
print()
print('Only difference in columns between train and test is our target variable: {}'.format(( set(train.columns ).symmetric_difference(set(test.columns)))))
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )<drop_column>
print(f'=== OVERALL 5 Fold CV Jaccard = {np.mean(jac)}')
 print(f'=== OVERALL 5 Fold CV Jaccard - Post Processed = {np.mean(jac1)}' )
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 train.reset_index(drop=True, inplace=True )<feature_engineering>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1, add_special_tokens = False)
 if test.loc[k, 'sentiment'] == 'neutral':
st = " ".join(test.loc[k,'text'].split())
 else:
if a > b:
st = text1
start_sort = np.argsort(preds_start[k,])[::-1]
end_sort = np.argsort(preds_end[k,])[::-1]
a1 = start_sort[1]
b1 = end_sort[1]
a2 = start_sort[2]
b2 = end_sort[2]
if a1 < b1:
st = tokenizer.decode(enc[a1-1:b1])
 elif a2 < b2:
st = tokenizer.decode(enc[a2-1:b2])
 else:
st = tokenizer.decode(enc[a-1:b])
 all.append(st )
complete_df = pd.concat(( train.iloc[:,0:-1],test.iloc[:,0:len(test.columns)]))
def data_quality(df):
print('Data Quality for Complete dataframe')
 df_temp = pd.DataFrame(index=df.columns, columns=['Missing_Count', 'Unique_Values', 'Scarcity', 'Datatype'])
 df_temp['Missing_Count'] = df.isnull().sum()
df_temp['Unique_Values'] = [df[i].nunique() for i in df.columns]
df_temp['Scarcity'] = round(((df.isnull().sum())/(df.shape[0])*100),2)
 df_temp['Datatype'] = [df[i].dtype for i in df.columns]
return df_temp.head(len(df.columns)).sort_values(by=['Missing_Count'], ascending=False)
 dq_df = data_quality(complete_df)
 display(dq_df )<drop_column>
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )