Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<filter>
print('TF version',tf.__version__ )
traintest_drop3[traintest_drop3['PassengerId']==63]<count_values>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
traintest_drop3[traintest_drop3['Pclass']==1]['Embarked'].value_counts()<feature_engineering>
train["sentiment"].value_counts()
traintest_drop3.loc[(traintest_drop3['Embarked'].isna()),'Embarked'] = 'S'<categorify>
train_dict = dict()
train_dict_index = dict()
for sentiment in all_sentiments:
train_dict[sentiment] = train[train["sentiment"]==sentiment]
train_dict_index[sentiment] = list(train_dict[sentiment].index)
 train_dict[sentiment] = train_dict[sentiment].reset_index(drop=True )
oenc = OrdinalEncoder()
traintest_objenc = pd.DataFrame(oenc.fit_transform(traintest_drop3.select_dtypes('object')) ,columns = traintest_drop3.select_dtypes('object' ).columns )<merge>
def get_training_data(train):
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
return input_ids, attention_mask, token_type_ids, start_tokens, end_tokens
traintest_num = traintest_drop3.select_dtypes(['int64','float64'] ).drop(['index','PassengerId'], axis=1 ).join(traintest_objenc )<create_dataframe>
train_data_dict = dict()
for sentiment in all_sentiments:
train_data_dict[sentiment] = get_training_data(train_dict[sentiment] )
traintest_norm = traintest_drop3[['index','PassengerId']].join(pd.DataFrame(StandardScaler().fit_transform(traintest_num), columns = traintest_num.columns)) 
<drop_column>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('' )
train_norm = traintest_norm[0:891].drop('index', axis=1 )<drop_column>
test_dict = dict()
test_dict_index = dict()
for sentiment in all_sentiments:
test_dict[sentiment] = test[test["sentiment"]==sentiment]
test_dict_index[sentiment] = list(test_dict[sentiment].index)
 test_dict[sentiment] = test_dict[sentiment].reset_index(drop=True )
test_norm = traintest_norm[891::].drop('index', axis=1 ).reset_index().drop('index',axis=1 )<split>
def get_test_data(test):
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
return input_ids_t, attention_mask_t, token_type_ids_t
X_train, X_test, y_train, y_test = train_test_split(train_norm, y )<compute_train_metric>
test_data_dict = dict()
for sentiment in all_sentiments:
test_data_dict[sentiment] = get_test_data(test_dict[sentiment] )
modelgbr = GradientBoostingClassifier(random_state=42)
 modelgbr.fit(X_train, y_train)
 y_predgbr = modelgbr.predict(X_test)
 accuracy_score(y_test, y_predgbr)
<create_dataframe>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
pd.DataFrame(modelgbr.feature_importances_, X_train.columns ).sort_values(by=0,ascending=False)
<predict_on_test>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def model_test(testmodel):
model = testmodel()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 return accuracy_score(y_test, y_pred )<define_variables>
def find_a_b(start_array, end_array):
a = np.argmax(start_array)
 b = np.argmax(end_array)
 if a<b:
return a,b
else:
if a<len(start_array)-1:
end_array_copy = np.array(end_array)
 end_array_copy[b]=0.0
return find_a_b(start_array, end_array_copy)
 else:
if b<len(start_array)-1:
start_array_copy = np.array(start_array)
 start_array_copy[a]=0.0
return find_a_b(start_array_copy, end_array)
 else:
return 0, len(end_array)-1
models = [RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier,
BaggingClassifier, ExtraTreesClassifier, DecisionTreeClassifier, KNeighborsClassifier]<train_model>
def train_model_for_sentiment(train_data_dict, test_data_dict, sentiment, train_dict):
input_ids, attention_mask, token_type_ids, start_tokens, end_tokens = train_data_dict[sentiment]
input_ids_t, attention_mask_t, token_type_ids_t = test_data_dict[sentiment]
train = train_dict[sentiment]
jac = []; VER='v1'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 print('
print('
print('
for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i-%s.h5'%(VER,fold,sentiment), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i-%s.h5'%(VER,fold,sentiment))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a,b = find_a_b(oof_start[k,], oof_end[k,])
 text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
return preds_start, preds_end
for i in models:
print(i)
 print(model_test(i))<train_on_grid>
def get_string_predictions(test_data_dict, sentiment, predictions, test_dict):
preds_start, preds_end = predictions[sentiment]
input_ids_t, attention_mask_t, token_type_ids_t = test_data_dict[sentiment]
test = test_dict[sentiment]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 return all
modelclf = GradientBoostingClassifier()
parameters = {'n_estimators':[50,200,300], 'learning_rate':[0.001,0.01,0.1],'max_depth':[1,3,6]}
clf = GridSearchCV(modelclf, parameters)
 clf.fit(X_train, y_train)
 y_predclf = clf.predict(X_test)
 accuracy_score(y_test, y_predclf)
 clf.cv_results_
clf.best_params_ 
<train_model>
def get_string_predictions_neutral(test_dict, sentiment):
test = test_dict[sentiment]
return list(test["text"] )
modelsub = GradientBoostingClassifier(learning_rate = 0.01, max_depth = 3, n_estimators = 300)
 modelsub.fit(train_norm.drop('PassengerId',axis=1), y)
 preds_test = modelsub.predict(test_norm.drop('PassengerId',axis=1))<save_to_csv>
predictions = dict()
string_predictions = dict()
for sentiment in all_sentiments:
if not sentiment == "neutral":
predictions[sentiment] = train_model_for_sentiment(train_data_dict, test_data_dict, sentiment, train_dict)
 string_predictions[sentiment] = get_string_predictions(test_data_dict, sentiment, predictions, test_dict)
 else:
string_predictions[sentiment] = get_string_predictions_neutral(test_dict, "neutral" )
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": preds_test
})
 submission.to_csv('submission.csv', index=False)
<import_modules>
concat_string_predictions = []
concat_index = []
for sentiment in all_sentiments:
concat_string_predictions += string_predictions[sentiment]
concat_index += test_dict_index[sentiment]
sorted_concat_string_predictions = sorted(list(zip(concat_index, concat_string_predictions)) , key=lambda x:x[0])
 sorted_concat_string_predictions_list = list(zip(*sorted_concat_string_predictions)) [1]
import pandas as pd
import numpy as np
from datetime import timedelta, date
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder
from lightgbm import LGBMClassifier<set_options>
test['selected_text'] = sorted_concat_string_predictions_list
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )
warnings.filterwarnings('ignore' )<load_from_csv>
def get_predictions_on_oof_for_sentiment(train_data_dict, test_data_dict, sentiment, train_dict):
input_ids, attention_mask, token_type_ids, start_tokens, end_tokens = train_data_dict[sentiment]
input_ids_t, attention_mask_t, token_type_ids_t = test_data_dict[sentiment]
train = train_dict[sentiment]
PATH_SAVED_WEIGHTS = ''
train["prediction_oof"] = ""
train["jaccard_sim"] = np.nan
jac = []; VER='v1'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 print('
print('
print('
for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
print('Loading model...')
 model.load_weights(PATH_SAVED_WEIGHTS+'%s-roberta-%i-%s.h5'%(VER,fold,sentiment))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 jaccard_sim = jaccard(st,train.loc[k,'selected_text'])
 train.at[k, "jaccard_sim"] = jaccard_sim
train.at[k, "prediction_oof"] = st
all.append(jaccard_sim)
 jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
return train
train = pd.read_csv('/kaggle/input/car-loan/car_loan_train.csv')
 test = pd.read_csv('/kaggle/input/car-loan/car_loan_test.csv')
 train.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in train.columns]
test.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in test.columns]<categorify>
oof_predictions = get_predictions_on_oof_for_sentiment(train_data_dict, test_data_dict, "positive", train_dict )
train = train.fillna('other')
 test = test.fillna('other' )<categorify>
pd.set_option('display.max_colwidth', -1)
 oof_predictions.sample(20 )
train = train.replace({'PERFORM_CNS_SCORE_DESCRIPTION':{'C-Very Low Risk':'Low', 'A-Very Low Risk':'Low',
'B-Very Low Risk':'Low', 'D-Very Low Risk':'Low',
'F-Low Risk':'Low', 'E-Low Risk':'Low', 'G-Low Risk':'Low',
'H-Medium Risk': 'Medium', 'I-Medium Risk': 'Medium',
'J-High Risk':'High', 'K-High Risk':'High','L-Very High Risk':'High',
'M-Very High Risk':'High','Not Scored: More than 50 active Accounts found':'Not Scored',
'Not Scored: Only a Guarantor':'Not Scored','Not Scored: Not Enough Info available on the customer':'Not Scored',
'Not Scored: No Activity seen on the customer(Inactive)':'Not Scored','Not Scored: No Updates available in last 36 months':'Not Scored',
'Not Scored: Sufficient History Not Available':'Not Scored', 'No Bureau History Available':'Not Scored'
}})
 test = test.replace({'PERFORM_CNS_SCORE_DESCRIPTION':{'C-Very Low Risk':'Low', 'A-Very Low Risk':'Low',
'B-Very Low Risk':'Low', 'D-Very Low Risk':'Low',
'F-Low Risk':'Low', 'E-Low Risk':'Low', 'G-Low Risk':'Low',
'H-Medium Risk': 'Medium', 'I-Medium Risk': 'Medium',
'J-High Risk':'High', 'K-High Risk':'High','L-Very High Risk':'High',
'M-Very High Risk':'High','Not Scored: More than 50 active Accounts found':'Not Scored',
'Not Scored: Only a Guarantor':'Not Scored','Not Scored: Not Enough Info available on the customer':'Not Scored',
'Not Scored: No Activity seen on the customer(Inactive)':'Not Scored','Not Scored: No Updates available in last 36 months':'Not Scored',
'Not Scored: Sufficient History Not Available':'Not Scored', 'No Bureau History Available':'Not Scored'
}} )<data_type_conversions>
oof_predictions["len_selected_text"] = oof_predictions["selected_text"].map(lambda x:str(x ).strip() ).map(len)
 oof_predictions["len_prediction_oof"] = oof_predictions["prediction_oof"].map(lambda x:str(x ).strip() ).map(len )
<feature_engineering><EOS> 
oof_predictions["num_words_text"] = oof_predictions["text"].map(lambda x:len(x.split()))
oof_predictions["num_words_selected_text"] = oof_predictions["selected_text"].map(lambda x:len(x.split()))
oof_predictions["num_words_prediction_oof"] = oof_predictions["prediction_oof"].map(lambda x:len(x.split()))