Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<train_model>
print('TF version',tf.__version__ )
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
<save_to_csv>
df_train=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv" ).fillna('')
 df_test=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv" ).fillna('' )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
print("Training Data
%s" %df_train.head())
 print("Test Data
%s" %df_test.head() )
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder<string_transform>
df_train.isnull().sum()
def extract(m):
m = m.split(',')[1]
m = m.split('.')[0]
return m[1:]<load_from_csv>
df_test.isnull().sum()
path = '/kaggle/input/titanic/train.csv'
df = pd.read_csv(path)
 df['Name'] = df['Name'].apply(extract)
 df['Name'] = df['Name'].apply(lambda x: x if x in ['Mr','Mrs','Miss','Master'] else 'Others')
 df['Parch'] = df['Parch'].apply(lambda x: x if x in [0,1,2] else 4.5)
<categorify>
import seaborn as sns
import matplotlib.pyplot as plt
df = df.fillna(df.mean())
 df['Embarked'] = df['Embarked'].apply(lambda x : x if(x=='C' or x=='Q')else 'S')
 df['Embarked'].unique()
print(df.count())
 le = LabelEncoder()
le.fit(df['Sex'])
 df['Sex'] = le.transform(df['Sex'])
 le.fit(df['Name'])
 df['Name'] = le.transform(df['Name'])
 le.fit(df['Embarked'])
 df['Embarked'] = le.transform(df['Embarked'] )<prepare_x_and_y>
ct = df_train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(df_train.shape[0]):
text1 = " "+" ".join(df_train.loc[k,'text'].split())
 text2 = " ".join(df_train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[df_train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
features = ['Pclass','Sex','SibSp','Parch','Fare','Embarked','Name']
y = df['Survived']
X = df[features]<train_model>
ct = df_test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(df_test.shape[0]):
text1 = " "+" ".join(df_test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[df_test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
for i in range(1):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 model = GradientBoostingClassifier(n_estimators = 200, max_depth = 3)
 model.fit(X_train,y_train)
 print(i,(model.predict(X_train)-y_train==0 ).sum() *100/len(y_train))
print(( model.predict(X_test)-y_test==0 ).sum() *100/len(y_test),'
' )<load_from_csv>
def R_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
df3 = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 df3<categorify>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
model.fit(X,y)
 df2 = pd.read_csv('/kaggle/input/titanic/test.csv')
 df2['Name'] = df2['Name'].apply(extract)
 df2['Name'] = df2['Name'].apply(lambda x: x if x in ['Mr','Mrs','Miss','Master'] else 'Others')
 df2['Parch'] = df2['Parch'].apply(lambda x: x if x in [0,1,2] else 4.5)
 df2 = df2.fillna(df.mean())
 df2['Embarked'] = df2['Embarked'].apply(lambda x : x if(x=='C' or x=='Q')else 'S')
 df2['Embarked'].unique()
le = LabelEncoder()
le.fit(df2['Sex'])
 df2['Sex'] = le.transform(df2['Sex'])
 le.fit(df2['Name'])
 df2['Name'] = le.transform(df2['Name'])
 le.fit(df2['Embarked'])
 df2['Embarked'] = le.transform(df2['Embarked'])
 X_sub = df2[features]
df2['Survived'] = model.predict(X_sub)
 sub = pd.concat([df2['PassengerId'],df2['Survived']],axis = 1)
 sub<save_to_csv>
train=df_train
test=df_test
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=3,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = R_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=5, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
sub.to_csv('Submission.csv',index=False )<import_modules>
print('>>>> OVERALL 3Fold CV Jaccard =',np.mean(jac))
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<count_values><EOS> 
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )