Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<feature_engineering>
stop=set(stopwords.words('english'))
warnings.filterwarnings("ignore")
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
df_train['FamilySize'] = df_train['SibSp'] + df_train['Parch'] + 1
df_all['FamilySize'] = df_all['SibSp'] + df_all['Parch'] + 1<create_dataframe>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
Actual_1 = 'Twitter Sentiment Analysis'
Predict_1 = 'Sentiment Analysis'
Actual_2 = 'Twitter Sentiment Analysis'
Predict_2 = 'Analysis'
print("Jaccard score for first set of scentences: {}".format(jaccard(Actual_1,Predict_1)))
 print("Jaccard score for second set of scentences: {}".format(jaccard(Actual_2,Predict_2)) )
family_passengers = df_train['FamilySize'].value_counts(normalize=True, sort=False ).to_frame().reset_index()
family_passengers.columns = ['FamilySize', '%total']
print('{:=<70}'.format(''))
print(family_passengers)
 print('{:=<70}'.format(''))<merge>
ori_train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 ori_test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train = ori_train
test = ori_test
ori_train = ori_train.fillna("")
 ori_test = ori_test.fillna("")
 train.head(10)
 print("There are {} rows and {} columns in train file".format(train.shape[0],train.shape[1]))
print("There are {} rows and {} columns in test file".format(test.shape[0],test.shape[1]))
df_family = pd.merge(left=family_passengers,
right=family_survived,
how='inner',
on='FamilySize')
 df_family_melted = df_family.melt(id_vars=['FamilySize'],
var_name='ratio type',
value_name='ratio value')
 ax = sns.catplot(data=df_family_melted,
x='ratio type',
y='ratio value',
col='FamilySize',
col_wrap=9,
kind='point',
height=3,
aspect=0.8)
 ax.set_axis_labels('', 'Percentage')
 plt.subplots_adjust(top=0.7)
 plt.suptitle('Proportions of passengers: total and survivors')
 plt.show()<compute_test_metric>
train = ori_train
test = ori_test
get_chi2(summary_family )<create_dataframe>
def count_values(df,feature):
total=df.loc[:,feature].value_counts(dropna=False)
 percent=round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'] )
df_median_age = pd.pivot_table(data=df_age,
index=['Sex','Pclass'],
values='Age',
aggfunc=np.median ).reset_index()
def fill_age(data):
median = df_median_age
age = data['Age']
sex = data['Sex']
pclass = data['Pclass']
if pd.isnull(age):
if sex == 'female' and pclass == 1:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 1)]['Age'].values[0]
if sex == 'female' and pclass == 2:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 2)]['Age'].values[0]
if sex == 'female' and pclass == 3:
return median[(median['Sex'] == 'female')&(median['Pclass'] == 3)]['Age'].values[0]
if sex == 'male' and pclass == 1:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 1)]['Age'].values[0]
if sex == 'male' and pclass == 2:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 2)]['Age'].values[0]
if sex == 'male' and pclass == 3:
return median[(median['Sex'] == 'male')&(median['Pclass'] == 3)]['Age'].values[0]
else:
return age
df_train['Age'] = df_train[['Age', 'Sex', 'Pclass']].apply(fill_age, axis=1)
 df_all['Age'] = df_all[['Age', 'Sex', 'Pclass']].apply(fill_age, axis=1)
 print('{:=<70}'.format(''))
print('Missing age values in the combined dataset:')
 print(df_all['Age'].isnull().sum())
 print('{:=<70}'.format(''))<count_values>
def hover(hover_color="
return dict(selector="tr:hover",
props=[("background-color", "%s" % hover_color)] )
print(df_train['Embarked'].value_counts(normalize=True))<categorify>
def find_link(string):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return "".join(url)
 train['target_url']=train['selected_text'].apply(lambda x: find_link(x))
df2=pd.DataFrame(train.loc[train['target_url']!=""]['sentiment'].value_counts() ).reset_index()
df2.rename(columns={"index": "sentiment", "sentiment": "url_count"},inplace = True)
 styles = [
hover() ,
dict(selector="th", props=[("font-size", "150%"), 
("text-align", "center")]),
dict(selector="caption", props=[("caption-side", "bottom")])
 ]
html =(df2.style.set_table_styles(styles)
.set_caption("Hover to highlight."))
html
df_train['Embarked'].fillna(value='S', inplace=True)
 df_all['Embarked'].fillna(value='S', inplace=True )<compute_test_metric>
def create_corpus_text(target):
corpus=[]
for x in train[train['sentiment']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
get_chi2(summary_embarked )<filter>
corpus=create_corpus_text("positive")
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top_0=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:20]
corpus=create_corpus_text("negative")
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top_1=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:20]
corpus=create_corpus_text("neutral")
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top_2=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:20]
df_all[df_all['Fare'].isnull() == True]<data_type_conversions>
def create_corpus_selected_text(target):
corpus=[]
for x in train[train['sentiment']==target]['selected_text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
median_fare = df_train['Fare'][df_train['Pclass']==3].median()
df_all['Fare'].fillna(value=median_fare, inplace=True )<feature_engineering>
print('TF version',tf.__version__ )
d_ticket_count = dict(df_all['Ticket'].value_counts())
 df_all['TicketCount'] = df_all['Ticket'].map(d_ticket_count)
 df_all['AdjFare'] = df_all['Fare'] / df_all['TicketCount']
df_train['AdjFare'] = df_all[df_all['Dataset'] == 'train']['AdjFare']<feature_engineering>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = ori_train
train.head()
df_all['LastName'] = df_all['Name'].str.extract(pat= '^([^,]*),', expand=True)
<feature_engineering>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
df_all['TicketNum'] = df_all['Ticket'].str.replace(pat= '(\D)', repl= '')
<count_values>
test = ori_test
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
d_ticket_count = dict(df_all['Ticket'].value_counts())
 df_all['TicketCount'] = df_all['Ticket'].map(d_ticket_count )<feature_engineering>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
df_all['SharedTicket'] = [1 if i > 1 else 0 for i in df_all['TicketCount']]
d_shared_name = dict(df_all['LastName'].value_counts())
 df_all['SharedName'] = [1 if i > 1 else 0 for i in df_all['LastName'].map(d_shared_name)]
df_all['SharedFeatures'] = df_all['SharedTicket'] + df_all['SharedName']<define_variables>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.ReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.ReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
shared_features = [
df_all['SharedFeatures'] == 0,
df_all['SharedFeatures'] == 1, 
(df_all['SharedTicket'] == 0)&(df_all['SharedName'] == 1)&(df_all['FamilySize'] == 1), 
(df_all['SharedTicket'] == 0)&(df_all['SharedName'] == 1)&(df_all['FamilySize'] > 1), 
(df_all['SharedTicket'] == 1)&(df_all['SharedName'] == 0)
 ]
is_single = [1, 0, 1, 0, 0]
df_all['IsSingle'] = np.select(shared_features, is_single )<import_modules>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix<categorify>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
df_all['Sex'] = [1 if i == 'female' else 0 for i in df_all['Sex']]
l_predictors = ['Dataset', 'Survived', 'Pclass', 'Sex', 'Embarked', 'IsSingle', 'FamilySize']
df_final = df_all[l_predictors]
df_final = pd.get_dummies(data = df_final,
columns = ['Pclass', 'Embarked', 'FamilySize'] )<split>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
<train_model><EOS> 
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )