Kaggle competition: Tweet Sentiment Extraction
mean_sale_subclass=train.groupby(['MSSubClass'] ).mean()
df_mean_sale_subclass=pd.DataFrame(mean_sale_subclass['SalePrice'].sort_values(ascending=True)).reset_index()
df_mean_sale_subclass[:2]<define_variables>
print('TF version',tf.__version__)
subclass_order=list(df_mean_sale_subclass['MSSubClass'])
 subclass_order<categorify>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
qual_map = {0: 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5}
qual_col=['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond','ExterQual',
'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual']
for col in qual_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(qual_map)
 furnit_map = {0: 0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ':6 }
furn_col=['BsmtFinType1', 'BsmtFinType2']
for col in furn_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(furnit_map)
 functional_map={0: 0, 'Sal': 1, 'Sev': 2, 'Maj2': 3, 'Maj1': 4, 'Mod': 5, 'Min2':6,'Min1': 7, 'Typ':8}
func_col=['Functional']
for col in func_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(functional_map)
 fence_map={0: 0, 'MnWw': 1, 'GdWo': 2, 'GdWo': 3, 'MnPrv': 4, 'GdPrv': 5}
fence_col=['Fence']
for col in fence_col:
all_data_rep_enc_1[col] = all_data_rep_enc_1[col].map(fence_map)
 bsm_exp_map={0: 0, 'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4}
all_data_rep_enc_1['BsmtExposure'] = all_data_rep_enc_1['BsmtExposure'].map(bsm_exp_map)
 garag_fin_map={0: 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}
all_data_rep_enc_1['GarageFinish'] = all_data_rep_enc_1['GarageFinish'].map(garag_fin_map)
 slope_map={'Sev': 1, 'Mod': 2, 'Gtl': 3}
all_data_rep_enc_1['LandSlope'] = all_data_rep_enc_1['LandSlope'].map(slope_map)
 shape_map={'IR3': 1, 'IR2': 2, 'IR1': 3, 'Reg': 4}
all_data_rep_enc_1['LotShape'] = all_data_rep_enc_1['LotShape'].map(shape_map)
 pave_map={'N': 1, 'P': 2, 'Y': 3}
all_data_rep_enc_1['PavedDrive'] = all_data_rep_enc_1['PavedDrive'].map(pave_map)
 street_map={0:0,'Pave': 1, 'Grvl': 2}
all_data_rep_enc_1['Street'] = all_data_rep_enc_1['Street'].map(street_map)
 alley_map={0: 0, 'Pave': 1, 'Grvl': 2}
all_data_rep_enc_1['Alley'] = all_data_rep_enc_1['Alley'].map(alley_map)
 air_map={ 'N': 1, 'Y': 2}
all_data_rep_enc_1['CentralAir'] = all_data_rep_enc_1['CentralAir'].map(air_map)
 class_map={0:0,30:1, 180:2, 45:3, 190:4, 90:5, 160:6, 50:7, 85:8, 40:9, 70:10, 80:11, 20:12, 75:13, 120:14, 150:14, 60:15}
all_data_rep_enc_1['MSSubClass'] = all_data_rep_enc_1['MSSubClass'].map(class_map)
 all_data_rep_enc_1['YrSold'].astype('int')
 mo_map={0:0, 4:1, 5:2, 6:7, 7:7.5, 8:9, 9:11, 10:2.5, 11:8.5, 2:8.5, 12:9.5, 1:1.5, 3:7.5}
all_data_rep_enc_1['MoSold'] = all_data_rep_enc_1['MoSold'].map(mo_map )<sort_values>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
all_data_rep_enc_1.isnull().sum().sort_values(ascending=False)[:3]<feature_engineering>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
train["SalePrice"] = np.log1p(train["SalePrice"])
 check_skewness('SalePrice' )<prepare_x_and_y>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
y_train=train["SalePrice"]<filter>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes=='object'].index<feature_engineering>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.3
for feat in skewed_features:
all_data_rep_enc_1[feat] = boxcox1p(all_data_rep_enc_1[feat], lam )<filter>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
all_data_rep_enc_1.dtypes[all_data_rep_enc_1.dtypes== "object"].index 
<count_unique_values>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<create_dataframe><EOS> 
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )