Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<define_search_space>
print('TF version',tf.__version__ )
xgb_params = {"learning_rate": [0.1,0.01,0.5],
"max_depth": [2,3,4,5,8],
"n_estimators": [100,200,500,1000,3000,6000],
"colsample_bytree": [0.4,0.7,1],
"min_child_weight":[0,1,2],
"gamma":[0.6,0.8,0.2],
"subsample":[0.7,0.5,0.9],
"colsample_bytree":[0.7,0.01,0.2],
"nthread":[-1,-2],
"scale_pos_weight":[1,2,5,7],
"seed":[27,5,40,60],
"reg_alpha":[0.0000,0.005,0.5,0.00006]}<choose_model_class>
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].str.lower().astype(str)
 train['selected_text']=train['selected_text'].str.lower().astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].str.lower().astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
xgb_tuned = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42 ).fit(X_train, y_train )<predict_on_test>
t = train_df[:27000]
v = train_df[27000:]
train_df = t
validation_df = v
validation_df.reset_index(inplace=True,drop=True )
y_pred = xgb_tuned.predict(X_test )<compute_test_metric>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
np.sqrt(mean_squared_error(y_test, y_pred)) 
<import_modules>
MAX_LEN = 100
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
from sklearn.metrics import mean_absolute_error<compute_test_metric>
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
print("Mean Absolute Error : " + str(mean_absolute_error(y_test, y_pred)) )<data_type_conversions>
ct = validation_df.shape[0]
input_ids_v = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_v = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_v = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(validation_df.shape[0]):
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[validation_df.loc[k,'sentiment']]
input_ids_v[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_v[k,:len(enc.ids)+5] = 1
ids=test["Id"]
test= test.drop(["Id"], axis=1 ).astype("float64" )<save_to_csv>
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
prediction=xgb_tuned.predict(test)
 output=pd.DataFrame({"Id":ids, "SalePrice":prediction})
 output.to_csv("submission_xgb.csv", index=False )<set_options>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(2 )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(2 )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
%matplotlib inline<load_from_csv>
n_splits = 4
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
jac = []; VER='v1'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)
 sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=4, batch_size=8, verbose=DISPLAY, callbacks=[sv, reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('./v1-roberta-%i.h5'%(fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 all = []
for k in idxV:
start_sorted_indexes = np.argsort(-oof_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-oof_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
st = train_df.loc[k,'text']
all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<concatenate>
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition',],
test.loc[:,'MSSubClass':'SaleCondition',]))<feature_engineering>
all = []
counter = 0
for k in range(input_ids_t.shape[0]):
start_sorted_indexes = np.argsort(-preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = test_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on test set" )
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<categorify>
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )
all_data = pd.get_dummies(all_data )<data_type_conversions>
train_preds_start = np.zeros(( input_ids.shape[0],MAX_LEN))
train_preds_end = np.zeros(( input_ids.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Train...')
 train_preds = model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 train_preds_start += train_preds[0]/n_splits
train_preds_end += train_preds[1]/n_splits
all_data = all_data.fillna(all_data.mean())
 all_data<prepare_x_and_y>
all = []
counter = 0
for k in range(input_ids.shape[0]):
start_sorted_indexes = np.argsort(-train_preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-train_preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = train_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on train set" )
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<compute_train_metric>
train_df['predicted_text'] = all
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 10))
return(rmse )<choose_model_class>
validation_preds_start = np.zeros(( input_ids_v.shape[0],MAX_LEN))
validation_preds_end = np.zeros(( input_ids_v.shape[0],MAX_LEN))
DISPLAY=1
for i in range(n_splits):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('./v1-roberta-%i.h5'%(i))
print('Predicting Validation...')
 validation_preds = model.predict([input_ids_v,attention_mask_v,token_type_ids_v],verbose=DISPLAY)
 validation_preds_start += validation_preds[0]/n_splits
validation_preds_end += validation_preds[1]/n_splits
model_ridge = Ridge()<define_search_space>
all = []
counter = 0
for k in range(input_ids_v.shape[0]):
start_sorted_indexes = np.argsort(-validation_preds_start[k,], kind='quicksort', order=None)
 end_sorted_indexes = np.argsort(-validation_preds_end[k,], kind='quicksort', order=None)
 if start_sorted_indexes[0] <= end_sorted_indexes[0]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[0]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[0] <= end_sorted_indexes[1]:
a = start_sorted_indexes[0]
b = end_sorted_indexes[1]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[0]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[0]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 elif start_sorted_indexes[1] <= end_sorted_indexes[1]:
a = start_sorted_indexes[1]
b = end_sorted_indexes[1]
text1 = " "+" ".join(validation_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 else:
counter += 1
st = validation_df.loc[k,'text']
all.append(st)
 print(counter, " row cant predicted on validation set" )
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75,100,125,150,175,200,225,250]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]<train_on_grid>
validation_df['predicted_text'] = all
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y )<find_best_params>
cumulativeJaccard = []
for index, row in validation_df.iterrows() :
cumulativeJaccard.append(jaccard(row["predicted_text"],row['selected_text']))
print("Roberta Jaccard on validation = ", np.mean(cumulativeJaccard), " Expected value is ~0.711" )
rmse_cv(model_lasso ).mean()<compute_test_metric>
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in train_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["selected_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 train_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 train_df_se
print("Lasso regularization selected " + str(sum(coef != 0)) + "variables and eliminated " + str(sum(coef == 0)) + " variables" )<concatenate>
for index, row in train_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )
imp_coef = pd.concat([coef.sort_values().head(10),
coef.sort_values().tail(10)] )<import_modules>
counter = 0
for index, row in train_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Sentiment = ",row["sentiment"].upper() + "--" +row["position"])
 print("Real word = ",row["text_se"])
 print("Selected word = ",row["selected_text_se"])
 print("==========================================")
 print("Different word number in train set =", counter )
import xgboost as xgb<train_on_grid>
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in validation_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["predicted_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 validation_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 validation_df_se
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=700, early_stopping_rounds=150 )<train_model>
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )
model_xgb = xgb.XGBRegressor(n_estimators=500, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X_train, y )<predict_on_test>
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in test_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["selected_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 test_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 test_df_se
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))<create_dataframe>
for index, row in test_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )<prepare_output>
counter = 0
for index, row in test_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Sentiment = ",row["sentiment"].upper() + "--" +row["position"])
 print("Real word = ",row["text_se"])
 print("Selected word = ",row["selected_text_se"])
 print("==========================================")
 print("Different word number in test set =", counter )
preds = 0.7*lasso_preds + 0.3*xgb_preds<save_to_csv>
spacy.prefer_gpu()
output = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 output.to_csv("ridge_sol.csv", index = False )<load_from_csv>
def save_model(output_dir, nlp, new_model_name):

 output_dir = f'.. /working/{output_dir}'
if output_dir is not None:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 nlp.meta["name"] = new_model_name
nlp.to_disk(output_dir)
 print("Saved model to", output_dir )
train = pd.read_csv('.. /input/train.csv',index_col = 'Id')
 test = pd.read_csv('.. /input/test.csv',index_col = 'Id' )<feature_engineering>
def train(train_data, output_dir, n_iter=20, cont=False):

 ""
if cont and os.path.exists(output_dir):
nlp = spacy.load(output_dir)
 print("Loaded model '%s'" % output_dir)
 else:
nlp = spacy.blank("en")
 print("Created blank 'en' model")
 if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
 nlp.add_pipe(ner, last=True)
 else:
ner = nlp.get_pipe("ner")
 for _, annotations in train_data:
for ent in annotations.get("entities"):
ner.add_label(ent[2])
 other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
with nlp.disable_pipes(*other_pipes):
if cont and os.path.exists(output_dir):
nlp.resume_training()
else:
nlp.begin_training()
for itn in tqdm(range(n_iter)) :
random.shuffle(train_data)
 batches = minibatch(train_data, size=compounding(4.0, 500.0, 1.001))
losses = {}
for batch in batches:
texts, annotations = zip(*batch)
 nlp.update(texts,
annotations,
drop=0.5,
losses=losses, 
)
 print("Losses", losses)
 save_model(output_dir, nlp, 'st_ner' )
train.price= np.log1p(train["price"] )<filter>
def get_model_out_path(position):

 model_out_path = None
if position == 'Start':
model_out_path = 'models/model_Start'
elif position == 'End':
model_out_path = 'models/model_End'
return model_out_path
train = train[~(( train.bathrooms>6)&(train.price<14)) ]<prepare_x_and_y>
position = 'Start'
train_data = get_training_data(train_df_se, position)
 model_path = get_model_out_path(position)
 train(train_data, model_path, n_iter=3, cont=False )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.price.values
all_data = pd.concat(( train, test))
all_data.drop(['price'], axis=1, inplace=True)
 all_data.shape 
<create_dataframe>
position = 'End'
train_data = get_training_data(train_df_se, position)
 model_path = get_model_out_path(position)
 train(train_data, model_path, n_iter=3, cont=False )
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )<feature_engineering>
def predict_entities(text, model):
doc = model(text)
 ent_array = []
for ent in doc.ents:
start = text.find(ent.text)
 end = start + len(ent.text)
 new_int = [start, end, ent.label_]
if new_int not in ent_array:
ent_array.append([start, end, ent.label_])
 if len(ent_array)> 0:
selected_text = text[ent_array[0][0]: ent_array[0][1]]
else:
selected_text = text
return selected_text
all_data['zipcode'] = all_data['zipcode'].apply(str )<feature_engineering>
selected_texts = []
MODELS_BASE_PATH = '.. /working/models/'
if MODELS_BASE_PATH is not None:
print("Loading Models from ", MODELS_BASE_PATH)
 model_Start = spacy.load(MODELS_BASE_PATH + 'model_Start')
 model_End = spacy.load(MODELS_BASE_PATH + 'model_End')
 for index, row in validation_df_se.iterrows() :
text = row.text_se
output_str = ""
if row.position == 'Start-Neutral' or row.position == 'End-Neutral':
selected_texts.append(text)
 elif row.position == 'Start':
selected_texts.append(predict_entities(text, model_Start))
elif row.position == 'End':
selected_texts.append(predict_entities(text, model_End))
validation_df_se['ner_predicted_text'] = selected_texts
all_data['YrSold'] =all_data['date'].apply(lambda s:int(s[:4]))
all_data['MonthSold'] =all_data['date'].apply(lambda s:(s[4:6]))
all_data['DaySold'] =all_data['date'].apply(lambda s:int(s[6:8]))
all_data.drop(['date'], axis=1, inplace=True)
 all_data['rooms'] = all_data['bedrooms']+all_data['bathrooms']
all_data.head()<feature_engineering>
counter = 0
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Different word count in validation set =",counter )
numerical_cols = [cname for cname in all_data.columns if
all_data[cname].dtype in ['int64', 'float64']]
skewness = all_data[numerical_cols].apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.9]
print(str(skewness.shape[0])+ " skewed numerical features to log transform")
 skewed_features = list(skewness.index)
 all_data[skewed_features] = np.log1p(all_data[skewed_features] )<categorify>
counter = 0
for index, row in validation_df_se.iterrows() :
if row["text_se"] == row["selected_text_se"]:
counter+=1
print("Same word count in validation set =",counter )
Dummies_all_data = pd.get_dummies(all_data )<prepare_x_and_y>
counter = 0
for index, row in validation_df_se.iterrows() :
if row["ner_predicted_text"] != row["text_se"]:
counter+=1
print("Changed word by NER in validation set =",counter )
X_train = Dummies_all_data[:ntrain]
X_test = Dummies_all_data[ntrain:]<compute_train_metric>
counter = 0
for index, row in validation_df_se.iterrows() :
if row["ner_predicted_text"] != row["text_se"]:
if row["ner_predicted_text"] != row["selected_text_se"]:
counter+=1
print("Changed word by NER in validation set but unsuccesfull changing=",counter )
n_folds=5
def rmsle_cv(model):
rmsle= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 5))
print("
score rmsle: {:.4f}({:.4f})
 ".format(rmsle.mean() , rmsle.std()))
return(rmsle)
<choose_model_class>
predict_entities("badly??!?!??!!?", model_End )
model_xgb = XGBRegressor(colsample_bytree=1, gamma=0.0468,
learning_rate=0.053, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 score_xgb = rmsle_cv(model_xgb)
<choose_model_class>
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
print(row["text_se"], row["selected_text_se"] )
model_LGB = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score_LGB = rmsle_cv(model_LGB )<choose_model_class>
for index, row in validation_df.iterrows() :
predicted_words = row["predicted_text"].split()
predicted_words[0] = validation_df_se["ner_predicted_text"][2*index]
predicted_words[-1] = validation_df_se["ner_predicted_text"][(2*index)+1]
validation_df["predicted_text"][index] = " ".join(predicted_words )
model_GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 score_GBoost = rmsle_cv(model_GBoost )<compute_train_metric>
totalJaccard = 0
for index, row in validation_df.iterrows() :
predict = row.predicted_text
true = row.selected_text
j = jaccard(predict, true)
 totalJaccard += j
meanJaccard = totalJaccard / len(validation_df['predicted_text'])
 print("NER Validation results : ", meanJaccard )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
score_ENet = rmsle_cv(ENet )<choose_model_class>
selected_texts = []
MODELS_BASE_PATH = '.. /working/models/'
if MODELS_BASE_PATH is not None:
print("Loading Models from ", MODELS_BASE_PATH)
 model_Start = spacy.load(MODELS_BASE_PATH + 'model_Start')
 model_End = spacy.load(MODELS_BASE_PATH + 'model_End')
 for index, row in test_df_se.iterrows() :
text = row.text_se
output_str = ""
if row.position == 'Start-Neutral' or row.position == 'End-Neutral':
selected_texts.append(text)
 elif row.position == 'Start':
selected_texts.append(predict_entities(text, model_Start))
elif row.position == 'End':
selected_texts.append(predict_entities(text, model_End))
test_df_se['ner_predicted_text'] = selected_texts
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
score_lasso = rmsle_cv(lasso )<train_model>
for index, row in test_df.iterrows() :
predicted_words = row["selected_text"].split()
predicted_words[0] = test_df_se["ner_predicted_text"][2*index]
predicted_words[-1] = test_df_se["ner_predicted_text"][(2*index)+1]
test_df["selected_text"][index] = " ".join(predicted_words )
<import_modules><EOS> 
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )