Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<prepare_x_and_y>
warnings.filterwarnings("ignore" )
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
%%javascript
IPython.OutputArea.prototype._should_scroll = function() {
return False;
}
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
!apt-get install p7zip
warnings.filterwarnings('ignore' )<set_options>
train_data = pd.read_csv("train.tsv", sep='\t')
 test_data = pd.read_csv(".. /input/mercari-price-suggestion-challenge/test_stg2.tsv.zip" , sep='\t' )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
train_copy = train_data.copy()
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
gc.collect()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<train_model>
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label" )
def dist_between(start_logits, end_logits, device='cpu', max_seq_len=96):

 linear_func = torch.tensor(np.linspace(0, 1, max_seq_len, endpoint=False), requires_grad=False)
 linear_func = linear_func.to(device)
 start_pos =(start_logits*linear_func ).sum(axis=1)
 end_pos =(end_logits*linear_func ).sum(axis=1)
 diff = end_pos-start_pos
return diff.sum(axis=0)/diff.size(0)
 def dist_loss(start_logits, end_logits, start_positions, end_positions, device=torch.device('cuda'), max_seq_len=96, scale=1):
start_logits = torch.nn.Softmax(1 )(start_logits)
 end_logits = torch.nn.Softmax(1 )(end_logits)
 start_one_hot = torch.nn.functional.one_hot(start_positions, num_classes=max_seq_len ).to(device)
 end_one_hot = torch.nn.functional.one_hot(end_positions, num_classes=max_seq_len ).to(device)
 pred_dist = dist_between(start_logits, end_logits, device, max_seq_len)
 gt_dist = dist_between(start_one_hot, end_one_hot, device, max_seq_len)
 diff =(gt_dist-pred_dist)
 rev_diff_squared = 1-torch.sqrt(diff*diff)
 loss = -torch.log(rev_diff_squared)
 return loss*scale<compute_test_metric>
train_copy['main_category'], train_copy['subcat_1'], train_copy['subcat_2'] = zip(*train_copy['category_name'].apply(lambda x: split_cat(x)) )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 idx_loss = start_loss + end_loss
distan_loss = dist_loss(start_logits, end_logits, start_positions, end_positions)
 return idx_loss+distan_loss<statistical_test>
def toNumeric(data,to):
if train_copy[data].dtype == type(object):
le = preprocessing.LabelEncoder()
train_copy[to] = le.fit_transform(train_copy[data].astype(str))
toNumeric('name','n_name')
 toNumeric('category_name','n_category_name')
 toNumeric('brand_name','n_brand_name')
 toNumeric('main_category','n_main_category')
 toNumeric('subcat_1','n_subcat_1')
 toNumeric('subcat_2','n_subcat_2')
 train_copy.head()
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
train_copy.isnull().any()
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
def fill_missing_data(data):
data.category_name.fillna(value = "Other/Other/Other", inplace = True)
 data.brand_name.fillna(value = "Unknown brand", inplace = True)
 data.item_description.fillna(value = "No description", inplace = True)
 return data
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
train_copy = fill_missing_data(train_copy)
 train_copy = train_copy.dropna()
print(np.shape(train_copy))
train_copy.head()
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
sns.set(context='notebook')
 sns.set_style("whitegrid", {'axes.grid' : False})
 plt.tight_layout()
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
train_copy['price'] = np.log1p(train_copy['price'] )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
gc.collect()
import pandas as pd
import numpy as np
import torch
import os
import sklearn
import json 
<install_modules>
r_data= train_copy[['item_condition_id','shipping','n_name','n_brand_name','n_main_category','n_subcat_1','n_subcat_2']]
X_train, x_test, Y_train, y_test = train_test_split(r_data, train_copy['price'], test_size=0.25, random_state=12345)
 def run_model(model, X_train, Y_train, x_test, y_test, verbose = False):
Y_train = Y_train[:, np.newaxis].ravel()
model.fit(X_train, Y_train)
 y_predict = model.predict(x_test)
 mse = mean_squared_error(y_test,y_predict)
 r_sq = r2_score(y_test,y_predict)
 print("Mean Squared Error Value : "+"{:.2f}".format(mse))
print("R-Squared Value : "+"{:.2f}".format(r_sq))
return model, mse, r_sq
!pip install '.. /input/required-packages/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '.. /input/required-packages/simpletransformers-0.22.1-py3-none-any.whl' -q<define_variables>
ridge_reg = linear_model.Ridge()
print("Ridge Regression")
 print("----------------")
 model_1, mse_1, r_sq_1 = run_model(ridge_reg, X_train, Y_train, x_test, y_test )
INPUT_FILES_PATH = "/kaggle/input/tweet-sentiment-extraction/"<load_from_csv>
lgbm_reg = lightgbm.LGBMRegressor()
print("LGBM Regression")
 print("---------------")
 model_2, mse_2, r_sq_2 = run_model(lgbm_reg, X_train, Y_train, x_test, y_test )
df_train = pd.read_csv(INPUT_FILES_PATH + "train.csv")
 df_train = df_train.dropna()
df_train.reset_index(drop=True, inplace=True)
 df_test = pd.read_csv(INPUT_FILES_PATH + "test.csv")
 df_test = df_test.dropna()
df_test.reset_index(drop=True, inplace=True)
 sample_submission = pd.read_csv(INPUT_FILES_PATH + "sample_submission.csv")
<set_options>
xgb_params = {'n_estimators':500, 'max_depth':8}
xgb_reg = xgboost.XGBRegressor(**xgb_params)
 print("XGBoost Regression")
 print("------------------")
 model_3, mse_3, r_sq_3 = run_model(xgb_reg, X_train, Y_train, x_test, y_test )
use_cuda = torch.cuda.is_available()
use_cuda<string_transform>
combined_data = pd.concat([train_data,test_data])
 train_size = len(train_data)
 submission = test_data[['test_id']]
def getSelectedTextStartIndex(text, selected_text):
return text.lower().find(selected_text.lower())
 start_indices = []
for tupl in df_train.itertuples() :
start_indices.append(getSelectedTextStartIndex(tupl.text, tupl.selected_text))<train_model>
combined_frac = combined_data.sample(frac=0.1 ).reset_index(drop=True )
<categorify>
punctuation
<categorify>
punctuation_symbols = []
for symbol in punctuation:
punctuation_symbols.append(( symbol, ''))
def train_preprocessing(df_train):
out = []
for tupl in df_train.itertuples() :
question = tupl.sentiment
answer = tupl.selected_text
context = tupl.text
qid = tupl.Index
answer_start = start_indices[qid]
ans = [{"answer_start": answer_start, "text": answer.lower() }]
qas = [{"question": question, "id": qid, "is_impossible": False, "answers": ans}]
out.append({"context": context.lower() , "qas": qas})
 return out
def test_preprocessing(df_test):
out = []
for tupl in df_test.itertuples() :
qas = []
con = []
ans = []
question = tupl.sentiment
context = tupl.text
qid = tupl.Index
ans = [{"answer_start": 1000000, "text": "__None__"}]
qas = [{"question": question, "id": qid, "is_impossible": False, "answers": ans}]
out.append({"context": context.lower() , "qas": qas})
 return out<choose_model_class>
def remove_punctuation(sentence: str)-> str:
return sentence.translate(str.maketrans('', '', string.punctuation))
BERT_PATH = ".. /input/required-packages/distilbert-base-uncased-distilled-squad"
start = time.time()
args={
'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 4,
'doc_stride': 64,
'fp16': False,
}
QA_model_pos = QuestionAnsweringModel('distilbert', BERT_PATH, args=args, use_cuda=use_cuda)
 QA_model_neg = QuestionAnsweringModel('distilbert', BERT_PATH, args=args, use_cuda=use_cuda)
 end = time.time()
print(end - start )<train_model>
def remove_digits(x):
x = ''.join([i for i in x if not i.isdigit() ])
 return x
train_data = train_preprocessing(df_train[df_train["sentiment"] == "positive"])
 QA_model_pos.train_model(train_data = train_data, output_dir=MODEL_POS, show_running_loss=True )<train_model>
stop = stopwords.words('english')
 def remove_stop_words(x):
x = ' '.join([i for i in x.lower().split(' ')if i not in stop])
 return x
train_data = train_preprocessing(df_train[df_train["sentiment"] == "negative"])
 QA_model_neg.train_model(train_data = train_data, output_dir=MODEL_POS, show_running_loss=True )<predict_on_test>
def to_lower(x):
return x.lower()
predictions_pos = QA_model_pos.predict(test_preprocessing(df_test_pos))<prepare_output>
def transform_category_name(category_name):
try:
main, sub1, sub2= category_name.split('/')
 return main, sub1, sub2
except:
return np.nan, np.nan, np.nan
train_data['category_main'], train_data['subcat_1'], train_data['subcat_2'] = zip(*train_data['category_name'].apply(transform_category_name))
cat_train = train_data[['category_main','subcat_1','subcat_2', 'price']]
df_test_neg = df_test[df_test["sentiment"]=="negative"].copy()
df_test_neg.head()<predict_on_test>
gc.collect()
predictions_neg = QA_model_neg.predict(test_preprocessing(df_test_neg))<create_dataframe>
combined_data.item_description = combined_data.item_description.astype(str)
 descr = combined_data[['item_description', 'price']]
descr['count'] = descr['item_description'].apply(lambda x : len(str(x)))
 descr['item_description'] = descr['item_description'].apply(remove_digits)
 descr['item_description'] = descr['item_description'].apply(remove_punctuation)
 descr['item_description'] = descr['item_description'].apply(remove_stop_words)
 descr.head(20 )
prediction_test_pos = pd.DataFrame.from_dict(predictions_pos)
 prediction_test_pos.index = df_test_pos.index.copy()
prediction_test_neg = pd.DataFrame(predictions_neg)
 prediction_test_neg.index = df_test_neg.index.copy()
prediction_test_neu = df_test[df_test["sentiment"] == "neutral"].copy()
prediction_test_neu.rename(columns = {"text": "answer" }, inplace=True)
 submission_df = pd.concat([prediction_test_pos, prediction_test_neg, prediction_test_neu], axis=0)
 submission_df.sort_index(inplace=True )<save_to_csv>
porter = PorterStemmer()
descr['item_description'] = descr['item_description'].apply(porter.stem)
 descr.tail(20 )
sample_submission["selected_text"] = submission_df["answer"]
sample_submission.to_csv("submission.csv", index = False)
 print("File submitted successfully.")
<import_modules>
def handle_missing_values(df):
df['category_name'].fillna(value='missing', inplace=True)
 df['brand_name'].fillna(value='None', inplace=True)
 df['item_description'].fillna(value='None', inplace=True )
print('TF version',tf.__version__)
<define_variables>
def to_categorical(df):
df['brand_name'] = df['brand_name'].astype('category')
 df['category_name'] = df['category_name'].astype('category')
 df['item_condition_id'] = df['item_condition_id'].astype('category' )
max_len = 128
train_mode = False
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file = '.. /input/tf-roberta/vocab-roberta-base.json',
merges_file = '.. /input/tf-roberta/merges-roberta-base.txt',
add_prefix_space = True,
lowercase = True )<load_from_csv>
handle_missing_values(combined_frac)
 to_categorical(combined_frac )
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
data = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 data.head()<correct_missing_values>
handle_missing_values(combined_data)
 to_categorical(combined_data )
data['text'].replace('', np.nan, inplace=True)
 data.dropna(subset=['text'], inplace=True)
 data.reset_index(drop=True, inplace=True )<split>
gc.collect()
x_train,x_test = train_test_split(data, test_size = 0.05, random_state=42)
 x_train,x_cv = train_test_split(x_train, test_size = 0.2, random_state = 42)
 print("x_train shape is", x_train.shape)
 print("x_cv shape is", x_cv.shape)
 print("x_test shape is", x_test.shape)
 x_train.reset_index(drop=True, inplace=True)
 x_cv.reset_index(drop=True, inplace=True)
 x_test.reset_index(drop=True, inplace=True )<define_variables>
combined_frac.item_description = combined_frac.item_description.astype(str)
 combined_frac['item_description'] = combined_frac['item_description'].apply(remove_digits)
 combined_frac['item_description'] = combined_frac['item_description'].apply(remove_punctuation)
 combined_frac['item_description'] = combined_frac['item_description'].apply(remove_stop_words)
 combined_frac['item_description'] = combined_frac['item_description'].apply(to_lower)
 combined_frac['name'] = combined_frac['name'].apply(remove_digits)
 combined_frac['name'] = combined_frac['name'].apply(remove_punctuation)
 combined_frac['name'] = combined_frac['name'].apply(remove_stop_words)
 combined_frac['name'] = combined_frac['name'].apply(to_lower)
 combined_frac.head()
tweet = 'WHY DO WE FALL? SO THAT WE CAN LEARN TO PICK OURSELVES BACK UP.'
selected_text = 'LEARN TO PICK OURSELVES BACK UP'
sentiment = 'positive'
print('text',tweet)
 print('select_text:',selected_text)
 print('sentiment:', sentiment)
 idx0 = None
idx1 = None
st_len = len(selected_text)
 tweet = " " + tweet
for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
print('char_targets:',char_targets)
 tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
print('input_ids:',input_ids)
 print('offsets:',offsets)
 targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
print('target_start:',target_start)
 print('target_end:', target_end)
 ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
<categorify>
combined_data.item_description = combined_data.item_description.astype(str)
 combined_data['item_description'] = combined_data['item_description'].apply(remove_digits)
 combined_data['item_description'] = combined_data['item_description'].apply(remove_punctuation)
 combined_data['item_description'] = combined_data['item_description'].apply(remove_stop_words)
 combined_data['item_description'] = combined_data['item_description'].apply(to_lower)
 combined_data['name'] = combined_data['name'].apply(remove_digits)
 combined_data['name'] = combined_data['name'].apply(remove_punctuation)
 combined_data['name'] = combined_data['name'].apply(remove_stop_words)
 combined_data['name'] = combined_data['name'].apply(to_lower)
 combined_data.head()
def text_process(tweet, sentiment, tokenizer, max_len, selected_text=None):

 if selected_text!=None:
tweet = " " + tweet
idx0 = None
idx1 = None
st_len = len(selected_text)
 for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'target_start':target_start,
'target_end':target_end,
'offsets':offsets
}
else:
tweet = " " + tweet
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'offsets':offsets
} 
<categorify>
gc.collect()
rows = x_train.shape[0]
train_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_mask = np.zeros(( rows,max_len), dtype = 'int32')
 train_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 train_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_train.shape[0]):
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']<categorify>
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
i = 10
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']
tweet = x_train.loc[i,'text']
select_text = x_train.loc[i,'selected_text']
target_start = np.argmax(train_start_idx[i,])
 target_end = np.argmax(train_end_idx[i, ])
 offsets = encoding['offsets']
print('tweet:',tweet)
 print('selected_text:',select_text)
 print(tweet[offsets[target_start][0]:offsets[target_end][-1]] )<categorify>
cv = CountVectorizer(min_df=10)
 X_name = cv.fit_transform(combined_data['name'] )
rows = x_cv.shape[0]
cv_ids = np.zeros(( rows,max_len), dtype = 'int32')
 cv_mask = np.zeros(( rows,max_len), dtype = 'int32')
 cv_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 cv_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 cv_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_cv.shape[0]):
encoding = text_process(x_cv.loc[i,'text'], x_cv.loc[i,'sentiment'], tokenizer, max_len,x_cv.loc[i,'selected_text'])
 cv_ids[i] = encoding['ids']
cv_start_idx[i,encoding['target_start']] = 1
cv_end_idx[i, encoding['target_end']] = 1
cv_type_ids[i] = encoding['token_type_ids']
cv_mask[i] = encoding['mask']<categorify>
cv = CountVectorizer()
X_category = cv.fit_transform(combined_data['category_name'] )
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len,x_test.loc[i,'selected_text'])
 test_ids[i] = encoding['ids']
test_start_idx[i,encoding['target_start']] = 1
test_end_idx[i, encoding['target_end']] = 1
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask']<string_transform>
tv = TfidfVectorizer(max_features=55000, ngram_range=(1, 2), stop_words='english')
 X_description = tv.fit_transform(combined_data['item_description'] )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<load_pretrained>
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(combined_data['brand_name'] )
config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 print(config )<load_pretrained>
X_dummies = csr_matrix(pd.get_dummies(combined_data[['item_condition_id', 'shipping']], sparse=True ).values )
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 roberta = TFRobertaForQuestionAnswering.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5', config = config)
 x = roberta(ids, attention_mask = att, token_type_ids = tok)
 x1 = tf.keras.layers.Dropout(0.3 )(x[0])
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.3 )(x[1])
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs = [ids, att, tok], outputs=[x1, x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate = 3e-5)
 model.compile(loss= 'categorical_crossentropy', optimizer=optimizer)
 return model<set_options>
sparse_merge = hstack(( X_dummies, X_description, X_brand, X_category, X_name)).tocsr()
gc.collect()<train_model>
X_train_sparse = sparse_merge[:train_size]
X_test = sparse_merge[train_size:]
if train_mode:
model.fit([train_ids, train_mask, train_type_ids], [train_start_idx, train_end_idx],
epochs = 3,
batch_size = 16,
verbose = True,
callbacks = [checkpoint],
validation_data =([cv_ids,cv_mask,cv_type_ids], [cv_start_idx, cv_end_idx]),
shuffle = True)
 else:
model.load_weights('.. /input/tse-best-roberta/best_model(6 ).h5' )<define_variables>
kf = KFold(n_splits=3, shuffle=True, random_state=12345)
 y = np.log1p(train_data['price'])
 i = 0;
for train_indicies, valid_indicies in kf.split(X_train_sparse):
X_train, y_train = X_train_sparse[train_indicies], y[train_indicies]
X_valid, y_valid = X_train_sparse[valid_indicies], y[valid_indicies]
rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))<predict_on_test>
gc.collect()
preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]
preds_end += preds[1]<predict_on_test>
def run_model_advText(model, X_train, y_train, X_valid, y_valid, verbose = False):
model.fit(X_train, y_train)
 preds_valid = model.predict(X_valid)
 mse = mean_squared_error(y_valid,preds_valid)
 r_sq = r2_score(y_valid,preds_valid)
 print("Mean Squared Error Value : "+"{:.2f}".format(mse))
print("R-Squared Value : "+"{:.2f}".format(r_sq))
return model, mse, r_sq
score = 0
for k in range(x_test.shape[0]):
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
score+=jaccard(x_test.loc[k,'selected_text'], pred)
 score=score/x_test.shape[0]
print('score on local test_data',score )<categorify>
ridge_reg = linear_model.Ridge(solver = "saga", fit_intercept=False)
 print("Ridge Regression(After advanced Text Pre-processing)")
 print("-----------------------------------------------------")
 model_11, mse_11, r_sq_11 = run_model_advText(ridge_reg, X_train, y_train, X_valid, y_valid )
for k in range(0,x_test.shape[0],100):
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
print('text:', x_test.text[k])
 print('selected text:', x_test.selected_text[k])
 print('sentiment:',x_test.sentiment[k])
 print('predicted:', pred)
 print('jaccard_score:', jaccard(pred, x_test.loc[k,'selected_text']))
print(' 
<load_from_csv>
lgbm_reg = lightgbm.LGBMRegressor()
print("LGBM Regression(After advanced Text Pre-processing)")
 print("----------------------------------------------------")
 model_22, mse_22, r_sq_22 = run_model_advText(lgbm_reg, X_train, y_train, X_valid, y_valid )
x_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))<categorify>
xgb_params = {'n_estimators':500, 'max_depth':8}
xgb_reg = xgboost.XGBRegressor(**xgb_params)
 print("XGB Regression(After advanced Text Pre-processing)")
 print("---------------------------------------------------")
 model_33, mse_33, r_sq_33 = run_model_advText(xgb_reg, X_train, y_train, X_valid, y_valid )
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len)
 test_ids[i] = encoding['ids']
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask'] 
<predict_on_test>
mse_before = [mse_1, mse_2, mse_3]
r_sq_before = [r_sq_1, r_sq_2, r_sq_3]
mse_after = [mse_11, mse_22, mse_33]
r_sq_after = [r_sq_11, r_sq_22, r_sq_33]
model_data = {'Model': ['Ridge','LGBM','XGB'],
'MSE_without_Advanced_TextProcessing': mse_before,
'R_Square_without_Advanced_TextProcessing': r_sq_before,
'MSE_with_Advanced_TextProcessing': mse_after,
'R_Square_with_Advanced_TextProcessing': r_sq_after}
data_compare = pd.DataFrame(model_data)
 fig,(ax[0],ax[1])= plt.subplots(1,2, figsize=(13,4))
ax[0]=data_compare.plot(kind='line', x='Model', y='MSE_without_Advanced_TextProcessing', color='DarkBlue', linewidth=0.7, marker='o', markersize=6, ax=ax[0])
 ax[0]=data_compare.plot(kind='line', x='Model', y='MSE_with_Advanced_TextProcessing', secondary_y=False,color='Red', linewidth=0.7, marker='o', markersize=6, ax=ax[0])
 ax[1]=data_compare.plot(kind='line', x='Model', y='R_Square_without_Advanced_TextProcessing', color='DarkBlue', linewidth=0.7, marker='^', markersize=6, ax=ax[1])
 ax[1]=data_compare.plot(kind='line', x='Model', y='R_Square_with_Advanced_TextProcessing', secondary_y=False,color='Red', linewidth=0.7, marker='^', markersize=6, ax=ax[1])
 ax[0].set_title("Improvement in MSE after Advanced Text Processing", fontsize = 13)
 ax[0].set_ylabel("Mean Square Error")
 ax[1].set_title("Improvement in R-Square after Advanced Text Processing", fontsize = 13)
 ax[1].set_ylabel("R-Square Value")
 plt.tight_layout()
plt.show()
preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]
preds_end += preds[1]<load_from_csv>
cvectorizer = CountVectorizer(max_features=20000,stop_words='english',lowercase=True)
 cvz = cvectorizer.fit_transform(combined_frac['item_description'])
 lda_model = LatentDirichletAllocation(n_components=10,random_state=12345)
 X_topics = lda_model.fit_transform(cvz)
 n_top_words = 10
topic_summaries = []
topic_word = lda_model.components_
vocab = cvectorizer.get_feature_names()
for i, topic_dist in enumerate(topic_word):
topic_words = np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words+1):-1]
topic_summaries.append(' '.join(topic_words))
print('Topic {}: {}'.format(i, ' | '.join(topic_words)) )
submission = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv' )<categorify>
predictions = ridge_reg.predict(X_test)
 submission["price"] = np.expm1(predictions)
 submission.head()
<save_to_csv><EOS> 
submission.to_csv("submission.csv", index = False )