Kaggle competition: Tweet Sentiment Extraction
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape )<split>
py.init_notebook_mode(connected=True)
 nltk.download('stopwords')
 stop=set(stopwords.words('english'))
X = final_features.iloc[:len(y), :]
X_pred = final_features.iloc[len(X):, :]
X_train = X
X_test = X_pred
y_train = y
print('X', X.shape, 'y', y.shape, 'X_pred', X_pred.shape )<compute_train_metric>
def read_train() :
train=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )<choose_model_class>
def remove_stopwords(text):
if text is not None:
tokens = [x for x in word_tokenize(text)if x not in stop]
return " ".join(tokens)
 else:
return None
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
exclude = set(string.punctuation)
 s = ''.join(ch for ch in text if ch not in exclude)
 return s
def clean_df(df, train=True):
df["dirty_text"] = df['text']
df["text"] = df['text'].apply(lambda x : x.lower())
 df['text']=df['text'].apply(lambda x: remove_emoji(x))
df['text']=df['text'].apply(lambda x : remove_URL(x))
df['text']=df['text'].apply(lambda x : remove_html(x))
df['text'] =df['text'].apply(lambda x : remove_stopwords(x))
df['text']=df['text'].apply(lambda x : remove_punct(x))
df.text = df.text.replace('\s+', ' ', regex=True)
 if train:
df["selected_text"] = df['selected_text'].apply(lambda x : x.lower())
 df['selected_text']=df['selected_text'].apply(lambda x: remove_emoji(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_URL(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_html(x))
df['selected_text'] =df['selected_text'].apply(lambda x : remove_stopwords(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_punct(x))
df.selected_text = df.selected_text.replace('\s+', ' ', regex=True)
 return df
model_ridge = Ridge(alpha = 10 )<choose_model_class>
def analyze_neutral_lenght(train_df):
neutral_df = train_df[train_df["sentiment"] == "neutral"]
equal_selected_with_text = neutral_df[neutral_df['selected_text'] == neutral_df["text"]]
print("Total number of neutral: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(neutral_df)))
 count = train_df.text.str.count(' ')<= 2
less_than_T = train_df.loc[count]
equal_selected_with_text = less_than_T[less_than_T['selected_text'] == less_than_T["text"]]
print("Total number of tweets with less than 3 words per text: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(less_than_T)))
 def post_process(submission_df, test_df):
index_to_selected_text = {}
for i, row in test_df.iterrows() :
_id = row[0]
text = row[1]
sentiment = row[2]
if len(text.split(" ")) <= 3 or sentiment == "neutral":
index_to_selected_text[_id] = text
submission_rows = submission_df.to_dict("records")
 new_rows = []
for row in submission_rows:
_id = row['textID']
if _id in index_to_selected_text:
new_row = deepcopy(row)
 new_row['selected_text'] = index_to_selected_text[_id]
else:
new_row = row
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 train_df = read_train()
test_df = read_test()
submission_df = read_submission()
analyze_neutral_lenght(train_df)
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0004))<compute_train_metric>
def analyze_cleaning(df, cleaning_lambda, field="selected_text"):
def diff_strings(row):
count = {}
A = row[0]
B = row[1]
if A is None:
A = "TEMP"
if B is None:
B = "TEMP"
for word in A.split() :
count[word] = count.get(word, 0)+ 1
for word in B.split() :
count[word] = count.get(word, 0)+ 1
diff = [word for word in count if count[word] == 1]
_max_len = max(len(A), len(B))
return len(diff)/_max_len
to_analyze = df[[field]]
to_analyze['cleaned_selected_text'] = to_analyze[field]
to_analyze["diff_ratio"] = None
to_analyze['cleaned_selected_text']=to_analyze['cleaned_selected_text'].apply(lambda x: cleaning_lambda(x))
to_analyze["diff_ratio"] = to_analyze.apply(lambda x: diff_strings(x), axis=1)
 print("Cleaning function used: {}".format(cleaning_lambda.__name__))
print("Average difference ratio: {}".format(to_analyze["diff_ratio"].mean()))
print("")
 train_df = read_train()
test_df = read_test()
cleaning_functions = [remove_stopwords, remove_URL, remove_html, remove_emoji, remove_punct]
for func in cleaning_functions:
analyze_cleaning(train_df, func, field="selected_text" )
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()<prepare_output>
results = pd.DataFrame({
'Model': ['Ridge','Lasso'],
'Score': [cv_ridge,cv_lasso]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head()<predict_on_test>
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test)) 
<compute_test_metric>
tqdm.pandas()
BERT_PATH = '/kaggle/input/bert-base-uncased-huggingface-transformer/'
def find_offset(x,y):

 x_str = ' '.join(x)
 y_str = ' '.join(y)
 idx0=0
for ind in(i for i, e in enumerate(x_str)if e == y_str[0]):
if(x_str[ind: ind+len(y_str)] == y_str)or(x_str[ind: ind+len(y_str.replace('
idx0 = ind
idx1 = ind + len(y_str)- 1
break
t = 0
for offset,i in enumerate(x):
if t +len(i)+1>idx0:
break
t = t+len(i)+1
return offset
def create_targets(df, tokenizer):
df['t_text'] = df['text'].apply(lambda x: tokenizer.tokenize(str(x)))
 df['t_selected_text'] = df['selected_text'].apply(lambda x: tokenizer.tokenize(str(x)))
 def func(row):
x,y = row['t_text'],row['t_selected_text'][:]
_offset = 0
for offset in range(len(x)) :
_offset = offset
d = dict(zip(x[offset:],y))
check = [k==v for k,v in d.items() ]
if all(check)== True:
break
targets = [0]*_offset + [1]*len(y)+ [0]*(len(x)-_offset-len(y))
if len(targets)!= len(x):
offset = find_offset(x,y)
 targets = [0]*offset + [1]*len(y)+ [0] *(len(x)-offset-len(y))
return targets
df['targets'] = df.apply(func,axis=1)
 return df
def _convert_to_transformer_inputs(text, tokenizer, max_sequence_length):
def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids, input_masks, input_segments = return_id(text, None, 'longest_first', max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arrays(df, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df.iterrows()):
t = str(instance.text)
 ids, masks, segments= _convert_to_transformer_inputs(t,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns].values.tolist())
 def create_model() :
id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 attn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = True
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 _,_, hidden_states = bert_model(id, attention_mask=mask, token_type_ids=attn)
 h12 = tf.reshape(hidden_states[-1][:,0],(-1,1,768))
h11 = tf.reshape(hidden_states[-2][:,0],(-1,1,768))
h10 = tf.reshape(hidden_states[-3][:,0],(-1,1,768))
h09 = tf.reshape(hidden_states[-4][:,0],(-1,1,768))
concat_hidden = tf.keras.layers.Concatenate(axis=2 )([h12, h11, h10, h09])
 x = tf.keras.layers.GlobalAveragePooling1D()(concat_hidden)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(MAX_TARGET_LEN, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[id, mask, attn], outputs=x)
 return model
def convert_pred_to_text(df,pred):
temp_output = []
for idx,p in enumerate(pred):
indexes = np.where(p>0.5)
 current_text = df['t_text'][idx]
if len(indexes[0])>0:
start = indexes[0][0]
end = indexes[0][-1]
else:
start = 0
end = len(current_text)
 if end >= len(current_text):
end = len(current_text)
 temp_output.append(' '.join(current_text[start:end+1]))
return temp_output
def correct_op(row):
placeholder = row['temp_output']
for original_token in str(row['text'] ).split() :
token_str = ' '.join(tokenizer.tokenize(original_token))
placeholder = placeholder.replace(token_str,original_token,1)
 return placeholder
def replacer(row):
if row['sentiment'] == 'neutral':
return row['text']
else:
return row['temp_output2']
Stack =(lasso_pred + ridge_pred)/ 2<save_to_csv>
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_distil = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model = QuestionAnsweringModel('distilbert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150,
"use_cuda":True
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_distil['selected_text'] = predictions_df['answer']
submission_df_distil = post_process(submission_df_distil, test_df)
submission = pd.DataFrame({'Id':test_ID, 'SalePrice':lasso_pred})
 submission.to_csv('Stack.csv', index=False )<load_from_csv>
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_albert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/pretrained-albert-pytorch/albert-base-v2/'
model = QuestionAnsweringModel('albert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_albert['selected_text'] = predictions_df['answer']
submission_df_albert = post_process(submission_df_albert, test_df)
train= pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test= pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 submission= pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv' )<feature_engineering>
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_bert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/bert-base-uncased/'
model = QuestionAnsweringModel('bert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_bert['selected_text'] = predictions_df['answer']
submission_df_bert = post_process(submission_df_bert, test_df)
<create_dataframe><EOS> 
sentiment_analyzer = SentimentIntensityAnalyzer()
def merge_predictions(submission_distil, sumbission_albert, submission_bert):
def merge(row):
pred_1 = set(str(row[1] ).split())
 pred_2 = set(str(row[2] ).split())
 pred_3 = set(str(row[3] ).split())
 res = []
res = list(pred_1.intersection(pred_2, pred_3))
if len(res)== 0:
res = list(pred_1.union(pred_2, pred_3))
return " ".join(res)
 def merge_compound(row):
pred_1 = str(row[1])
 pred_2 = str(row[2])
 pred_3 = str(row[3])
 preds = [pred_1, pred_2, pred_3]
comp_1 = abs(sentiment_analyzer.polarity_scores(pred_1 ).get("compound",0))
comp_2 = abs(sentiment_analyzer.polarity_scores(pred_2 ).get("compound",0))
comp_3 = abs(sentiment_analyzer.polarity_scores(pred_3 ).get("compound",0))
comps = [comp_1, comp_2, comp_3]
best_comp = comps.index(max(comps))
return preds[best_comp]
def capitalize_selected(df):
rows = df.to_dict("records")
 new_rows = []
for r in rows:
text = r["text"]
lower_text = text.lower()
pred_distil = r["selected_text_distil"]
pred_albert = r["selected_text_albert"]
pred_bert = r["selected_text_bert"]
new_pred_distil = text[lower_text.find(pred_distil):len(pred_distil)]
new_pred_albert = text[lower_text.find(pred_albert):len(pred_albert)]
new_pred_bert = text[lower_text.find(pred_bert):len(pred_bert)]
new_row = {
"textID": r['textID'],
"selected_text_distil": new_pred_distil,
"selected_text_albert": new_pred_albert,
"selected_text_bert": new_pred_bert
}
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 sumbission_albert = sumbission_albert.rename({'selected_text': 'selected_text_albert'}, axis=1)
 del sumbission_albert["textID"]
submission_distil = submission_distil.rename({'selected_text': 'selected_text_distil'}, axis=1)
 submission_bert = submission_bert.rename({'selected_text': 'selected_text_bert'}, axis=1)
 del submission_bert["textID"]
final_sub = pd.concat([submission_distil, sumbission_albert, submission_bert], axis=1)
 final_sub["selected_text"] = final_sub.apply(lambda x: merge_compound(x), axis=1)
 del final_sub["selected_text_albert"]
del final_sub["selected_text_distil"]
del final_sub["selected_text_bert"]
return final_sub

 submission = merge_predictions(submission_df_distil, submission_df_albert, submission_df_bert)
 submission.to_csv('submission.csv', index=False)