Kaggle competition: Tweet Sentiment Extraction
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )<groupby>
train_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 question_p="identify the positive sentence? "
question_n="identify the negative sentence?"
question_neu="identify the neutral sentence? "
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
<groupby>
def make_question(x):
if x=="positive":
return question_p
elif x=="negative":
return question_n
else:
return question_neu
train_df["sentiment"]=train_df["sentiment"].apply(make_question)
 train_df["sentiment"]
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).agg(['mean', 'count', 'sum'])
<data_type_conversions>
train_df=train_df.dropna()
title_and_sex = original_train.copy() [['Name', 'Sex']]
title_and_sex['Title'] = title_and_sex['Name'].apply(get_title)
 title_and_sex['Sex'] = title_and_sex['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_and_sex[['Title', 'Sex']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
<compute_train_metric>
def answer_index(x):
return x["text"].find(str(x["selected_text"]))
def answer_end(x):
return x["text"].find(str(x["selected_text"])) +len(str(x["selected_text"])) -1
train_df["answer_index"]=train_df.apply(answer_index,axis=1)
 train_df["answer_end"]=train_df.apply(answer_end,axis=1 )
def get_gini_impurity(survived_count, total_count):
survival_prob = survived_count/total_count
not_survival_prob =(1 - survival_prob)
 random_observation_survived_prob = survival_prob
random_observation_not_survived_prob =(1 - random_observation_survived_prob)
 mislabelling_survided_prob = not_survival_prob * random_observation_survived_prob
mislabelling_not_survided_prob = survival_prob * random_observation_not_survived_prob
gini_impurity = mislabelling_survided_prob + mislabelling_not_survided_prob
return gini_impurity<choose_model_class>
train_data=[]
def make_train_data(df):
for ind in df.index:
main_d=dict()
context=df.loc[ind,"text"].lower()
main_d["context"]=context
qas=[]
inside_qas={}
inside_qas["id"]=df.loc[ind,"textID"]
inside_qas["question"]=df.loc[ind,"sentiment"]
inside_answer=[{"text":df.loc[ind,"selected_text"].lower() ,"answer_start": int(df.loc[ind,"answer_index"]),"answer_end":int(df.loc[ind,"answer_end"])}]
inside_qas["answers"]=inside_answer
inside_qas["is_impossible"]=False
qas.append(inside_qas)
 main_d["qas"]=qas
train_data.append(main_d)
 return train_data
gini_impurity_starting_node = get_gini_impurity(342, 891)
 gini_impurity_starting_node<statistical_test>
train_data=make_train_data(train_df )
gini_impurity_men = get_gini_impurity(109, 577)
 gini_impurity_men<statistical_test>
with open('/kaggle/working/train.json', 'w')as outfile:
json.dump(train_data, outfile)
gini_impurity_women = get_gini_impurity(233, 314)
 gini_impurity_women<define_variables>
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model=QuestionAnsweringModel("distilbert",MODEL_PATH,
args={'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 2,
'max_seq_length': 128,
'doc_stride': 64,
'fp16': False,
},
use_cuda=True)
men_weight = 577/891
women_weight = 314/891
weighted_gini_impurity_sex_split =(gini_impurity_men * men_weight)+(gini_impurity_women * women_weight)
 sex_gini_decrease = weighted_gini_impurity_sex_split - gini_impurity_starting_node
sex_gini_decrease<compute_train_metric>
model.train_model('/kaggle/working/train.json' )
gini_impurity_title_1 = get_gini_impurity(81, 517)
 gini_impurity_title_1<statistical_test>
test_data=[]
def make_test_data(df):
for ind in df.index:
main_d=dict()
context=df.loc[ind,"text"].lower()
main_d["context"]=context
qas=[]
inside_qas={}
inside_qas["id"]=df.loc[ind,"textID"]
inside_qas["question"]=df.loc[ind,"sentiment"]
inside_qas["is_impossible"]=False
qas.append(inside_qas)
 main_d["qas"]=qas
test_data.append(main_d)
 return test_data
gini_impurity_title_others = get_gini_impurity(261, 374)
 gini_impurity_title_others<define_variables>
test_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 test_df["sentiment"]=test_df["sentiment"].apply(make_question)
 test_data=make_test_data(test_df)
 test_data_json=json.dumps(test_data)
 with open('/kaggle/working/test.json', 'w')as outfile:
json.dump(test_data, outfile )
title_1_weight = 517/891
title_others_weight = 374/891
weighted_gini_impurity_title_split =(gini_impurity_title_1 * title_1_weight)+(gini_impurity_title_others * title_others_weight)
 title_gini_decrease = weighted_gini_impurity_title_split - gini_impurity_starting_node
title_gini_decrease<choose_model_class>
predictions=model.predict(test_data )
<save_to_csv><EOS> 
predictions_df = pd.DataFrame.from_dict(predictions)
 sub_df=pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv")
 sub_df['selected_text'] = predictions_df['answer']
sub_df.to_csv('submission.csv', index=False )