Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<save_to_csv>
import numpy as np
import pandas as pd
import json
final=(0.2*final1+0.6*final2+0.2*final3)
 final_submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 final_submission.to_csv("final_submission.csv", index=False)
 final_submission.head()<load_from_csv>
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
X_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 X_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 X_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 X_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 X_train.rename(columns={'Province_State':'State'}, inplace=True)
 X_test.rename(columns={'Province_State':'State'}, inplace=True)
 X_train.Date = pd.to_datetime(X_train.Date, infer_datetime_format=True)
 X_test.Date = pd.to_datetime(X_test.Date, infer_datetime_format=True)
 EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
train = np.array(pd_train)
 test = np.array(pd_test )
X_xTrain = X_train.copy()
X_xTrain.State.fillna(EMPTY_VAL, inplace=True)
 X_xTrain.State = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain.Date = X_xTrain.Date.astype(int)
 X_xTest = X_test.copy()
X_xTest.State.fillna(EMPTY_VAL, inplace=True)
 X_xTest.State = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest.Date = X_xTest.Date.astype(int)
<categorify>
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
le = preprocessing.LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain.State = le.fit_transform(X_xTrain.State)
 X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest.State = le.fit_transform(X_xTest.State )<categorify>
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in train:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/train.json', 'w')as outfile:
json.dump(output, outfile )
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_xTrain.Country.unique()<prepare_x_and_y>
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in test:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/test.json', 'w')as outfile:
json.dump(output, outfile )
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS.State = le.fit_transform(X_xTrain_CS.State)
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS.State = le.fit_transform(X_xTest_CS.State)
 xmodel1 = DecisionTreeRegressor()
xmodel1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = xmodel1.predict(X_xTest_CS)
 xmodel2 = DecisionTreeRegressor()
xmodel2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = xmodel2.predict(X_xTest_CS)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0 )<save_to_csv>
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type roberta \
--model_name_or_path roberta-large \
--do_lower_case \
--do_train \
--do_eval \
--data_dir./data \
--cache_dir /kaggle/input/cached-roberta-large-pretrained/cache \
--train_file train.json \
--predict_file test.json \
--learning_rate 3e-5 \
--num_train_epochs 3 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir results_roberta_large \
--per_gpu_eval_batch_size=16 \
--per_gpu_train_batch_size=16 \
--save_steps=100000
xout.ForecastId = xout.ForecastId.astype('int')
 xout.tail()
xout.to_csv('submission.csv', index=False )<set_options>
def post_process(selected):
return " ".join(set(selected.lower().split()))
%matplotlib inline
plt.style.use('fivethirtyeight')
<load_from_csv>
predictions = json.load(open('results_roberta_large/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if pd_test['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = pd_test['text'][i]
else:
submission.loc[i, 'selected_text'] = post_process(predictions[id_] )
df_train = pd.read_csv(".. /input/titanic/train.csv", header=0)
 df_train.head()<load_from_csv>
submission.to_csv('submission.csv', index=False )