Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<define_variables>
import numpy as np
import pandas as pd
import json
desc = {
'SalePrice': "The property's sale price in dollars.This is the target variable to be predicted",
'MSSubClass': "The building class",
'MSZoning': "The general zoning classification",
'LotFrontage': "Linear feet of street connected to property",
'LotArea': "Lot size in square feet",
'Street': "Type of road access",
'Alley': "Type of alley access",
'LotShape': "General shape of property",
'LandContour': "Flatness of the property",
'Utilities': "Type of utilities available",
'LotConfig': "Lot configuration",
'LandSlope': "Slope of property",
'Neighborhood': "Physical locations within Ames city limits",
'Condition1': "Proximity to main road or railroad",
'Condition2': "Proximity to main road or railroad(if a second is present)",
'BldgType': "Type of dwelling",
'HouseStyle': "Style of dwelling",
'OverallQual': "Overall material and finish quality",
'OverallCond': "Overall condition rating",
'YearBuilt': "Original construction date",
'YearRemodAdd': "Remodel date",
'RoofStyle': "Type of roof",
'RoofMatl': "Roof material",
'Exterior1st': "Exterior covering on house",
'Exterior2nd': "Exterior covering on house(if more than one material)",
'MasVnrType': "Masonry veneer type",
'MasVnrArea': "Masonry veneer area in square feet",
'ExterQual': "Exterior material quality",
'ExterCond': "Present condition of the material on the exterior",
'Foundation': "Type of foundation",
'BsmtQual': "Height of the basement",
'BsmtCond': "General condition of the basement",
'BsmtExposure': "Walkout or garden level basement walls",
'BsmtFinType1': "Quality of basement finished area",
'BsmtFinSF1': "Type 1 finished square feet",
'BsmtFinType2': "Quality of second finished area(if present)",
'BsmtFinSF2': "Type 2 finished square feet",
'BsmtUnfSF': "Unfinished square feet of basement area",
'TotalBsmtSF': "Total square feet of basement area",
'Heating': "Type of heating",
'HeatingQC': "Heating quality and condition",
'CentralAir': "Central air conditioning",
'Electrical':" Electrical system",
'1stFlrSF': "First Floor square feet",
'2ndFlrSF': "Second floor square feet",
'LowQualFinSF': "Low quality finished square feet(all floors)",
'GrLivArea': "Above grade(ground)living area square feet",
'BsmtFullBath': "Basement full bathrooms",
'BsmtHalfBath': "Basement half bathrooms",
'FullBath': "Full bathrooms above grade",
'HalfBath': "Half baths above grade",
'Bedroom': "Number of bedrooms above basement level",
'Kitchen': "Number of kitchens",
'KitchenQual': "Kitchen quality",
'TotRmsAbvGrd': "Total rooms above grade(does not include bathrooms)",
'Functional': "Home functionality rating",
'Fireplaces': "Number of fireplaces",
'FireplaceQu': "Fireplace quality",
'GarageType': "Garage location",
'GarageYrBlt': "Year garage was built",
'GarageFinish': "Interior finish of the garage",
'GarageCars': "Size of garage in car capacity",
'GarageArea': "Size of garage in square feet",
'GarageQual': "Garage quality",
'GarageCond': "Garage condition",
'PavedDrive': "Paved driveway",
'WoodDeckSF': "Wood deck area in square feet",
'OpenPorchSF': "Open porch area in square feet",
'EnclosedPorch': "Enclosed porch area in square feet",
'3SsnPorch': "Three season porch area in square feet",
'ScreenPorch': "Screen porch area in square feet",
'PoolArea': "Pool area in square feet",
'PoolQC': "Pool quality",
'Fence': "Fence quality",
'MiscFeature': "Miscellaneous feature not covered in other categories",
'MiscVal': "Value of miscellaneous feature",
'MoSold': "Month Sold",
'YrSold': "Year Sold",
'SaleType': "Type of sale",
'SaleCondition': "Condition of sale"
}<load_from_disk>
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
json.dump(desc, open("desc.json",'w'))
with open("desc.json", "r")as read_file:
d= json.load(read_file )<load_from_csv>
train = np.array(pd_train)
 test = np.array(pd_test )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 numerical = [x for x in train.columns if train.dtypes[x] != 'object']
numerical.remove('SalePrice')
 categorical = [x for x in train.columns if train.dtypes[x] == 'object']
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 data_overview(features )<prepare_x_and_y>
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in train:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/train.json', 'w')as outfile:
json.dump(output, outfile )
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True )<sort_values>
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in test:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/test.json', 'w')as outfile:
json.dump(output, outfile )
corr_dict = train.corr() ['SalePrice'][(train.corr() ['SalePrice'] > 0.5)&(train.corr() ['SalePrice'] < 1.0)].sort_values(ascending=False ).to_dict()
print("Higher correlations between predictor variables and the response variable ")
 print("-------------------------------------------------------------------------")
 for x in corr_dict.keys() :
print("(" , round(corr_dict[x],3), ")",x, ":" , " -",desc[x] )<data_type_conversions>
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type roberta \
--model_name_or_path roberta-large \
--do_lower_case \
--do_train \
--do_eval \
--data_dir./data \
--cache_dir /kaggle/input/cached-roberta-large-pretrained/cache \
--train_file train.json \
--predict_file test.json \
--learning_rate 5e-5 \
--num_train_epochs 2 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir results_roberta_large \
--per_gpu_eval_batch_size=16 \
--per_gpu_train_batch_size=16 \
--save_steps=100000
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )<data_type_conversions>
predictions = json.load(open('results_roberta_large/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if pd_test['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = pd_test['text'][i]
else:
submission.loc[i, 'selected_text'] = predictions[id_]
<data_type_conversions><EOS> 
submission.to_csv('submission.csv', index=False )