Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<categorify>
%%bash
apt-get install -y xarchiver || true
SAVED_MODEL_PATH="/kaggle/input/tse2020-roberta-pytorch-multi-tpu-10-skfd-d"
NEW_MODEL_PATH="/kaggle/working"
for model_file in $(ls $SAVED_MODEL_PATH/*.pth)
 do
just_filename=$(basename "${model_file%.*}")
 if [[ ! -e "$NEW_MODEL_PATH/$just_filename.pth" ]]; then
echo "Copying $model_file to $NEW_MODEL_PATH"
cp $model_file /tmp
cd /tmp/
mkdir -p $just_filename
echo 3 > $just_filename/version
echo ""
zip -u $just_filename.pth $just_filename/version
echo "Moving model file $just_filename.pth to $NEW_MODEL_PATH"
mv $just_filename.pth $NEW_MODEL_PATH/$just_filename.pth
fi
echo "(After)Contents of '$just_filename/version' in the model file '$model_file'"
unzip -p $NEW_MODEL_PATH/$(basename $model_file)$just_filename/version
done
Bsmt = ['BsmtFinType1', 'BsmtFinType2', 'BsmtCond', 'BsmtExposure',]
for B in Bsmt:
fulldf[B].fillna('None', inplace = True )<data_type_conversions>
warnings.filterwarnings("ignore")
k = ['LotFrontage','MasVnrType', 'MasVnrArea', 'Fence', 'MiscFeature']
for K in k:
fulldf.loc[:,K].fillna(fulldf[K].mode() [0], inplace = True )<sort_values>
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 16
EPOCHS = 5
ROBERTA_PATH = "/kaggle/input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 SAVED_MODEL_PATH="/kaggle/working/"
NUM_OF_SAVED_MODELS = 10
cpu_count = os.cpu_count()
fulldf.loc[:,['Alley', 'Street']].sort_values(by = 'Alley', na_position = 'first' ).head(3 )<drop_column>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 seed = 42
seed_everything(seed )
fulldf.drop(['Id', 'Alley'], axis = 1, inplace = True )<categorify>
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader
ordinal_fea = ['BsmtCond', 'BsmtQual', 'ExterCond', 'ExterQual', 'FireplaceQu', 'GarageCond', 'GarageQual', 'HeatingQC','KitchenQual', 'PoolQC']
for i in ordinal_fea:
fulldf[i] = fulldf[i].replace(['Ex','Fa','Gd','TA','Po','None'],[5, 4, 3, 2, 1, 0] )<feature_engineering>
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 f'{ROBERTA_PATH}/config.json', output_hidden_states=True)
 config.output_hidden_states = True
self.roberta = RobertaModel.from_pretrained(
 f'{ROBERTA_PATH}/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
num_fea = fulldf.select_dtypes(exclude = ['object'] ).columns
skewed_fea = fulldf[num_fea].apply(lambda x: skew(x))
skewed_col_names = skewed_fea.index
skewed_col_names = skewed_col_names.str.contains('(Area)|(SF)|(Frontage)')
 for col_name in skewed_fea[skewed_col_names].index:
fulldf[col_name] = np.log1p(fulldf[col_name] )<feature_engineering>
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )
fulldf['HouseAge'] = fulldf['YearRemodAdd'] - fulldf['YearBuilt']
fulldf['CompleteHouseArea'] = fulldf['LotArea'] + fulldf['MasVnrArea'] + fulldf['GrLivArea'] + fulldf['GarageArea'] + fulldf['PoolArea'] + fulldf['TotalBsmtSF'] 
<categorify>
%env JOBLIB_TEMP_FOLDER=/tmp
%env JOBLIB_START_METHOD="forkserver"
%env TMPDIR=/tmp
num_cat_fea = list(((( set(num_fea)) - set(ordinal_fea)) - set(skewed_fea[skewed_col_names].index)))
 Le = LabelEncoder()
for col_name in num_cat_fea:
fulldf[col_name] = Le.fit_transform(fulldf[col_name] )<categorify>
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
cat_fea = list(set(unique_feature_data.columns[1:])- set(num_fea))
print('The shapes of train and test Dataframes before conversion is respectively:({}),({})'.format(traindf.shape, testdf.shape))
fulldf = pd.get_dummies(fulldf, columns = cat_fea, drop_first = True)
 traindf, testdf = sepdf(fulldf)
 print('The shapes of train and test Dataframes After conversion is respectively:({}),({})'.format(traindf.shape, testdf.shape))<import_modules>
%%time
models = {}
for index in range(NUM_OF_SAVED_MODELS):
model_filename=f"{SAVED_MODEL_PATH}/roberta_fold{index}.pth"
print(f"Loading model {index} from {model_filename}")
 models[index] = TweetModel()
models[index].to(device)
 models[index].load_state_dict(torch.load(model_filename))
models[index].eval()
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.pipeline import make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone<compute_train_metric>
%%time
test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
for data in test_loader:
print('Reading test data via the test loader...')
 ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
print(f'Processing model {model}...')
 with torch.no_grad() :
output = models[model](ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 print('Preparing predictions...')
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )
<find_best_params><EOS> 
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 sample.loc[:, 'selected_text'] = predictions
sample[['textID','selected_text']].to_csv("submission.csv", index=False )