Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<string_transform>
torch.manual_seed(0)
 np.random.seed(0 )
if IS_POSTPROCESSING:
def post_processing(text):
if len(text.split())> 0 and 'http' in text.split() [-1]:
return ' '.join(text.split() [:-1])
 else:
return text<string_transform>
class OttoDataset(Dataset):
def __init__(self, data, target, idx, mode):
self.data = data
self.target = target
self.idx = np.asarray(idx)
 self.mode = mode
def __len__(self):
return self.idx.shape[0]
def __getitem__(self, index):
index = self.idx[index]
row = self.data[index]
if self.mode == 'test':
return torch.tensor(row)
 else:
label = self.target[index]
return torch.tensor(row), torch.tensor(label )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<string_transform>
train = pd.read_csv('.. /input/otto-group-product-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/otto-group-product-classification-challenge/test.csv')
 sample_submit = pd.read_csv('.. /input/otto-group-product-classification-challenge/sampleSubmission.csv' )
if IS_SUBSTITUTING_URLS:
def preprocess_urls(replacements, full_text, sub_text, k):
text_split = full_text.split()
urls_done = 0
for word in text_split:
urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', word)
 if len(urls)> 0:
replacements.append(( k, urls[0], 'website' + str(urls_done)))
 full_text = full_text.replace(urls[0], 'website' + str(urls_done), 1)
 sub_text = sub_text.replace(urls[0], 'website' + str(urls_done), 1)
 urls_done += 1
return replacements, full_text, sub_text
def deprocess_urls(replacements, full_text, sub_text, k):
replaces = [x for x in replacements if x[0] == k]
for replacer in replaces:
full_text = full_text.replace(replacer[2], replacer[1], 1)
 sub_text = sub_text.replace(replacer[2], replacer[1], 1)
 return full_text, sub_text
replacements, test, sub_test = preprocess_urls(replacements, test, sub_test, k)
 test, sub_test = deprocess_urls(replacements, test, sub_test, k )<feature_engineering>
train['target'] = train['target'].str.replace('Class_', '')
 train['target'] = train['target'].astype(int)- 1
if IS_SUBSTITUTING_URLS:
train_web_replacements = []
for k in range(train.shape[0]):
text = train.loc[k, 'text']
sub_text = train.loc[k, 'selected_text']
train_web_replacements, train.loc[k, 'text'], train.loc[k, 'selected_text'] = preprocess_urls(train_web_replacements, text, sub_text, k)
 if IS_USING_PSEUDO_LABELS:
pseudo_web_replacements = []
for fold in range(5):
pseudo_replacements = []
for k in range(train.shape[0]):
text = pseudo_labelled_folds[fold].loc[k, 'text']
sub_text = pseudo_labelled_folds[fold].loc[k, 'selected_text']
pseudo_replacements, pseudo_labelled_folds[fold].loc[k, 'text'], pseudo_labelled_folds[fold].loc[k, 'selected_text'] = preprocess_urls(pseudo_replacements, text, sub_text, k)
 pseudo_web_replacements.append(pseudo_replacements)
 test_web_replacements = []
for k in range(test.shape[0]):
text = test.loc[k, 'text']
sub_text = ''
test_web_replacements, test.loc[k, 'text'], throwaway = preprocess_urls(train_web_replacements, text, sub_text, k)
 del(throwaway )<statistical_test>
excluded_column = ['target', 'id']
cols = [c for c in train.columns if c not in(excluded_column + [])]
data = pd.concat([train, test] ).reset_index()
for c in cols:
data[c] = np.log(1+data[c])
 data_scale = StandardScaler().fit_transform(data[cols])
 data = pd.concat([data[['id', 'target']], pd.DataFrame(data_scale)], axis=1)
 train = data[~data['target'].isnull() ].reset_index(drop=True)
 test = data[data['target'].isnull() ].reset_index(drop=True )
if IS_ADDING_SENTIMENT_WORDS:
polarity_threshold = 0.3
pos_exist_token = 8000
npos_exist_token = 8001
neg_exist_token = 9000
nneg_exist_token = 9001
sid = SentimentIntensityAnalyzer()
def get_high_polarity_words(data):
high_polarity_words = []
for index, row in data.iterrows() :
row_polarising = []
for word in row['text'].split() :
if(sid.polarity_scores(word)['compound'] >= polarity_threshold)or(sid.polarity_scores(word)['compound'] <= -polarity_threshold):
row_polarising += tokenizer.encode(word ).ids
high_polarity_words.append(row_polarising)
 return np.array(high_polarity_words)
 def get_high_polarity_tokens(data):
high_polarity_tokens = []
for index, row in data.iterrows() :
pos_words = [word for word in row['text'].split() if sid.polarity_scores(word)['compound'] >= polarity_threshold]
neg_words = [word for word in row['text'].split() if sid.polarity_scores(word)['compound'] <= -polarity_threshold]
high_polarity_tokens.append([pos_exist_token if len(pos_words)> 0 else npos_exist_token,
neg_exist_token if len(neg_words)> 0 else nneg_exist_token])
 return np.array(high_polarity_tokens)
 train_sentiment_words = get_high_polarity_tokens(train)
 test_sentiment_words = get_high_polarity_tokens(test )<define_variables>
excluded_column = ['target', 'id']
cols = [c for c in train.columns if c not in excluded_column]
test_dataset = OttoDataset(test[cols].values, np.zeros(test.shape[0]), list(range(test.shape[0])) , 'test')
 test_loader = torch.utils.data.DataLoader(
 dataset=test_dataset,
batch_size=64,
shuffle=False,
num_workers=2 )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
if IS_ADDING_SENTIMENT_WORDS:
input_ids[k,:len(enc.ids)+len(train_sentiment_words[k])+3] = [0, s_tok] + list(train_sentiment_words[k])+ enc.ids + [2]
attention_mask[k,:len(enc.ids)+len(train_sentiment_words[k])+3] = 1
else:
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<define_variables>
class MLPNet(nn.Module):
def __init__(self):
super(MLPNet, self ).__init__()
self.fc1 = nn.Linear(93, 512)
 self.fc2 = nn.Linear(512, 512)
 self.fc3 = nn.Linear(512, 512)
 self.fc4 = nn.Linear(512, 9)
 self.dropout1 = nn.Dropout2d(0.2)
 self.dropout2 = nn.Dropout2d(0.2)
 self.dropout3 = nn.Dropout2d(0.2)
 def forward(self, x):
x = F.relu(self.fc1(x))
x = self.dropout1(x)
 x = F.relu(self.fc2(x))
x = self.dropout2(x)
 x = F.relu(self.fc3(x))
x = self.dropout3(x)
 x = self.fc4(x)
 return x
if IS_USING_PSEUDO_LABELS:
pseudo_labelled_folds_tokens = []
for fold in range(5):
pseudo_labels = pseudo_labelled_folds[fold]
ct = pseudo_labels.shape[0]
input_ids_ps = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_ps = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_ps = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens_ps = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens_ps = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(pseudo_labels.shape[0]):
text1 = " "+" ".join(pseudo_labels.loc[k,'text'].split())
 text2 = " ".join(pseudo_labels.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[pseudo_labels.loc[k,'sentiment']]
input_ids_ps[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_ps[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens_ps[k,toks[0]+2] = 1
end_tokens_ps[k,toks[-1]+2] = 1
pseudo_labelled_folds_tokens.append(( input_ids_ps, attention_mask_ps, token_type_ids_ps, start_tokens_ps, end_tokens_ps))<categorify>
device = 'cuda' if torch.cuda.is_available() else 'cpu'
num_epochs = 50
NFOLDS = 5
RANDOM_STATE = 871972
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE )
ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
if IS_ADDING_SENTIMENT_WORDS:
input_ids_t[k,:len(enc.ids)+len(test_sentiment_words[k])+3] = [0, s_tok] + list(test_sentiment_words[k])+ enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+len(test_sentiment_words[k])+3] = 1
else:
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<create_dataframe>
y_preds = []
oof = np.zeros(( len(train), 9))
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y = train['target'])) :
X_train, X_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = X_train['target'].astype(int), X_valid['target'].astype(int)
 train_dataset = OttoDataset(X_train[cols].values, y_train.values, list(range(X_train.shape[0])) , 'train')
 valid_dataset = OttoDataset(X_valid[cols].values, y_valid.values, list(range(X_valid.shape[0])) , 'train')
 train_loader = torch.utils.data.DataLoader(
 dataset=train_dataset,
batch_size=64,
shuffle=True,
num_workers=2)
 valid_loader = torch.utils.data.DataLoader(
 dataset=valid_dataset,
batch_size=64,
shuffle=False,
num_workers=2)
 net = MLPNet().to(device)
 criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters() , lr=0.01, momentum=0.9, weight_decay=5e-4)
 for epoch in range(num_epochs):
train_loss, train_acc, val_loss, val_acc = 0, 0, 0, 0
net.train()
for i,(datas, labels)in enumerate(train_loader):
datas, labels = datas.to(device), labels.to(device)
 optimizer.zero_grad()
feature = net(datas.float())
 outputs = feature
loss = criterion(outputs, labels)
 train_loss += loss.item()
acc =(outputs.max(1)[1] == labels ).sum()
train_acc += acc.item()
loss.backward()
optimizer.step()
avg_train_loss = train_loss / len(train_loader.dataset)
 avg_train_acc = train_acc / len(train_loader.dataset)
 net.eval()
with torch.no_grad() :
for datas, labels in valid_loader:
datas, labels = datas.to(device), labels.to(device)
 feature = net(datas.float())
 outputs = feature
loss = criterion(outputs, labels)
 val_loss += loss.item()
acc =(outputs.max(1)[1] == labels ).sum()
val_acc += acc.item()
avg_val_loss = val_loss / len(valid_loader.dataset)
 avg_val_acc = val_acc / len(valid_loader.dataset)
 print('Epoch [{}/{}], Loss: {loss:.4f}, val_loss: {val_loss:.4f}, val_acc: {val_acc:.4f}' 
.format(epoch+1, num_epochs, i+1, loss=avg_train_loss, val_loss=avg_val_loss, val_acc=avg_val_acc))
net.eval()
valid_predict = []
with torch.no_grad() :
for datas, labels in valid_loader:
datas, labels = datas.to(device), labels.to(device)
 feature = net(datas.float())
 outputs = F.softmax(feature, dim=1)
 valid_predict.append(outputs)
 oof[valid_index] = torch.cat(valid_predict, dim=0)
 test_predict = []
with torch.no_grad() :
for datas in test_loader:
datas = datas.to(device)
 feature = net(datas.float())
 outputs = F.softmax(feature, dim=1)
 test_predict.append(outputs)
 y_preds.append(torch.cat(test_predict, dim=0))
if IS_EXPECTATION_MAXIMISE:
jem_preds = np.zeros(( input_ids_t.shape[0], N_SPLIT * 3))
jem_preds = pd.DataFrame(jem_preds, columns = ['start0', 'end0', 'string0', 'start1', 'end1', 'string1', 'start2', 'end2', 'string2', 'start3', 'end3', 'string3', 'start4', 'end4', 'string4'] )<define_variables>
column_name = ['nn_' + str(i)for i in range(9)]
pd.DataFrame(oof, columns = column_name ).to_csv('oof_nn.csv', index=False)
 y_pred_nn = np.zeros(( len(y_preds[0]), 9))
for i in range(len(y_preds)) :
y_pred_nn += y_preds[i].cpu().numpy() / len(y_preds)
 pd.DataFrame(y_pred_nn, columns = column_name ).to_csv('submit_nn.csv', index=False )
<compute_test_metric><EOS> 
submit = pd.concat([sample_submit[['id']], pd.DataFrame(y_pred_nn)], axis = 1)
 submit.columns = sample_submit.columns
submit.to_csv('submit.csv', index=False )