Kaggle competition: AS-bow-2019-2020
for row in range(shops.shape[0]):
for col in range(shops.shape[0]):
mat[row, col] = textdistance.levenshtein.normalized_similarity(shops["shop_name"][row], shops["shop_name"][col])
 mat = np.where(mat == 1 , 0, mat )<count_duplicates>
class SentimentDataset(Dataset):

 def __init__(self,datalines):
self.xydata = datalines
def __len__(self):
return len(self.xydata)
 def __getitem__(self,idx):
return self.xydata[idx]
def load_data_set(filename):

 istream = open(filename)
 istream.readline()
xydataset = [ ]
for line in istream:
fields = line.split(',')
 label = fields[0]
text = ','.join(fields[1:])
 xydataset.append(( text,label))
istream.close()
return SentimentDataset(xydataset)
 train_set = load_data_set('.. /input/sentimentIMDB_train.csv')
 print('Loaded %d examples as train set.'%(len(train_set)))
 print('Train data Sample(1st batch only)')
 train_loader = DataLoader(train_set, batch_size=4, shuffle=True)
 for text_batch, label_batch in train_loader:
for text,label in zip(text_batch, label_batch):
print(' ',text[:50],'...',label)
 break
test_set = load_data_set('.. /input/sentimentIMDB_test.csv')
 print('Loaded %d examples as test set.'%(len(test_set)) )
shop_name_duplicated = pd.DataFrame(np.transpose(np.nonzero(np.where(mat > 0.7, mat, 0))))
shop_name_duplicated = shop_name_duplicated[~shop_name_duplicated.apply(frozenset, axis=1 ).duplicated() ].reset_index(drop=True)
 print(shop_name_duplicated )<count_duplicates>
def make_w2idx(dataset):

 wordset = set([])
 for text,label in dataset:
words = text.split()
wordset.update(words)
 return dict(zip(wordset,range(len(wordset))))
def vectorize_text(text,w2idx):
counts = Counter(text.split())
 xvec = torch.zeros(len(w2idx))
for word in counts:
if word in w2idx:
xvec[w2idx[word]] = counts[word]
return xvec.squeeze()
def vectorize_target(ylabel):
return torch.tensor(float(ylabel))
for index, row in shop_name_duplicated.iterrows() :
print("shop_id:", row[0], "shop_name:", shops.loc[shops["shop_id"] == row[0], "shop_name"].values[0])
 print("shop_id:", row[1], "shop_name:", shops.loc[shops["shop_id"] == row[1], "shop_name"].values[0])
 print()<count_duplicates>
class SentimentAnalyzer(nn.Module):
def __init__(self):
super(SentimentAnalyzer, self ).__init__()
self.reset_structure(1,1)
 def reset_structure(self,vocab_size, num_labels):
self.W = nn.Linear(vocab_size, num_labels)
 def forward(self, text_vec):
return torch.sigmoid(self.W(text_vec))
def train(self,train_set,learning_rate,epochs):
self.w2idx = make_w2idx(train_set)
 self.reset_structure(len(self.w2idx),1)
 loss_func = nn.BCELoss()
optimizer = optim.SGD(self.parameters() , lr=learning_rate)
 train_dataset, dev_dataset = random_split(train_set, [20000, 5000])
 data_loader = DataLoader(train_dataset, batch_size=len(train_set), shuffle=True)
 max_acc = 0
for epoch in range(epochs):
global_logloss = 0.0
for Xbatch,Ybatch in data_loader:
for X, Y in zip(Xbatch,Ybatch):
self.zero_grad()
xvec = vectorize_text(X,self.w2idx)
 yvec = vectorize_target(Y)
 prob = self(xvec ).squeeze()
loss = loss_func(prob,yvec)
 loss.backward()
optimizer.step()
global_logloss += loss.item()
validation_acc = self.eval_test(dev_dataset)
 print("Epoch %d, mean cross entropy = %f, Validation accurracy : %f"%(epoch,global_logloss/len(train_set),validation_acc))
if validation_acc >= max_acc:
torch.save(self.state_dict() , 'sentiment_model.wt')
 max_acc = validation_acc
self.load_state_dict(torch.load('sentiment_model.wt'))
def eval_test(self,dev_set):
with torch.no_grad() :
data_loader = DataLoader(dev_set, batch_size=len(dev_set), shuffle=False)
 ncorrect = 0
N = 0
for Xbatch,Ybatch in data_loader:
for X,Y in zip(Xbatch,Ybatch):
xvec = vectorize_text(X,self.w2idx)
 prob = self(xvec ).squeeze()
if int(prob > 0.5)== int(Y):
ncorrect += 1
N += 1
return float(ncorrect)/float(N)
 def run_test(self,test_set,pred_filename):
with torch.no_grad() :
data_loader = DataLoader(test_set, batch_size=len(train_set), shuffle=False)
 idxList = []
sentList = []
for Xbatch,idxbatch in data_loader:
for X,idx in zip(Xbatch,idxbatch):
xvec = vectorize_text(X,self.w2idx)
 prob = self(xvec ).squeeze()
idxList.append(idx)
 sentList.append(int(prob > 0.5))
df = pa.DataFrame({'idx':idxList,'sentY':sentList})
 df.to_csv(pred_filename,index=False)
 print('done.' )
<feature_engineering><EOS> 
sent = SentimentAnalyzer()
sent.train(train_set,0.01,50 )