Kaggle competition: Quora Insincere Questions Classification
matplotlib.style.use('fivethirtyeight' )<load_from_csv>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )
data = pd.read_csv('.. /input/train.csv')
 stratifiedCV = model_selection.StratifiedShuffleSplit(n_splits=1, test_size=0.10, random_state=1)
 trainInds, validInds = next(stratifiedCV.split(data['text'], data['author']))
trainText = data.loc[trainInds,'text'].reset_index(drop=True)
 validText = data.loc[validInds,'text'].reset_index(drop=True)
 trainLabel = data.loc[trainInds,'author'].reset_index(drop=True)
 validLabel = data.loc[validInds,'author'].reset_index(drop=True )<categorify>
print('Preproccesing texts.... ')
 print('lower...')
 df["question_text"] = df["question_text"].apply(lambda x: x.lower())
 df_final["question_text"] = df_final["question_text"].apply(lambda x: x.lower())
 contraction_mapping = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have" }
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
print('contractions...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_contractions(x,contraction_mapping))
punct_mapping = {"‘": "'",
"₹": "e",
"´": "'",
"°": "",
"€": "e",
"™": "tm",
"√": " sqrt ",
"×": "x",
"²": "2",
"—": "-",
"–": "-",
"’": "'",
"_": "-",
"`": "'",
'“': '"',
'”': '"',
'“': '"',
"£": "e",
'∞': 'infinity',
'θ': 'theta',
'÷': '/',
'α': 'alpha',
'•': '.',
'à': 'a',
'−': '-',
'β': 'beta',
'∅': '',
'³': '3',
'π': 'pi',
}
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
print('clean special chars...')
 df["question_text"] = df["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_final["question_text"] = df_final["question_text"].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
mispell_dict = {'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization'}
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
print('clean misspellings...')
 df["question_text"] = df["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))
df_final["question_text"] = df_final["question_text"].apply(lambda x: correct_spelling(x,mispell_dict))
toEnglishDict = {}
srcStr = ['à','â','ä','å','æ','ç','è','é','ê','ë','ï','î','ñ','ô','ö','õ','ü','û','α','δ','ν','ο','π','ς','υ','ἶ']
dstStr = ['a','a','a','a','a','c','e','e','e','e','i','i','n','o','o','o','u','u','a','d','n','o','p','s','y','i']
for src,dst in zip(srcStr,dstStr):
toEnglishDict[src] = dst
def myunidecode(inString):
outString = ''
for ch in inString:
if ch in toEnglishDict.keys() :
outString += toEnglishDict[ch]
else:
outString += ch
return outString<categorify>
dim = 300
num_words = 95000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(list(df['question_text'])+list(df_final['question_text']))
print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape)
 x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )
charsDict = {}
for key in ['all','EAP','HPL','MWS']:
charsDict[key] = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 chars = [char for char in decodedSentence]
charsDict['all'] += chars
charsDict[author] += chars<categorify>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
charEncoder = preprocessing.LabelEncoder()
charEncoder.fit(charsDict['all'])
 charCounts_EAP = np.histogram(charEncoder.transform(charsDict['EAP']),range(len(charEncoder.classes_)+1),density=True)[0]
charCounts_HPL = np.histogram(charEncoder.transform(charsDict['HPL']),range(len(charEncoder.classes_)+1),density=True)[0]
charCounts_MWS = np.histogram(charEncoder.transform(charsDict['MWS']),range(len(charEncoder.classes_)+1),density=True)[0]
sortedChars = np.flipud(np.argsort(charCounts_EAP + charCounts_HPL + charCounts_MWS))<categorify>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_para.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
authorsList = ['EAP','HPL','MWS']
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
chars = [char for char in myunidecode(sentence.lower())]
logP_EAP = sum([np.log(charCounts_EAP[charEncoder.classes_ == ch])for ch in chars])
 logP_HPL = sum([np.log(charCounts_HPL[charEncoder.classes_ == ch])for ch in chars])
 logP_MWS = sum([np.log(charCounts_MWS[charEncoder.classes_ == ch])for ch in chars])
 authorPredictionList.append(authorsList[np.argmax(np.array([logP_EAP,logP_HPL,logP_MWS])) ])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )<define_variables>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss
np.random.seed(1234)
 maxSentenceLength = 95
numSentencesPerAuthor = 5
charProbModel = {}
charProbModel['all'] =(charCounts_EAP + charCounts_HPL + charCounts_MWS)/3.0
charProbModel['EAP'] = charCounts_EAP
charProbModel['HPL'] = charCounts_HPL
charProbModel['MWS'] = charCounts_MWS
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
generatedSentence = ''
for j in range(maxSentenceLength):
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )<create_dataframe>
class CyclicLR(object):

 def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, torch.optim.Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
def chaos_lr(r):
lr_lambda = lambda iters: rel_val(iters)
 def rel_val(iteration):
x = 0.99
for i in range(iteration):
x = x*r*(1.-x)
 return x
return lr_lambda
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
best threshold is % f with score %f'%(best_threshold,best_score))
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 X_train1,X_val,y_train1,y_val = train_test_split(x_train, y_train)
 X_train2,X_test,y_train2,y_test = train_test_split(X_train1, y_train1)
 x_train_tensor = torch.tensor(X_train2, dtype=torch.long ).cuda()
y_train_tensor = torch.tensor(y_train2, dtype=torch.float32 ).cuda()
train_dataset = torch.utils.data.TensorDataset(x_train_tensor,y_train_tensor)
 train_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
 x_val_tensor = torch.tensor(X_val, dtype=torch.long ).cuda()
y_val_tensor = torch.tensor(y_val, dtype=torch.float32 ).cuda()
val_dataset = torch.utils.data.TensorDataset(x_val_tensor,y_val_tensor)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=False)
 class Attention(nn.Module):
def __init__(self,hidden_lstm_size):
super(Attention,self ).__init__()
self.hidden_lstm_size = hidden_lstm_size
self.seq_len = max_len
weights = torch.zeros(self.hidden_lstm_size,1)
 nn.init.xavier_uniform_(weights)
 self.weights = nn.Parameter(weights)
 def forward(self,x):
x_c = x.contiguous().view(-1, self.hidden_lstm_size)
 a = torch.mm(x_c, self.weights)
 a = a.view(-1, self.seq_len)
 a = torch.tanh(a)
 s = F.softmax(a,dim=1)
 s = torch.unsqueeze(s, -1)
 weighted_input =s*x
return torch.sum(weighted_input, 1)
 class Sentiment(nn.Module):
def __init__(self,matrix_glove,matrix_para,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',matrix_glove.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.lin_dim = 64
self.n_layers = 1
self.embedding_glove = nn.Embedding(matrix_glove.shape[0],matrix_glove.shape[1])
 self.embedding_glove.weight = nn.Parameter(torch.tensor(matrix_glove, dtype=torch.float32))
self.embedding_glove.weight.requires_grad = False
self.embedding_para = nn.Embedding(matrix_para.shape[0],matrix_para.shape[1])
 self.embedding_para.weight = nn.Parameter(torch.tensor(matrix_para, dtype=torch.float32))
self.embedding_para.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=matrix_glove.shape[1]*2,
hidden_size=self.hidden_dim,
num_layers=self.n_layers,
bidirectional=True,
batch_first=True,
dropout=0.1)
 self.att_lstm = Attention(self.hidden_dim*2)
 self.linear1 = nn.Linear(2*self.hidden_dim,self.lin_dim)
 self.relu = nn.ReLU()
self.linear2 = nn.Linear(self.lin_dim,1)
 self.dropout = nn.Dropout(0.1)
 def forward(self,x):
hidden =(torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 hidden_gru = torch.zeros(2*self.n_layers, x.shape[0], self.hidden_dim ).cuda()
e_glove = self.embedding_glove(x)
 e_para = self.embedding_para(x)
 e = torch.cat([e_glove,e_para],dim=-1)
 lstm_out, hidden = self.lstm(e, hidden)
 out_lstm = self.att_lstm(lstm_out)
 out = self.linear1(out_lstm)
 out = self.relu(out)
 out = self.dropout(out)
 return self.linear2(out)
 model = Sentiment(embedding_matrix_glov,embedding_matrix_para,batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.RMSprop(model.parameters() ,lr=1e-3)
 scheduler = CyclicLR(optimizer,base_lr=1e-3, max_lr=4e-3,
step_size=300., mode='exp_range',
gamma=0.99994)
 losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
val_list = list(val_loader)
 for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
for batch,(x_batch,y_true)in enumerate(list(iter(train_loader)) ,1):
model.train()
optimizer.zero_grad()
y_pred = model(x_batch ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
scheduler.batch_step()
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,(x_val_batch,y_true)in enumerate(val_list,1):
y_pred = model(x_val_batch ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )
historyLength = 1
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )<categorify>
print(os.listdir())
 model = Sentiment(embedding_matrix_glov,embedding_matrix_para,batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))
<categorify><EOS> 
print('Threshold:',search_result['threshold'])
 print(x_test.shape)
 submission_dataset = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long ).cuda())
 submission_loader = torch.utils.data.DataLoader(dataset=submission_dataset,batch_size=batch_size, shuffle=False)
 pred = []
with torch.no_grad() :
for x in list(submission_loader):
model.eval()
pred += torch.sigmoid(model(x[0] ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )