Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
tqdm.pandas()
warnings.filterwarnings("ignore", message="F-score is ill-defined and being set to 0.0 due to no predicted samples.")
 %matplotlib inline
np.random.seed(123)
 maxSentenceLength = 90
numSentencesPerAuthor = 6
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0009
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H1 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H1[author] = prior[0]*charCondProbModel_H1[author]
condP_H1[author] += prior[1]*charCondProbModel_H1['all']
condP_H1[author] += prior[2]*uniformPriorValue
condP_H1['all'] =(prior[0]+prior[1])*charCondProbModel_H1['all']
condP_H1['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
generatedSentence = firstChar
for j in range(maxSentenceLength-1):
encodedHistChar = charEncoder.transform([generatedSentence[-1]])[0]
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedHistChar,:])][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )<create_dataframe>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print('Train data dimension: ', train_df.shape)
 display(train_df.head())
 print('Test data dimension: ', test_df.shape)
 display(test_df.head() )
historyLength = 2
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )<categorify>
enable_local_test = True
if enable_local_test:
n_test = len(test_df)
 train_df, local_test_df =(train_df.iloc[:-n_test].reset_index(drop=True),
train_df.iloc[-n_test:].reset_index(drop=True))
else:
local_test_df = pd.DataFrame([[None, None, 0], [None, None, 0]], columns=['qid', 'question_text', 'target'])
 n_test = 2
historyLength = 2
charCondProbModel_H2 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H2[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H2 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H2[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H2[author][encodedHistory[0],encodedHistory[1],:] = encodedNextCharProb
charCondCountModel_H2[author][encodedHistory[0],encodedHistory[1],:] = encodedNextCharCounts
condCount = charCondCountModel_H2[author]
print('%s Sparsity level = %.1f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H2['all'] =(charCondProbModel_H2['EAP'] + charCondProbModel_H2['HPL'] + charCondProbModel_H2['MWS'])/3.0
charCondCountModel_H2['all'] = charCondCountModel_H2['EAP'] + charCondCountModel_H2['HPL'] + charCondCountModel_H2['MWS']
condCount = charCondCountModel_H2['all']
print('average Sparsity level = %.1f%s' %(100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))<categorify>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0001
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/len(charEncoder.classes_)
 condP_H2 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H2[author] = prior[0]*charCondProbModel_H2[author]
condP_H2[author] += prior[1]*charCondProbModel_H2['all']
condP_H2[author] += prior[2]*uniformPriorValue
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = 0.0; logP_HPL = 0.0; logP_MWS = 0.0
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H2['EAP'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 logP_HPL += np.log(condP_H2['HPL'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 logP_MWS += np.log(condP_H2['MWS'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )<categorify>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
np.random.seed(1000)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0009
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H2 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H2[author] = prior[0]*charCondProbModel_H2[author]
condP_H2[author] += prior[1]*charCondProbModel_H2['all']
condP_H2[author] += prior[2]*uniformPriorValue
condP_H2['all'] =(prior[0]+prior[1])*charCondProbModel_H2['all']
condP_H2['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
encodedFirstChar = charEncoder.transform([firstChar])[0]
secondChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedFirstChar,:])][0]
generatedSentence = firstChar + secondChar
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-2:]])
 currCondProb = condP_H2[author][encodedHistChars[0],encodedHistChars[1],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )<create_dataframe>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
historyLength = 3
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )<categorify>
embed_size = 300
max_features = 95000
maxlen = 70
historyLength = 3
charCondProbModel_H3 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H3[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H3 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H3[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H3[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],:] = encodedNextCharProb
charCondCountModel_H3[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],:] = encodedNextCharCounts
condCount = charCondCountModel_H3[author]
print('%s Sparsity level = %.1f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H3['all'] =(charCondProbModel_H3['EAP'] + charCondProbModel_H3['HPL'] + charCondProbModel_H3['MWS'])/3.0
charCondCountModel_H3['all'] = charCondCountModel_H3['EAP'] + charCondCountModel_H3['HPL'] + charCondCountModel_H3['MWS']
condCount = charCondCountModel_H3['all']
print('average Sparsity level = %.1f%s' %(100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))<categorify>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
uniformPriorFraction = 0.05
allAuthorsPriorFraction = 0.05
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H3 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H3[author] = prior[0]*charCondProbModel_H3[author]
condP_H3[author] += prior[1]*charCondProbModel_H3['all']
condP_H3[author] += prior[2]*uniformPriorValue
condP_H3['all'] =(prior[0]+prior[1])*charCondProbModel_H3['all']
condP_H3['all'] += prior[2]*uniformPriorValue
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = 0.0; logP_HPL = 0.0; logP_MWS = 0.0
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H3['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 logP_HPL += np.log(condP_H3['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 logP_MWS += np.log(condP_H3['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )<categorify>
for df in [train_df, test_df, local_test_df]:
df["question_text"] = df["question_text"].str.lower()
df["question_text"] = df["question_text"].apply(lambda x: clean_text(x))
df["question_text"].fillna("_
x_train = train_df["question_text"].values
x_test = test_df["question_text"].values
x_test_local = local_test_df["question_text"].values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test_local))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_test_local = tokenizer.texts_to_sequences(x_test_local)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 x_test_local = pad_sequences(x_test_local, maxlen=maxlen)
 y_train = train_df['target'].values
y_test = local_test_df['target'].values
np.random.seed(123)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
uniformPriorFraction = 0.05
allAuthorsPriorFraction = 0.05
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H3 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H3[author] = prior[0]*charCondProbModel_H3[author]
condP_H3[author] += prior[1]*charCondProbModel_H3['all']
condP_H3[author] += prior[2]*uniformPriorValue
condP_H3['all'] =(prior[0]+prior[1])*charCondProbModel_H3['all']
condP_H3['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
encodedFirstChar = charEncoder.transform([firstChar])[0]
secondChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedFirstChar,:])][0]
encodedSecondChar = charEncoder.transform([secondChar])[0]
thirdChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H2[author][encodedFirstChar,encodedSecondChar,:])][0]
generatedSentence = firstChar + secondChar + thirdChar
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-historyLength:]])
 currCondProb = condP_H3[author][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )<create_dataframe>
seed_everything()
glove_embeddings = load_glove(tokenizer.word_index, max_features)
 paragram_embeddings = load_para(tokenizer.word_index, max_features)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 np.shape(embedding_matrix )
historyLength = 4
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(15 )<categorify>
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10 ).split(x_train, y_train))
historyLength = 4
charCondProbModel_H4 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H4[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H4 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H4[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H4[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],encodedHistory[3],:] = encodedNextCharProb
charCondCountModel_H4[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],encodedHistory[3],:] = encodedNextCharCounts
condCount = charCondCountModel_H4[author]
print('%s Sparsity level = %.2f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H4['all'] =(charCondProbModel_H4['EAP'] + charCondProbModel_H4['HPL'] + charCondProbModel_H4['MWS'])/3.0
charCondCountModel_H4['all'] = charCondCountModel_H4['EAP'] + charCondCountModel_H4['HPL'] + charCondCountModel_H4['MWS']
condCount = charCondCountModel_H4['all']
print('average Sparsity level = %.2f%s' %(100*(( condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float)) ,'%'))<define_variables>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
condP_H4_PriorWeight_specific = 70
condP_H4_PriorWeight_all = 30
condP_H3_PriorWeight_specific = 70
condP_H3_PriorWeight_all = 30
uniformPriorWeight = 10
logP_EAP_prior = np.log(( trainLabel == 'EAP' ).mean())
 logP_HPL_prior = np.log(( trainLabel == 'HPL' ).mean())
 logP_MWS_prior = np.log(( trainLabel == 'MWS' ).mean())
 numChars = len(charEncoder.classes_)
 prior = np.array([condP_H4_PriorWeight_specific, condP_H4_PriorWeight_all,
condP_H3_PriorWeight_specific, condP_H3_PriorWeight_all, uniformPriorWeight])
 prior = prior.astype(float)/ prior.sum()
uniformPriorValue = 1.0/numChars
condP_H4 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H4[author] = prior[0]*charCondProbModel_H4[author]
condP_H4[author] += prior[1]*charCondProbModel_H4['all']
condP_H4_from_CondP_H3_specific = np.tile(charCondProbModel_H3[author][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4_from_CondP_H3_all = np.tile(charCondProbModel_H3['all'][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4[author] += prior[2]*condP_H4_from_CondP_H3_specific
condP_H4[author] += prior[3]*condP_H4_from_CondP_H3_all
condP_H4[author] += prior[4]*uniformPriorValue
condP_H4['all'] =(condP_H4['EAP'] + condP_H4['HPL'] + condP_H4['MWS'])/ 3.0
authorPredictionList = []
logProbGivenAuthor = np.zeros(( len(validLabel),3))
for i,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = logP_EAP_prior; logP_HPL = logP_HPL_prior; logP_MWS = logP_MWS_prior;
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H4['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_HPL += np.log(condP_H4['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_MWS += np.log(condP_H4['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logProbGivenAuthor[i,:] = [logP_EAP,logP_HPL,logP_MWS]
authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )<categorify>
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(480, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
minimalLogP = -15.0
uniformPriorWeight = 0.09
authorLogProb_norm = logProbGivenAuthor - np.tile(logProbGivenAuthor.max(axis=1)[:,np.newaxis], [1,3])
 authorLogProb_norm[authorLogProb_norm < minimalLogP] = minimalLogP
authorProb = np.exp(authorLogProb_norm)
 authorProb_norm = authorProb / np.tile(authorProb.sum(axis=1)[:,np.newaxis],[1, 3])
 y_Hat = uniformPriorWeight*(1/3.0)+(1.0-uniformPriorWeight)*authorProb_norm
labelEncoder = preprocessing.LabelEncoder()
y_GT = labelEncoder.fit_transform(validLabel)
 print(34*'-')
 print('Validation Set Log Loss = %.5f' %(log_loss(y_GT, y_Hat)))
 print(34*'-' )<categorify>
batch_size = 512
n_epochs = 5
np.random.seed(1000)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
condP_H4_PriorWeight_specific = 10
condP_H4_PriorWeight_all = 10
condP_H3_PriorWeight_specific = 1
condP_H3_PriorWeight_all = 1
uniformPriorWeight = 1
numChars = len(charEncoder.classes_)
 prior = np.array([condP_H4_PriorWeight_specific, condP_H4_PriorWeight_all,
condP_H3_PriorWeight_specific, condP_H3_PriorWeight_all, uniformPriorWeight])
 prior = prior.astype(float)/ prior.sum()
uniformPriorValue = 1.0/numChars
condP_H4 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H4[author] = prior[0]*charCondProbModel_H4[author]
condP_H4[author] += prior[1]*charCondProbModel_H4['all']
condP_H4_from_CondP_H3_specific = np.tile(charCondProbModel_H3[author][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4_from_CondP_H3_all = np.tile(charCondProbModel_H3['all'][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4[author] += prior[2]*condP_H4_from_CondP_H3_specific
condP_H4[author] += prior[3]*condP_H4_from_CondP_H3_all
condP_H4[author] += prior[4]*uniformPriorValue
condP_H4['all'] =(condP_H4['EAP'] + condP_H4['HPL'] + condP_H4['MWS'])/ 3.0
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
generatedSentence = 'disp'
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-historyLength:]])
 currCondProb = condP_H4[author][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )<load_from_csv>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, factor=0.6, min_lr=1e-4, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, torch.optim.Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
self.last_loss = np.inf
self.min_lr = min_lr
self.factor = factor
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def step(self, loss):
if loss > self.last_loss:
self.base_lrs = [max(lr * self.factor, self.min_lr)for lr in self.base_lrs]
self.max_lrs = [max(lr * self.factor, self.min_lr)for lr in self.max_lrs]
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
testData = pd.read_csv('.. /input/test.csv')
 testText = testData.loc[:,'text'].reset_index(drop=True)
 logProbGivenAuthor = np.zeros(( len(testText),3))
for i, sentence in enumerate(testText):
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = logP_EAP_prior; logP_HPL = logP_HPL_prior; logP_MWS = logP_MWS_prior;
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H4['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_HPL += np.log(condP_H4['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_MWS += np.log(condP_H4['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logProbGivenAuthor[i,:] = [logP_EAP,logP_HPL,logP_MWS]<save_to_csv>
def train_model(model, x_train, y_train, x_val, y_val, validate=True):
optimizer = torch.optim.Adam(model.parameters())
 step_size = 300
scheduler = CyclicLR(optimizer, base_lr=0.001, max_lr=0.003,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train, y_train)
 valid = torch.utils.data.TensorDataset(x_val, y_val)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean' ).cuda()
best_score = -np.inf
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( x_val_fold.size(0)))
 if validate:
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
search_result = threshold_search(y_val.cpu().numpy() , valid_preds)
 val_f1, val_threshold = search_result['f1'], search_result['threshold']
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} best_t={:.2f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))
else:
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
print('Validation loss: ', avg_val_loss)
 test_preds = np.zeros(( len(test_loader.dataset)))
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds_local = np.zeros(( len(test_local_loader.dataset)))
 for i,(x_batch,)in enumerate(test_local_loader):
y_pred = model(x_batch ).detach()
test_preds_local[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
return valid_preds, test_preds, test_preds_local
minimalLogP = -15.0
uniformPriorWeight = 0.09
authorLogProb_norm = logProbGivenAuthor - np.tile(logProbGivenAuthor.max(axis=1)[:,np.newaxis], [1,3])
 authorLogProb_norm[authorLogProb_norm < minimalLogP] = minimalLogP
authorProb = np.exp(authorLogProb_norm)
 authorProb_norm = authorProb / np.tile(authorProb.sum(axis=1)[:,np.newaxis],[1, 3])
 y_Hat = uniformPriorWeight*(1/3.0)+(1.0-uniformPriorWeight)*authorProb_norm
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,['EAP', 'HPL', 'MWS']] = y_Hat
submission.to_csv("Markov_char_given_4charHistory.csv", index=False)
 submission.head(10 )<train_on_grid>
seed = 6017
warnings.filterwarnings("ignore")
 ngramLength = 5
featureExtractionStartTime = time.time()
print('-'*52)
 print('fitting "CountVectorizer() " for bag of char %d-grams' %(ngramLength))
BagOfCharsExtractor = CountVectorizer(min_df=8, max_features=250000,
analyzer='char', ngram_range=(1,ngramLength),
binary=False,lowercase=True)
 BagOfCharsExtractor.fit(pd.concat(( trainText,validText,testText)))
 X_train_char = BagOfCharsExtractor.transform(trainText)
 X_valid_char = BagOfCharsExtractor.transform(validText)
 X_test_char = BagOfCharsExtractor.transform(testText)
 featureExtractionDurationInMinutes =(time.time() -featureExtractionStartTime)/60.0
print("feature extraction took %.2f minutes" %(featureExtractionDurationInMinutes))
print('number of "bag of char %d-gram" features = %d' %(ngramLength, X_train_char.shape[1]))
print('-'*52)
 stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train_char,X_valid_char,X_test_char))))
X_train_norm = stdScaler.transform(X_train_char)
 X_valid_norm = stdScaler.transform(X_valid_char)
 X_test_norm = stdScaler.transform(X_test_char)
 yLabelEncoder = preprocessing.LabelEncoder()
yLabelEncoder.fit(pd.concat(( trainLabel,validLabel)))
 y_train = yLabelEncoder.transform(trainLabel)
 y_valid = yLabelEncoder.transform(validLabel)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor_char = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor_char.fit(X_train_norm, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor_char.predict(X_train_norm))
validAccuracy = accuracy_score(y_valid, logisticRegressor_char.predict(X_valid_norm))
trainLogLoss = log_loss(y_train, logisticRegressor_char.predict_proba(X_train_norm))
validLogLoss = log_loss(y_valid, logisticRegressor_char.predict_proba(X_valid_norm))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42 )<save_to_csv>
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 x_test_local_cuda = torch.tensor(x_test_local, dtype=torch.long ).cuda()
test_local = torch.utils.data.TensorDataset(x_test_local_cuda)
 test_local_loader = torch.utils.data.DataLoader(test_local, batch_size=batch_size, shuffle=False )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor_char.predict_proba(X_test_norm)
 submission.to_csv("LogisticRegression_Over_BagOfCharNGrams.csv", index=False)
 submission.head(10 )<feature_engineering>
train_preds = np.zeros(len(train_df))
test_preds = np.zeros(( len(test_df), len(splits)))
 test_preds_local = np.zeros(( n_test, len(splits)))
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 seed_everything(seed + i)
 model = NeuralNet()
model.cuda()
valid_preds_fold, test_preds_fold, test_preds_local_fold = train_model(model,
x_train_fold,
y_train_fold,
x_val_fold,
y_val_fold, validate=True)
 train_preds[valid_idx] = valid_preds_fold
test_preds[:, i] = test_preds_fold
test_preds_local[:, i] = test_preds_local_fold
ngramLength = 2
featureExtractionStartTime = time.time()
print('-'*52)
 print('fitting "CountVectorizer() " for bag of word %d-grams' %(ngramLength))
BagOfWordsExtractor = CountVectorizer(min_df=5, max_features=250000,
analyzer='word', ngram_range=(1,ngramLength),
binary=False,lowercase=True)
 BagOfWordsExtractor.fit(pd.concat(( trainText,validText,testText)))
 X_train_word = BagOfWordsExtractor.transform(trainText)
 X_valid_word = BagOfWordsExtractor.transform(validText)
 X_test_word = BagOfWordsExtractor.transform(testText)
 featureExtractionDurationInMinutes =(time.time() -featureExtractionStartTime)/60.0
print("feature extraction took %.2f minutes" %(featureExtractionDurationInMinutes))
print('number of "bag of word %d-gram" features = %d' %(ngramLength, X_train_word.shape[1]))
print('-'*52)
 stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train_word,X_valid_word,X_test_word))))
X_train_norm = stdScaler.transform(X_train_word)
 X_valid_norm = stdScaler.transform(X_valid_word)
 X_test_norm = stdScaler.transform(X_test_word)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor_word = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor_word.fit(X_train_norm, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor_word.predict(X_train_norm))
validAccuracy = accuracy_score(y_valid, logisticRegressor_word.predict(X_valid_norm))
trainLogLoss = log_loss(y_train, logisticRegressor_word.predict_proba(X_train_norm))
validLogLoss = log_loss(y_valid, logisticRegressor_word.predict_proba(X_valid_norm))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42 )<save_to_csv>
search_result = threshold_search(y_train, train_preds)
 search_result
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor_word.predict_proba(X_test_norm)
 submission.to_csv("LogisticRegression_Over_BagOfWordNGrams.csv", index=False)
 submission.head(10 )<normalization>
f1_score(y_test, test_preds_local.mean(axis=1)> search_result['threshold'] )
X_train = scipy.sparse.hstack(( X_train_word,X_train_char))
X_valid = scipy.sparse.hstack(( X_valid_word,X_valid_char))
X_test = scipy.sparse.hstack(( X_test_word,X_test_char))
stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train,X_valid,X_test))))
X_train = stdScaler.transform(X_train)
 X_valid = stdScaler.transform(X_valid)
 X_test = stdScaler.transform(X_test)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor.fit(X_train, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor.predict(X_train))
trainLogLoss = log_loss(y_train, logisticRegressor.predict_proba(X_train))
validAccuracy = accuracy_score(y_valid, logisticRegressor.predict(X_valid))
validLogLoss = log_loss(y_valid, logisticRegressor.predict_proba(X_valid))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor.predict_proba(X_test)
 submission.to_csv("LogisticRegression_Over_BagOfWord_BagOfChar.csv", index=False)
 submission.head(10 )<set_options>
submission = test_df[['qid']].copy()
submission['prediction'] = test_preds.mean(axis=1)> search_result['threshold']
submission.to_csv('submission.csv', index=False )
np.random.seed(7 )<load_from_csv>
test_scores = [0.6894145809793863, 0.6904706309470233, 0.6905915253597362, 0.6908101789878276, 0.6910334464526553, 0.6916507797390641, 0.6903868185698696, 0.6908830283890897]
train_scores = [0.669555770620476, 0.6708382008438574, 0.6700974173065081, 0.6701065866112219, 0.6704778141088164, 0.6708436318389969, 0.6705310002773053, 0.6710429366071224]
seeds = [42853, 73399, 21152, 58237, 25688, 6017, 29547, 65803]
df = pd.read_csv('./.. /input/train.csv')
 a2c = {'EAP': 0, 'HPL' : 1, 'MWS' : 2}
y = np.array([a2c[a] for a in df.author])
 y = to_categorical(y )<categorify>
eval_df = pd.DataFrame()
eval_df['cv_score'] = train_scores
eval_df['local_test_score'] = test_scores
eval_df['seed'] = seeds
eval_df.head()
<categorify><EOS> 
eval_df.loc[[eval_df['local_test_score'].idxmax() ]]