Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<categorify>
matplotlib.style.use('fivethirtyeight' )
Y_train = to_categorical(target_train, num_classes = 10 )<split>
data = pd.read_csv('.. /input/train.csv')
 stratifiedCV = model_selection.StratifiedShuffleSplit(n_splits=1, test_size=0.10, random_state=1)
 trainInds, validInds = next(stratifiedCV.split(data['text'], data['author']))
trainText = data.loc[trainInds,'text'].reset_index(drop=True)
 validText = data.loc[validInds,'text'].reset_index(drop=True)
 trainLabel = data.loc[trainInds,'author'].reset_index(drop=True)
 validLabel = data.loc[validInds,'author'].reset_index(drop=True )
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=0 )<choose_model_class>
toEnglishDict = {}
srcStr = ['à','â','ä','å','æ','ç','è','é','ê','ë','ï','î','ñ','ô','ö','õ','ü','û','α','δ','ν','ο','π','ς','υ','ἶ']
dstStr = ['a','a','a','a','a','c','e','e','e','e','i','i','n','o','o','o','u','u','a','d','n','o','p','s','y','i']
for src,dst in zip(srcStr,dstStr):
toEnglishDict[src] = dst
def myunidecode(inString):
outString = ''
for ch in inString:
if ch in toEnglishDict.keys() :
outString += toEnglishDict[ch]
else:
outString += ch
return outString
clf_keras = Sequential()<choose_model_class>
charsDict = {}
for key in ['all','EAP','HPL','MWS']:
charsDict[key] = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 chars = [char for char in decodedSentence]
charsDict['all'] += chars
charsDict[author] += chars
clf_keras.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', strides=1, activation ='relu',
input_shape =(28,28,1)))
 clf_keras.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation ='relu'))
clf_keras.add(keras.layers.BatchNormalization())
 clf_keras.add(keras.layers.MaxPooling2D(pool_size=(2,2)) )<choose_model_class>
charEncoder = preprocessing.LabelEncoder()
charEncoder.fit(charsDict['all'])
 charCounts_EAP = np.histogram(charEncoder.transform(charsDict['EAP']),range(len(charEncoder.classes_)+1),density=True)[0]
charCounts_HPL = np.histogram(charEncoder.transform(charsDict['HPL']),range(len(charEncoder.classes_)+1),density=True)[0]
charCounts_MWS = np.histogram(charEncoder.transform(charsDict['MWS']),range(len(charEncoder.classes_)+1),density=True)[0]
sortedChars = np.flipud(np.argsort(charCounts_EAP + charCounts_HPL + charCounts_MWS))
clf_keras.add(Conv2D(filters = 64, kernel_size =(3,3), strides=2, padding = 'same', activation ='relu'))
clf_keras.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'same', activation ='relu'))
clf_keras.add(keras.layers.BatchNormalization())
 clf_keras.add(keras.layers.MaxPooling2D(pool_size=(2,2)) )<choose_model_class>
authorsList = ['EAP','HPL','MWS']
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
chars = [char for char in myunidecode(sentence.lower())]
logP_EAP = sum([np.log(charCounts_EAP[charEncoder.classes_ == ch])for ch in chars])
 logP_HPL = sum([np.log(charCounts_HPL[charEncoder.classes_ == ch])for ch in chars])
 logP_MWS = sum([np.log(charCounts_MWS[charEncoder.classes_ == ch])for ch in chars])
 authorPredictionList.append(authorsList[np.argmax(np.array([logP_EAP,logP_HPL,logP_MWS])) ])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )
clf_keras.add(Dropout(rate = 0.5))<choose_model_class>
np.random.seed(1234)
 maxSentenceLength = 95
numSentencesPerAuthor = 5
charProbModel = {}
charProbModel['all'] =(charCounts_EAP + charCounts_HPL + charCounts_MWS)/3.0
charProbModel['EAP'] = charCounts_EAP
charProbModel['HPL'] = charCounts_HPL
charProbModel['MWS'] = charCounts_MWS
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
generatedSentence = ''
for j in range(maxSentenceLength):
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )
clf_keras.add(Dense(10, activation = "softmax"))<choose_model_class>
historyLength = 1
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )
optimizer = keras.optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004 )<choose_model_class>
charCondProbModel_H1 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H1[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H1 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H1[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([history])[0]
encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H1[author][encodedHistory,:] = encodedNextCharProb
charCondCountModel_H1[author][encodedHistory,:] = encodedNextCharCounts
condCount = charCondCountModel_H1[author]
print('%s Sparsity level = %.1f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H1['all'] =(charCondProbModel_H1['EAP'] + charCondProbModel_H1['HPL'] + charCondProbModel_H1['MWS'])/3.0
charCondCountModel_H1['all'] = charCondCountModel_H1['EAP'] + charCondCountModel_H1['HPL'] + charCondCountModel_H1['MWS']
print('average Sparsity level = %.1f%s' %(100*(charCondCountModel_H1['all'] < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
clf_keras.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0001
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/len(charEncoder.classes_)
 condP_H1 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H1[author] = prior[0]*charCondProbModel_H1[author]
condP_H1[author] += prior[1]*charCondProbModel_H1['all']
condP_H1[author] += prior[2]*uniformPriorValue
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = 0.0; logP_HPL = 0.0; logP_MWS = 0.0
for histChar, nextChar in zip(history,nextChar):
encodedHistChar = charEncoder.transform([histChar])[0]
encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H1['EAP'][encodedHistChar,encodedNextChar])
 logP_HPL += np.log(condP_H1['HPL'][encodedHistChar,encodedNextChar])
 logP_MWS += np.log(condP_H1['MWS'][encodedHistChar,encodedNextChar])
 authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', factor=0.2, patience=3, min_lr=0.00001, verbose=1 )<init_hyperparams>
np.random.seed(123)
 maxSentenceLength = 90
numSentencesPerAuthor = 6
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0009
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H1 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H1[author] = prior[0]*charCondProbModel_H1[author]
condP_H1[author] += prior[1]*charCondProbModel_H1['all']
condP_H1[author] += prior[2]*uniformPriorValue
condP_H1['all'] =(prior[0]+prior[1])*charCondProbModel_H1['all']
condP_H1['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
generatedSentence = firstChar
for j in range(maxSentenceLength-1):
encodedHistChar = charEncoder.transform([generatedSentence[-1]])[0]
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedHistChar,:])][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<define_variables>
historyLength = 2
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )
epochs = 30<define_variables>
historyLength = 2
charCondProbModel_H2 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H2[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H2 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H2[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H2[author][encodedHistory[0],encodedHistory[1],:] = encodedNextCharProb
charCondCountModel_H2[author][encodedHistory[0],encodedHistory[1],:] = encodedNextCharCounts
condCount = charCondCountModel_H2[author]
print('%s Sparsity level = %.1f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H2['all'] =(charCondProbModel_H2['EAP'] + charCondProbModel_H2['HPL'] + charCondProbModel_H2['MWS'])/3.0
charCondCountModel_H2['all'] = charCondCountModel_H2['EAP'] + charCondCountModel_H2['HPL'] + charCondCountModel_H2['MWS']
condCount = charCondCountModel_H2['all']
print('average Sparsity level = %.1f%s' %(100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
batch_size = 86<train_model>
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0001
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/len(charEncoder.classes_)
 condP_H2 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H2[author] = prior[0]*charCondProbModel_H2[author]
condP_H2[author] += prior[1]*charCondProbModel_H2['all']
condP_H2[author] += prior[2]*uniformPriorValue
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = 0.0; logP_HPL = 0.0; logP_MWS = 0.0
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H2['EAP'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 logP_HPL += np.log(condP_H2['HPL'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 logP_MWS += np.log(condP_H2['MWS'][encodedHistChars[0],encodedHistChars[1],encodedNextChar])
 authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )
start_time = time.perf_counter()
history = clf_keras.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size), epochs = epochs,
validation_data =(X_val,Y_val), verbose = 2,
steps_per_epoch=features_train.shape[0] // batch_size,
callbacks=[learning_rate_reduction],
use_multiprocessing=True)
 print(time.perf_counter() - start_time, "seconds" )<predict_on_test>
np.random.seed(1000)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
uniformPriorFraction = 0.0001
allAuthorsPriorFraction = 0.0009
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H2 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H2[author] = prior[0]*charCondProbModel_H2[author]
condP_H2[author] += prior[1]*charCondProbModel_H2['all']
condP_H2[author] += prior[2]*uniformPriorValue
condP_H2['all'] =(prior[0]+prior[1])*charCondProbModel_H2['all']
condP_H2['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
encodedFirstChar = charEncoder.transform([firstChar])[0]
secondChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedFirstChar,:])][0]
generatedSentence = firstChar + secondChar
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-2:]])
 currCondProb = condP_H2[author][encodedHistChars[0],encodedHistChars[1],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )
results = clf_keras.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<concatenate>
historyLength = 3
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(8 )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1 )<save_to_csv>
historyLength = 3
charCondProbModel_H3 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H3[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H3 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H3[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H3[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],:] = encodedNextCharProb
charCondCountModel_H3[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],:] = encodedNextCharCounts
condCount = charCondCountModel_H3[author]
print('%s Sparsity level = %.1f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H3['all'] =(charCondProbModel_H3['EAP'] + charCondProbModel_H3['HPL'] + charCondProbModel_H3['MWS'])/3.0
charCondCountModel_H3['all'] = charCondCountModel_H3['EAP'] + charCondCountModel_H3['HPL'] + charCondCountModel_H3['MWS']
condCount = charCondCountModel_H3['all']
print('average Sparsity level = %.1f%s' %(100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
submission.to_csv("submission_keras.csv",index=False )<import_modules>
uniformPriorFraction = 0.05
allAuthorsPriorFraction = 0.05
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H3 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H3[author] = prior[0]*charCondProbModel_H3[author]
condP_H3[author] += prior[1]*charCondProbModel_H3['all']
condP_H3[author] += prior[2]*uniformPriorValue
condP_H3['all'] =(prior[0]+prior[1])*charCondProbModel_H3['all']
condP_H3['all'] += prior[2]*uniformPriorValue
authorPredictionList = []
for k,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = 0.0; logP_HPL = 0.0; logP_MWS = 0.0
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H3['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 logP_HPL += np.log(condP_H3['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 logP_MWS += np.log(condP_H3['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedNextChar])
 authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )
import numpy
from keras import backend as K
from keras.datasets import mnist
from keras.utils import np_utils<import_modules>
np.random.seed(123)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
uniformPriorFraction = 0.05
allAuthorsPriorFraction = 0.05
prior = np.array([1.0-uniformPriorFraction-allAuthorsPriorFraction, allAuthorsPriorFraction, uniformPriorFraction])
 uniformPriorValue = 1.0/(len(charEncoder.classes_))
condP_H3 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H3[author] = prior[0]*charCondProbModel_H3[author]
condP_H3[author] += prior[1]*charCondProbModel_H3['all']
condP_H3[author] += prior[2]*uniformPriorValue
condP_H3['all'] =(prior[0]+prior[1])*charCondProbModel_H3['all']
condP_H3['all'] += prior[2]*uniformPriorValue
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
firstChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=charProbModel[author])][0]
encodedFirstChar = charEncoder.transform([firstChar])[0]
secondChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H1[author][encodedFirstChar,:])][0]
encodedSecondChar = charEncoder.transform([secondChar])[0]
thirdChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=condP_H2[author][encodedFirstChar,encodedSecondChar,:])][0]
generatedSentence = firstChar + secondChar + thirdChar
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-historyLength:]])
 currCondProb = condP_H3[author][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )
from keras.layers import Dense, Dropout,Flatten
from keras.layers.convolutional import Conv2D, MaxPooling2D<import_modules>
historyLength = 4
historyList = []
nextCharList = []
authorList = []
for k,(sentence, author)in enumerate(zip(trainText,trainLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
sentenceDF = pd.DataFrame(columns=['author','history','next char'])
 historyList += history
nextCharList += nextChar
authorList += [author]*len(history)
 corpusDF = pd.DataFrame(columns=['author','history','next char'])
 corpusDF['author'] = authorList
corpusDF['history'] = historyList
corpusDF['next char'] = nextCharList
corpusDF.head(15 )
from keras.models import Sequential
import pandas as pd<set_options>
historyLength = 4
charCondProbModel_H4 = {}
for author in ['EAP','HPL','MWS']:
charCondProbModel_H4[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 charCondCountModel_H4 = {}
for author in ['EAP','HPL','MWS']:
charCondCountModel_H4[author] = np.zeros(( 1+historyLength)*[charEncoder.classes_.shape[0]])
 corpusGroupedByAuthor = corpusDF.groupby(by='author',axis=0)
 for author in corpusDF['author'].unique() :
authorCorpusDF = corpusGroupedByAuthor.get_group(author ).loc[:,['history','next char']].reset_index(drop=True)
 authorCorpusGroupedByHistory = authorCorpusDF.groupby(by='history',axis=0)
 for history in authorCorpusDF['history'].unique() :
authorHistoryDF = authorCorpusGroupedByHistory.get_group(history ).reset_index(drop=True ).loc[:,'next char'].reset_index(drop=True)
 encodedHistory = charEncoder.transform([ch for ch in history])
 encodedNextCharCounts = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=False)[0]
encodedNextCharProb = np.histogram(charEncoder.transform(authorHistoryDF),range(len(charEncoder.classes_)+1),density=True)[0]
charCondProbModel_H4[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],encodedHistory[3],:] = encodedNextCharProb
charCondCountModel_H4[author][encodedHistory[0],encodedHistory[1],encodedHistory[2],encodedHistory[3],:] = encodedNextCharCounts
condCount = charCondCountModel_H4[author]
print('%s Sparsity level = %.2f%s' %(author, 100*(condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float),'%'))
charCondProbModel_H4['all'] =(charCondProbModel_H4['EAP'] + charCondProbModel_H4['HPL'] + charCondProbModel_H4['MWS'])/3.0
charCondCountModel_H4['all'] = charCondCountModel_H4['EAP'] + charCondCountModel_H4['HPL'] + charCondCountModel_H4['MWS']
condCount = charCondCountModel_H4['all']
print('average Sparsity level = %.2f%s' %(100*(( condCount < 1 ).sum() /(condCount > -1 ).sum().astype(float)) ,'%'))
K.set_image_data_format('channels_last')
 numpy.random.seed(0 )<load_from_csv>
condP_H4_PriorWeight_specific = 70
condP_H4_PriorWeight_all = 30
condP_H3_PriorWeight_specific = 70
condP_H3_PriorWeight_all = 30
uniformPriorWeight = 10
logP_EAP_prior = np.log(( trainLabel == 'EAP' ).mean())
 logP_HPL_prior = np.log(( trainLabel == 'HPL' ).mean())
 logP_MWS_prior = np.log(( trainLabel == 'MWS' ).mean())
 numChars = len(charEncoder.classes_)
 prior = np.array([condP_H4_PriorWeight_specific, condP_H4_PriorWeight_all,
condP_H3_PriorWeight_specific, condP_H3_PriorWeight_all, uniformPriorWeight])
 prior = prior.astype(float)/ prior.sum()
uniformPriorValue = 1.0/numChars
condP_H4 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H4[author] = prior[0]*charCondProbModel_H4[author]
condP_H4[author] += prior[1]*charCondProbModel_H4['all']
condP_H4_from_CondP_H3_specific = np.tile(charCondProbModel_H3[author][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4_from_CondP_H3_all = np.tile(charCondProbModel_H3['all'][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4[author] += prior[2]*condP_H4_from_CondP_H3_specific
condP_H4[author] += prior[3]*condP_H4_from_CondP_H3_all
condP_H4[author] += prior[4]*uniformPriorValue
condP_H4['all'] =(condP_H4['EAP'] + condP_H4['HPL'] + condP_H4['MWS'])/ 3.0
authorPredictionList = []
logProbGivenAuthor = np.zeros(( len(validLabel),3))
for i,(sentence, author)in enumerate(zip(validText,validLabel)) :
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = logP_EAP_prior; logP_HPL = logP_HPL_prior; logP_MWS = logP_MWS_prior;
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H4['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_HPL += np.log(condP_H4['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_MWS += np.log(condP_H4['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logProbGivenAuthor[i,:] = [logP_EAP,logP_HPL,logP_MWS]
authorPredictionList.append(authorsList[np.argmax([logP_EAP,logP_HPL,logP_MWS])])
 print(52*'-')
 print('==> Validation Set Classification Accuracy = %.1f%s' %(100*(validLabel == authorPredictionList ).mean() ,'%'))
print(52*'-' )
X = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 y = X["label"]
X.drop(["label"], inplace = True, axis = 1 )<split>
minimalLogP = -15.0
uniformPriorWeight = 0.09
authorLogProb_norm = logProbGivenAuthor - np.tile(logProbGivenAuthor.max(axis=1)[:,np.newaxis], [1,3])
 authorLogProb_norm[authorLogProb_norm < minimalLogP] = minimalLogP
authorProb = np.exp(authorLogProb_norm)
 authorProb_norm = authorProb / np.tile(authorProb.sum(axis=1)[:,np.newaxis],[1, 3])
 y_Hat = uniformPriorWeight*(1/3.0)+(1.0-uniformPriorWeight)*authorProb_norm
labelEncoder = preprocessing.LabelEncoder()
y_GT = labelEncoder.fit_transform(validLabel)
 print(34*'-')
 print('Validation Set Log Loss = %.5f' %(log_loss(y_GT, y_Hat)))
 print(34*'-' )
X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, test_size=0.2 , random_state=42 )<data_type_conversions>
np.random.seed(1000)
 maxSentenceLength = 95
numSentencesPerAuthor = 9
condP_H4_PriorWeight_specific = 10
condP_H4_PriorWeight_all = 10
condP_H3_PriorWeight_specific = 1
condP_H3_PriorWeight_all = 1
uniformPriorWeight = 1
numChars = len(charEncoder.classes_)
 prior = np.array([condP_H4_PriorWeight_specific, condP_H4_PriorWeight_all,
condP_H3_PriorWeight_specific, condP_H3_PriorWeight_all, uniformPriorWeight])
 prior = prior.astype(float)/ prior.sum()
uniformPriorValue = 1.0/numChars
condP_H4 = {}
authorsList = ['EAP','HPL','MWS']
for author in authorsList:
condP_H4[author] = prior[0]*charCondProbModel_H4[author]
condP_H4[author] += prior[1]*charCondProbModel_H4['all']
condP_H4_from_CondP_H3_specific = np.tile(charCondProbModel_H3[author][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4_from_CondP_H3_all = np.tile(charCondProbModel_H3['all'][np.newaxis,:,:,:],[numChars,1,1,1,1])
 condP_H4[author] += prior[2]*condP_H4_from_CondP_H3_specific
condP_H4[author] += prior[3]*condP_H4_from_CondP_H3_all
condP_H4[author] += prior[4]*uniformPriorValue
condP_H4['all'] =(condP_H4['EAP'] + condP_H4['HPL'] + condP_H4['MWS'])/ 3.0
for author in ['EAP','HPL','MWS','all']:
print(( 6+maxSentenceLength)*'-')
 print('Author %s:' %(author))
print(12*'-')
 for i in range(numSentencesPerAuthor):
generatedSentence = 'disp'
for j in range(maxSentenceLength-1):
encodedHistChars = charEncoder.transform([ch for ch in generatedSentence[-historyLength:]])
 currCondProb = condP_H4[author][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],:]
currCondProb = currCondProb/currCondProb.sum()
newChar = charEncoder.classes_[np.random.choice(range(len(charCounts_EAP)) ,size=1,p=currCondProb)][0]
generatedSentence += newChar
if(newChar == '.')or(j == maxSentenceLength):
break
print('%d: "%s"' %(i+1,generatedSentence))
print(( 4+maxSentenceLength)*'-' )
X_train = X_train.reshape(X_train.shape[0], 28, 28 , 1 ).astype('float32')
 X_test = X_test.reshape(X_test.shape[0], 28, 28 , 1 ).astype('float32' )<choose_model_class>
testData = pd.read_csv('.. /input/test.csv')
 testText = testData.loc[:,'text'].reset_index(drop=True)
 logProbGivenAuthor = np.zeros(( len(testText),3))
for i, sentence in enumerate(testText):
decodedSentence = myunidecode(sentence.lower())
 charSequences = [decodedSentence[k:k+historyLength+1] for k in range(len(decodedSentence)-historyLength)]
history = [seq[:-1] for seq in charSequences]
nextChar = [seq[ -1] for seq in charSequences]
logP_EAP = logP_EAP_prior; logP_HPL = logP_HPL_prior; logP_MWS = logP_MWS_prior;
for histChars, nextChar in zip(history,nextChar):
encodedHistChars = charEncoder.transform([ch for ch in histChars])
 encodedNextChar = charEncoder.transform([nextChar])[0]
logP_EAP += np.log(condP_H4['EAP'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_HPL += np.log(condP_H4['HPL'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logP_MWS += np.log(condP_H4['MWS'][encodedHistChars[0],encodedHistChars[1],encodedHistChars[2],encodedHistChars[3],encodedNextChar])
 logProbGivenAuthor[i,:] = [logP_EAP,logP_HPL,logP_MWS]
model = Sequential()
model.add(Conv2D(100, kernel_size=3, padding="valid", input_shape=(28, 28, 1), activation = 'relu'))<choose_model_class>
minimalLogP = -15.0
uniformPriorWeight = 0.09
authorLogProb_norm = logProbGivenAuthor - np.tile(logProbGivenAuthor.max(axis=1)[:,np.newaxis], [1,3])
 authorLogProb_norm[authorLogProb_norm < minimalLogP] = minimalLogP
authorProb = np.exp(authorLogProb_norm)
 authorProb_norm = authorProb / np.tile(authorProb.sum(axis=1)[:,np.newaxis],[1, 3])
 y_Hat = uniformPriorWeight*(1/3.0)+(1.0-uniformPriorWeight)*authorProb_norm
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,['EAP', 'HPL', 'MWS']] = y_Hat
submission.to_csv("Markov_char_given_4charHistory.csv", index=False)
 submission.head(10 )
model.add(Conv2D(100, kernel_size=3, padding="valid", activation = 'relu'))
model.add(Conv2D(100, kernel_size=3, padding="valid", activation = 'relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(1, 1)) )<choose_model_class>
warnings.filterwarnings("ignore")
 ngramLength = 5
featureExtractionStartTime = time.time()
print('-'*52)
 print('fitting "CountVectorizer() " for bag of char %d-grams' %(ngramLength))
BagOfCharsExtractor = CountVectorizer(min_df=8, max_features=250000,
analyzer='char', ngram_range=(1,ngramLength),
binary=False,lowercase=True)
 BagOfCharsExtractor.fit(pd.concat(( trainText,validText,testText)))
 X_train_char = BagOfCharsExtractor.transform(trainText)
 X_valid_char = BagOfCharsExtractor.transform(validText)
 X_test_char = BagOfCharsExtractor.transform(testText)
 featureExtractionDurationInMinutes =(time.time() -featureExtractionStartTime)/60.0
print("feature extraction took %.2f minutes" %(featureExtractionDurationInMinutes))
print('number of "bag of char %d-gram" features = %d' %(ngramLength, X_train_char.shape[1]))
print('-'*52)
 stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train_char,X_valid_char,X_test_char))))
X_train_norm = stdScaler.transform(X_train_char)
 X_valid_norm = stdScaler.transform(X_valid_char)
 X_test_norm = stdScaler.transform(X_test_char)
 yLabelEncoder = preprocessing.LabelEncoder()
yLabelEncoder.fit(pd.concat(( trainLabel,validLabel)))
 y_train = yLabelEncoder.transform(trainLabel)
 y_valid = yLabelEncoder.transform(validLabel)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor_char = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor_char.fit(X_train_norm, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor_char.predict(X_train_norm))
validAccuracy = accuracy_score(y_valid, logisticRegressor_char.predict(X_valid_norm))
trainLogLoss = log_loss(y_train, logisticRegressor_char.predict_proba(X_train_norm))
validLogLoss = log_loss(y_valid, logisticRegressor_char.predict_proba(X_valid_norm))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42 )
model.add(Flatten())
 model.add(Dense(units= 500, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation("softmax"))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )<data_type_conversions>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor_char.predict_proba(X_test_norm)
 submission.to_csv("LogisticRegression_Over_BagOfCharNGrams.csv", index=False)
 submission.head(10 )
y_train = np_utils.to_categorical(y_train ).astype('int32')
 y_test = np_utils.to_categorical(y_test )<init_hyperparams>
ngramLength = 2
featureExtractionStartTime = time.time()
print('-'*52)
 print('fitting "CountVectorizer() " for bag of word %d-grams' %(ngramLength))
BagOfWordsExtractor = CountVectorizer(min_df=5, max_features=250000,
analyzer='word', ngram_range=(1,ngramLength),
binary=False,lowercase=True)
 BagOfWordsExtractor.fit(pd.concat(( trainText,validText,testText)))
 X_train_word = BagOfWordsExtractor.transform(trainText)
 X_valid_word = BagOfWordsExtractor.transform(validText)
 X_test_word = BagOfWordsExtractor.transform(testText)
 featureExtractionDurationInMinutes =(time.time() -featureExtractionStartTime)/60.0
print("feature extraction took %.2f minutes" %(featureExtractionDurationInMinutes))
print('number of "bag of word %d-gram" features = %d' %(ngramLength, X_train_word.shape[1]))
print('-'*52)
 stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train_word,X_valid_word,X_test_word))))
X_train_norm = stdScaler.transform(X_train_word)
 X_valid_norm = stdScaler.transform(X_valid_word)
 X_test_norm = stdScaler.transform(X_test_word)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor_word = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor_word.fit(X_train_norm, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor_word.predict(X_train_norm))
validAccuracy = accuracy_score(y_valid, logisticRegressor_word.predict(X_valid_norm))
trainLogLoss = log_loss(y_train, logisticRegressor_word.predict_proba(X_train_norm))
validLogLoss = log_loss(y_valid, logisticRegressor_word.predict_proba(X_valid_norm))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42 )
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-3,
patience=25,
verbose=1)
 ]<train_model>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor_word.predict_proba(X_test_norm)
 submission.to_csv("LogisticRegression_Over_BagOfWordNGrams.csv", index=False)
 submission.head(10 )
<train_model><EOS> 
X_train = scipy.sparse.hstack(( X_train_word,X_train_char))
X_valid = scipy.sparse.hstack(( X_valid_word,X_valid_char))
X_test = scipy.sparse.hstack(( X_test_word,X_test_char))
stdScaler = preprocessing.StandardScaler(with_mean=False)
 stdScaler.fit(scipy.sparse.vstack(((X_train,X_valid,X_test))))
X_train = stdScaler.transform(X_train)
 X_valid = stdScaler.transform(X_valid)
 X_test = stdScaler.transform(X_test)
 validationStartTime = time.time()
print('-'*42)
 print('fitting "LogisticRegression() " classifier')
 logisticRegressor = linear_model.LogisticRegression(C=0.01, solver='sag')
 logisticRegressor.fit(X_train, y_train)
 trainAccuracy = accuracy_score(y_train, logisticRegressor.predict(X_train))
trainLogLoss = log_loss(y_train, logisticRegressor.predict_proba(X_train))
validAccuracy = accuracy_score(y_valid, logisticRegressor.predict(X_valid))
validLogLoss = log_loss(y_valid, logisticRegressor.predict_proba(X_valid))
validationDurationInMinutes =(time.time() -validationStartTime)/60.0
print('Validation took %.2f minutes' %(validationDurationInMinutes))
print('Train: %.1f%s Accuracy, log loss = %.4f' %(100*trainAccuracy,'%',trainLogLoss))
print('Valid: %.1f%s Accuracy, log loss = %.4f' %(100*validAccuracy,'%',validLogLoss))
print('-'*42)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.loc[:,yLabelEncoder.classes_.tolist() ] = logisticRegressor.predict_proba(X_test)
 submission.to_csv("LogisticRegression_Over_BagOfWord_BagOfChar.csv", index=False)
 submission.head(10 )