Kaggle competition: Digit Recognizer
def remove_stopwords(x):
x = [word for word in x.split() if word not in STOPWORDS]
x = ' '.join(x)
 return x<define_variables>
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing import image
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from tqdm import tqdm
contraction_mapping = {
"I'm": 'I am',
"I'm'a": 'I am about to',
"I'm'o": 'I am going to',
"I've": 'I have',
"I'll": 'I will',
"I'll've": 'I will have',
"I'd": 'I would',
"I'd've": 'I would have',
"i'm": 'i am',
"i'm'a": 'i am about to',
"i'm'o": 'i am going to',
"i've": 'i have',
"i'll": 'i will',
"i'll've": 'i will have',
"i'd": 'i would',
"i'd've": 'i would have',
'Whatcha': 'What are you',
'whatcha': 'what are you',
"amn't": 'am not',
"ain't": 'are not',
"aren't": 'are not',
"'cause": 'because',
"can't": 'can not',
"can't've": 'can not have',
"could've": 'could have',
"couldn't": 'could not',
"couldn't've": 'could not have',
"daren't": 'dare not',
"daresn't": 'dare not',
"dasn't": 'dare not',
"didn't": 'did not',
'didn’t': 'did not',
"don't": 'do not',
'don’t': 'do not',
"doesn't": 'does not',
"e'er": 'ever',
"everyone's": 'everyone is',
'finna': 'fixing to',
'gimme': 'give me',
"gon't": 'go not',
'gonna': 'going to',
'gotta': 'got to',
"hadn't": 'had not',
"hadn't've": 'had not have',
"hasn't": 'has not',
"haven't": 'have not',
"he've": 'he have',
"he's": 'he is',
"he'll": 'he will',
"he'll've": 'he will have',
"he'd": 'he would',
"he'd've": 'he would have',
"here's": 'here is',
"how're": 'how are',
"how'd": 'how did',
"how'd'y": 'how do you',
"how's": 'how is',
"how'll": 'how will',
"isn't": 'is not',
"it's": 'it is',
"'tis": 'it is',
"'twas": 'it was',
"it'll": 'it will',
"it'll've": 'it will have',
"it'd": 'it would',
"it'd've": 'it would have',
'kinda': 'kind of',
"let's": 'let us',
'luv': 'love',
"ma'am": 'madam',
"may've": 'may have',
"mayn't": 'may not',
"might've": 'might have',
"mightn't": 'might not',
"mightn't've": 'might not have',
"must've": 'must have',
"mustn't": 'must not',
"mustn't've": 'must not have',
"needn't": 'need not',
"needn't've": 'need not have',
"ne'er": 'never',
"o'": 'of',
"o'clock": 'of the clock',
"ol'": 'old',
"oughtn't": 'ought not',
"oughtn't've": 'ought not have',
"o'er": 'over',
"shan't": 'shall not',
"sha'n't": 'shall not',
"shalln't": 'shall not',
"shan't've": 'shall not have',
"she's": 'she is',
"she'll": 'she will',
"she'd": 'she would',
"she'd've": 'she would have',
"should've": 'should have',
"shouldn't": 'should not',
"shouldn't've": 'should not have',
"so've": 'so have',
"so's": 'so is',
"somebody's": 'somebody is',
"someone's": 'someone is',
"something's": 'something is',
'sux': 'sucks',
"that're": 'that are',
"that's": 'that is',
"that'll": 'that will',
"that'd": 'that would',
"that'd've": 'that would have',
'em': 'them',
"there're": 'there are',
"there's": 'there is',
"there'll": 'there will',
"there'd": 'there would',
"there'd've": 'there would have',
"these're": 'these are',
"they're": 'they are',
"they've": 'they have',
"they'll": 'they will',
"they'll've": 'they will have',
"they'd": 'they would',
"they'd've": 'they would have',
"this's": 'this is',
"those're": 'those are',
"to've": 'to have',
'wanna': 'want to',
"wasn't": 'was not',
"we're": 'we are',
"we've": 'we have',
"we'll": 'we will',
"we'll've": 'we will have',
"we'd": 'we would',
"we'd've": 'we would have',
"weren't": 'were not',
"what're": 'what are',
"what'd": 'what did',
"what've": 'what have',
"what's": 'what is',
"what'll": 'what will',
"what'll've": 'what will have',
"when've": 'when have',
"when's": 'when is',
"where're": 'where are',
"where'd": 'where did',
"where've": 'where have',
"where's": 'where is',
"which's": 'which is',
"who're": 'who are',
"who've": 'who have',
"who's": 'who is',
"who'll": 'who will',
"who'll've": 'who will have',
"who'd": 'who would',
"who'd've": 'who would have',
"why're": 'why are',
"why'd": 'why did',
"why've": 'why have',
"why's": 'why is',
"will've": 'will have',
"won't": 'will not',
"won't've": 'will not have',
"would've": 'would have',
"wouldn't": 'would not',
"wouldn't've": 'would not have',
"y'all": 'you all',
"y'all're": 'you all are',
"y'all've": 'you all have',
"y'all'd": 'you all would',
"y'all'd've": 'you all would have',
"you're": 'you are',
"you've": 'you have',
"you'll've": 'you shall have',
"you'll": 'you will',
"you'd": 'you would',
"you'd've": 'you would have',
'jan.': 'january',
'feb.': 'february',
'mar.': 'march',
'apr.': 'april',
'jun.': 'june',
'jul.': 'july',
'aug.': 'august',
'sep.': 'september',
'oct.': 'october',
'nov.': 'november',
'dec.': 'december',
'I’m': 'I am',
'I’m’a': 'I am about to',
'I’m’o': 'I am going to',
'I’ve': 'I have',
'I’ll': 'I will',
'I’ll’ve': 'I will have',
'I’d': 'I would',
'I’d’ve': 'I would have',
'i’m': 'i am',
'i’m’a': 'i am about to',
'i’m’o': 'i am going to',
'i’ve': 'i have',
'i’ll': 'i will',
'i’ll’ve': 'i will have',
'i’d': 'i would',
'i’d’ve': 'i would have',
'amn’t': 'am not',
'ain’t': 'are not',
'aren’t': 'are not',
'’cause': 'because',
'can’t': 'can not',
'can’t’ve': 'can not have',
'could’ve': 'could have',
'couldn’t': 'could not',
'couldn’t’ve': 'could not have',
'daren’t': 'dare not',
'daresn’t': 'dare not',
'dasn’t': 'dare not',
'doesn’t': 'does not',
'e’er': 'ever',
'everyone’s': 'everyone is',
'gon’t': 'go not',
'hadn’t': 'had not',
'hadn’t’ve': 'had not have',
'hasn’t': 'has not',
'haven’t': 'have not',
'he’ve': 'he have',
'he’s': 'he is',
'he’ll': 'he will',
'he’ll’ve': 'he will have',
'he’d': 'he would',
'he’d’ve': 'he would have',
'here’s': 'here is',
'how’re': 'how are',
'how’d': 'how did',
'how’d’y': 'how do you',
'how’s': 'how is',
'how’ll': 'how will',
'isn’t': 'is not',
'it’s': 'it is',
'’tis': 'it is',
'’twas': 'it was',
'it’ll': 'it will',
'it’ll’ve': 'it will have',
'it’d': 'it would',
'it’d’ve': 'it would have',
'let’s': 'let us',
'ma’am': 'madam',
'may’ve': 'may have',
'mayn’t': 'may not',
'might’ve': 'might have',
'mightn’t': 'might not',
'mightn’t’ve': 'might not have',
'must’ve': 'must have',
'mustn’t': 'must not',
'mustn’t’ve': 'must not have',
'needn’t': 'need not',
'needn’t’ve': 'need not have',
'ne’er': 'never',
'o’': 'of',
'o’clock': 'of the clock',
'ol’': 'old',
'oughtn’t': 'ought not',
'oughtn’t’ve': 'ought not have',
'o’er': 'over',
'shan’t': 'shall not',
'sha’n’t': 'shall not',
'shalln’t': 'shall not',
'shan’t’ve': 'shall not have',
'she’s': 'she is',
'she’ll': 'she will',
'she’d': 'she would',
'she’d’ve': 'she would have',
'should’ve': 'should have',
'shouldn’t': 'should not',
'shouldn’t’ve': 'should not have',
'so’ve': 'so have',
'so’s': 'so is',
'somebody’s': 'somebody is',
'someone’s': 'someone is',
'something’s': 'something is',
'that’re': 'that are',
'that’s': 'that is',
'that’ll': 'that will',
'that’d': 'that would',
'that’d’ve': 'that would have',
'there’re': 'there are',
'there’s': 'there is',
'there’ll': 'there will',
'there’d': 'there would',
'there’d’ve': 'there would have',
'these’re': 'these are',
'they’re': 'they are',
'they’ve': 'they have',
'they’ll': 'they will',
'they’ll’ve': 'they will have',
'they’d': 'they would',
'they’d’ve': 'they would have',
'this’s': 'this is',
'those’re': 'those are',
'to’ve': 'to have',
'wasn’t': 'was not',
'we’re': 'we are',
'we’ve': 'we have',
'we’ll': 'we will',
'we’ll’ve': 'we will have',
'we’d': 'we would',
'we’d’ve': 'we would have',
'weren’t': 'were not',
'what’re': 'what are',
'what’d': 'what did',
'what’ve': 'what have',
'what’s': 'what is',
'what’ll': 'what will',
'what’ll’ve': 'what will have',
'when’ve': 'when have',
'when’s': 'when is',
'where’re': 'where are',
'where’d': 'where did',
'where’ve': 'where have',
'where’s': 'where is',
'which’s': 'which is',
'who’re': 'who are',
'who’ve': 'who have',
'who’s': 'who is',
'who’ll': 'who will',
'who’ll’ve': 'who will have',
'who’d': 'who would',
'who’d’ve': 'who would have',
'why’re': 'why are',
'why’d': 'why did',
'why’ve': 'why have',
'why’s': 'why is',
'will’ve': 'will have',
'won’t': 'will not',
'won’t’ve': 'will not have',
'would’ve': 'would have',
'wouldn’t': 'would not',
'wouldn’t’ve': 'would not have',
'y’all': 'you all',
'y’all’re': 'you all are',
'y’all’ve': 'you all have',
'y’all’d': 'you all would',
'y’all’d’ve': 'you all would have',
'you’re': 'you are',
'you’ve': 'you have',
'you’ll’ve': 'you shall have',
'you’ll': 'you will',
'you’d': 'you would',
'you’d’ve': 'you would have'
}
def clean_contractions(text):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(" ")])
 return text<string_transform>
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )
def data_cleaning(x):
x = clean_tag(x)
 x = clean_punct(x)
 x = correct_mispell(x)
 x = remove_stopwords(x)
 x = clean_contractions(x)
 return x<categorify>
features = [c for c in train.columns if c not in ['label']]
y=train['label']
X = train.drop(labels = ["label"],axis = 1)
 X = X.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1)
def Preprocess(doc):
corpus=[]
for text in tqdm(doc):
text=clean_contractions(text)
 text=correct_mispell(text)
 text=re.sub(r'[^a-z0-9A-Z]'," ",text)
 text=re.sub(r'[0-9]{1}',"
text=re.sub(r'[0-9]{2}','
text=re.sub(r'[0-9]{3}','
text=re.sub(r'[0-9]{4}','
text=re.sub(r'[0-9]{5,}','
corpus.append(text)
 return corpus<load_from_disk>
y = to_categorical(y )
%%time
!unzip.. /input/quora-insincere-questions-classification/embeddings.zip<load_pretrained>
X = X / 255.0
test = test / 255.0
%%time
file_name="./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin"
model_embed=KeyedVectors.load_word2vec_format(file_name,binary=True)
 def vocab_build(corpus):
vocab={}
for text in tqdm(corpus):
for word in text.split() :
try:
vocab[word]+=1
except KeyError:
vocab[word]=1
return vocab
def check_voc(vocab,model):
embed_words=[]
out_vocab={}
total_words=0
total_text=0
for i in tqdm(vocab):
try:
vec=model[i]
embed_words.append(vec)
 total_words+=vocab[i]
except KeyError:
out_vocab[i]=vocab[i]
total_text+=vocab[i]
print("The {:.2f}% of vocabularies have Covered of corpus".format(100*len(embed_words)/len(vocab)))
 print("The {:.2f}% of total text had coverded ".format(( 100*total_words/(total_words+total_text))))
return out_vocab<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.1 )
total_text=pd.concat([df_train.question_text_cleaned,df_test.question_text_cleaned])
 vocabulary=vocab_build(total_text)
 oov=check_voc(vocabulary,model_embed )<feature_engineering>
model = Sequential()
model.add(Conv2D(64, kernel_size=(3, 3), padding='same',activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64, 5, padding='same', activation='relu'))
model.add(Conv2D(64, 5, padding='same', activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
df_test['question_text_cleaned_2'] = Preprocess(df_test['question_text'])
 df_train['question_text_cleaned_2'] = Preprocess(df_train['question_text'])
 total_text_2=pd.concat([df_train.question_text_cleaned_2,df_test.question_text_cleaned_2])
 vocabulary2=vocab_build(total_text_2)
 oov2=check_voc(vocabulary2, model_embed )<sort_values>
model.compile(loss='categorical_crossentropy',optimizer='Adam',metrics=['accuracy'])
sort_oov=dict(sorted(oov2.items() , key=operator.itemgetter(1),reverse=True))
dict(list(sort_oov.items())[:50] )<drop_column>
datagen = ImageDataGenerator(
 rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
 datagen.fit(X_train )
del oov, oov2,sort_oov,total_text,total_text_2
gc.collect()<categorify>
batch_size = 80
results = model.fit_generator(datagen.flow(X_train, y_train, batch_size = batch_size), epochs=50,
steps_per_epoch=X_train.shape[0] // batch_size,
validation_data=(X_test, y_test))
def get_word_index(vocab):
word_index=dict(( w,i+1)for i,w in enumerate(vocab.keys()))
return word_index
def fit_one_hot(word_index,corpus):
sent=[]
for text in tqdm(corpus):
li=[]
for word in text.split() :
try:
li.append(word_index[word])
 except KeyError:
li.append(0)
 sent.append(li)
 return sent<split>
final_loss, final_acc = model.evaluate(X_test, y_test, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
<find_best_params><EOS> 
prediction = model.predict(test)
 prediction = np.argmax(prediction,axis = 1)
 output=pd.DataFrame({"ImageId": list(range(1,len(prediction)+1)) ,
"Label": prediction})
 output.to_csv("output.csv", index=False, header=True )