Kaggle competition: Digit Recognizer
!pip install gensim -i http://pypi.douban.com/simple --trusted-host pypi.douban.com<string_transform>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
stop_words = stopwords.words('english')
 for word in ['us','no','yet']:
stop_words.append(word)
 data_list = []
text_series = data['text']
for i in range(len(text_series)) :
content = text_series.iloc[i]
cutwords = [word for word in content.split(' ')if word not in stop_words if len(word)!= 0]
data_list.append(cutwords )<define_variables>
from fastai import *
from fastai.vision import *
import numpy as np
import pandas as pd
import re
for i in range(len(data_list)) :
content = data_list[i]
if len(content)<1:
print(i )<load_pretrained>
df_train = pd.read_csv('.. /input/train.csv')
 df_train['fn'] = df_train.index
df_train.head()
word2vec_path='./GoogleNews-vectors-negative300.bin.gz'
word2vec_model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True )<feature_engineering>
data =(src.transform(tfms=(rand_pad(padding=5,size=28,mode='zeros'),[])) 
.databunch(num_workers=2,bs=128)
.normalize(imagenet_stats))
def get_textVector(data_list, word2vec, textsVectors_list):
for i in range(len(data_list)) :
words_perText = data_list[i]
if len(words_perText)< 1:
words_vector = [np.zeros(300)]
else:
words_vector = [word2vec.wv[k] if k in word2vec_model else np.zeros(300)for k in words_perText]
text_vector = np.array(words_vector ).mean(axis=0)
 textsVectors_list.append(text_vector)
 return textsVectors_list<prepare_x_and_y>
data.show_batch(rows=3,figsize=(10,7))
textsVectors_list = []
get_textVector(data_list, word2vec_model, textsVectors_list)
 X = np.array(textsVectors_list )<count_missing_values>
learn = cnn_learner(data,models.resnet50,metrics=accuracy,model_dir='/kaggle/model' )
pd.isnull(X ).any()<split>
learn.lr_find(end_lr=100 )
word2vec_X = X[0:train_data.shape[0]]
y = data['target'][0:train_data.shape[0]]
word2vec_submit = X[train_data.shape[0]:-1]
X_train_word2vec, X_test_word2vec, y_train_word2vec, y_test_word2vec = train_test_split(word2vec_X, y,
test_size = 0.2, random_state = 4 )<train_model>
lr = 1e-2
word2vec_lr = LogisticRegression(class_weight = 'balanced', solver = 'lbfgs', n_jobs = -1)
 word2vec_lr.fit(X_train_word2vec, y_train_word2vec)
 y_predicted_word2vec_lr = word2vec_lr.predict(X_test_word2vec )<compute_test_metric>
learn.fit_one_cycle(5,slice(lr))
score_metrics(y_test_word2vec, y_predicted_word2vec_lr )<define_variables>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
compare_list = []
for(i,j)in zip(y_test_word2vec, y_predicted_word2vec_lr):
k = i - j
compare_list.append(k)
 wrong_num = [i for i,j in enumerate(compare_list)if j != 0]
text_series[0:train_data.shape[0]][wrong_num]<define_variables>
learn.fit_one_cycle(8,slice(2e-5,lr/5))
lenlen = []
for i in range(len(data_list)) :
content = data_list[i]
perlen = len(content)
 lenlen.append(perlen)
 print(max(lenlen))<define_variables>
df_test = pd.read_csv('.. /input/test.csv')
 df_test['label'] = 0
df_test['fn'] = df_test.index
df_test.head()
max_sequence_length = 26
embedding_dim = 300<train_model>
learn.data.add_test(PixelImageItemList.from_df(df_test,path='.',cols='fn'))
tokenizer = Tokenizer()
tokenizer.fit_on_texts(data_list)
 sequences = tokenizer.texts_to_sequences(data_list)
 word_index = tokenizer.word_index
cnn_data = pad_sequences(sequences, maxlen = max_sequence_length)
 cnn_label = to_categorical(np.asarray(train_data['target']))
print('len of word_index:', len(word_index))
print('shape of data tensor:', cnn_data.shape)
 print('shape of label tensoe:', cnn_label.shape )<split>
pred_test = learn.get_preds(ds_type=DatasetType.Test )
trainCNN_data = cnn_data[0:train_data.shape[0]]
X_train_cnn, X_test_cnn, y_train_cnn, y_test_cnn = train_test_split(trainCNN_data, cnn_label,
test_size = 0.2, random_state = 4)
 X_cnn, X_val_cnn, y_cnn, y_val_cnn = train_test_split(X_train_cnn, y_train_cnn,
test_size = 0.2, random_state = 4 )<choose_model_class>
test_result = torch.argmax(pred_test[0],dim=1)
 result = test_result.numpy()
<train_model><EOS> 
final = pd.Series(result,name='Label')
 submission = pd.concat([pd.Series(range(1,28001),name='ImageId'),final],axis=1)
 submission.to_csv('fastai-res34-0.992.csv',index=False )