Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<train_model>
print(os.listdir(".. /input"))
movie_train = pd.read_csv(".. /input/labeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 movie_test = pd.read_csv(".. /input/testData.tsv", header=0, delimiter="\t", quoting=3)
history = model.fit(partial_x_train,
partial_y_train,
epochs=30,
batch_size=512,
verbose=1 )<predict_on_test>
def clean_text(df, text_field):
df[text_field] = df[text_field].str.lower()
df[text_field] = df[text_field].apply(lambda elem: re.sub(r'http\S+', '', elem))
df[text_field] = df[text_field].apply(lambda elem: elem.replace('<br />', ' '))
df[text_field] = df[text_field].apply(lambda elem: elem.replace('\', ' '))
df[text_field] = df[text_field].apply(lambda elem: elem.replace('_', ' '))
remove_digits = str.maketrans('', '', digits)
 df[text_field] = df[text_field].apply(lambda elem: elem.translate(remove_digits))
return df
clean_movie_train = clean_text(movie_train, 'review')
 clean_movie_test = clean_text(movie_test, 'review' )
pred_val = np.rint(model.predict(x_val)).astype('int32' ).squeeze()
for i in range(50):
print(y_val[i], pred_val[i] )<predict_on_test>
tokenizer = RegexpTokenizer(r'\w+')
 clean_movie_train['tokens'] = clean_movie_train['review'].apply(tokenizer.tokenize)
 clean_movie_train.head()
clean_movie_test['tokens'] = clean_movie_test['review'].apply(tokenizer.tokenize )
pred = np.rint(model.predict(test_inputs)).astype('int32' ).squeeze()
print(pred.shape )<create_dataframe>
all_words = [word for tokens in clean_movie_train['tokens'] for word in tokens]
sentence_lengths = [len(tokens)for tokens in clean_movie_train['tokens']]
Vocab = sorted(list(set(all_words)))
 print('%s words total, with a vocabulary size of %s' %(len(all_words), len(Vocab)) )
commit_df = pd.DataFrame({'id': test_data['id'], 'sentiment':pred} )<save_to_csv>
print(Vocab[0:100])
 count_all_words = Counter(all_words)
 count_all_words.most_common(100 )
commit_df.to_csv('commit7.csv', index = False )<load_from_csv>
tfidf_vectorizer = TfidfVectorizer(analyzer = 'word', token_pattern = r'\w+')
 tfidf = dict()
tfidf['all_train'] =(tfidf_vectorizer.fit_transform(clean_movie_train['review']))
tfidf['test'] =(tfidf_vectorizer.transform(clean_movie_test['review']))
print(tfidf['all_train'].shape)
 print(tfidf['test'].shape )
commit_test = pd.read_csv("./commit7.csv")
 commit_test<load_from_csv>
sample = pd.read_csv(".. /input/word2vec-nlp-tutorial/sampleSubmission.csv")
 sample<load_from_csv>
classifier = LogisticRegression()
train_target = clean_movie_train['sentiment']
classifier.fit(tfidf['all_train'], train_target)
 submission = pd.DataFrame.from_dict({'id': clean_movie_test['id']})
 lr_predict = classifier.predict(tfidf['test'])
 submission = pd.DataFrame(data={"id":clean_movie_test["id"], "sentiment":lr_predict})
 submission.to_csv("submission.csv", index=False, quoting=3 )