Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<prepare_x_and_y>
!pip install bs4
train = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip", header=0, \
delimiter="\t", quoting=3)
 train = train.drop(['id'], axis=1)
 train.head()
X = train_test_data[:ntrain,:]
X = np.append(X,train_test_data[ntrain+25000: ,:])
 X = X.reshape(-1,123)
 y1 = y.append(imdb_data['sentiment'] )<split>
df2 = pd.read_csv('.. /input/imdb-review-dataset/imdb_master.csv',encoding="latin-1")
X_train, X_test, y_train, y_test = train_test_split(X , y1, test_size=0.2, random_state=42, shuffle=True )<choose_model_class>
df2 = df2.drop(['Unnamed: 0','type','file'],axis=1)
 df2.columns = ["review","sentiment"]
df2.head()
model = Sequential()
model.add(Embedding(input_dim = vocab_size, output_dim = embedding_vector_size,
input_length = max_len, weights = [embedding_matrix]))
model.add(Bidirectional(LSTM(64, dropout=0.25, recurrent_dropout=0.1)))
 model.add(Dense(10))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))<choose_model_class>
df2 = df2[df2.sentiment != 'unsup']
df2['sentiment'] = df2['sentiment'].map({'pos': 1, 'neg': 0})
 df2.head()
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
training = pd.concat([train, df2] ).reset_index(drop=True)
 training.shape
model.compile(optimizer='RMSprop', loss='binary_crossentropy', metrics=['acc'])
 print(model.summary())
 history = model.fit(X_train, y_train, epochs = 30, batch_size = 700, validation_data=(X_test, y_test),callbacks = [learning_rate_reduction] )<predict_on_test>
def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review, ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
meaningful_words = [w for w in words if not w in stops]
return(" ".join(meaningful_words))
y_test_pred = model.predict(X_test)
<compute_test_metric>
num_reviews = training["review"].size
clean_train_reviews = []
print("Cleaning and parsing the training set movie reviews...
 ")
 clean_train_reviews = []
for i in range(0, num_reviews):
if(( i+1)%10000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_train_reviews.append(review_to_words(training["review"][i]))
print("Done" )
roc_auc_score(y_test, y_test_pred, average = 'weighted' )<predict_on_test>
print("Creating the bag of words...
 ")
 vectorizer = CountVectorizer(analyzer = "word", \
tokenizer = None, \
preprocessor = None, \
stop_words = None, \
max_features = 6000)
 train_data_features = vectorizer.fit_transform(clean_train_reviews)
 train_data_features = train_data_features.toarray()
print("Done")
y_pred = model.predict(train_test_data[ntrain:ntrain+25000 , :] )<define_variables>
x_train,x_test,y_train,y_test=train_test_split(train_data_features,training["sentiment"],test_size=0.2,random_state=0)
 print("Splitting Done" )
predictions = [1 if(x>0.5)else 0 for x in y_pred ]<define_variables>
print("Training the random forest model...")
 forest = RandomForestClassifier(n_estimators = 100)
 forest = forest.fit(x_train, y_train)
 print("Done" )
ids = raw_test_data['id'].str.replace('"', '' )<save_to_csv>
result = forest.predict(x_test)
 print("Predictions Done" )
submission = pd.DataFrame({'id': ids, 'sentiment':predictions})
 submission.to_csv('word2vec.csv',index=False )<load_from_csv>
accuracy=accuracy_score(y_test,result)
 accuracy
train = pd.read_csv(".. /input/labeledTrainData.tsv", header=0, delimiter="\t", quoting=3 )<string_transform>
cm=confusion_matrix(y_test,result)
 cm
sw = stopwords.words('english')
 for w in ['film','see','get','make','fill']:
sw.append(w)
 sw = set(sw )<string_transform>
f1=f1_score(y_test,result)
 f1
def stage_one(text):
text = BeautifulSoup(text ).get_text()
reg = re.sub("[^a-zA-Z]", " ", text)
 low = reg.lower().split()
meaningful_words = [w for w in low if not w in sw]
return(" ".join(meaningful_words))<categorify>
test = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv.zip", header=0, delimiter="\t", \
quoting=3)
 print(test.shape)
 num_reviews = len(test["review"])
 clean_test_reviews = []
print("Cleaning and parsing the test set movie reviews...
 ")
 for i in range(0,num_reviews):
if(( i+1)% 5000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_review = review_to_words(test["review"][i])
 clean_test_reviews.append(clean_review)
 test_data_features = vectorizer.transform(clean_test_reviews)
 test_data_features = test_data_features.toarray()
result = forest.predict(test_data_features)
 output = pd.DataFrame(data={"id":test["id"], "sentiment":result})
 output.to_csv("Bag_of_Words_model.csv", index=False, quoting=3)
 print("Done" )