Kaggle competition: Bag of Words Meets Bags of Popcorn
train = pd.read_csv('.. /input/labeledTrainData.tsv', delimiter='\t')
 test = pd.read_csv('.. /input/testData.tsv', delimiter='\t')
 train.shape, test.shape<count_values>
train = pd.read_csv('.. /input/labeledTrainData.tsv', header = 0,
delimiter= "\t", quoting = 3)
 print(train.shape)
 display(train.head() )
print("number of rows for sentiment 1: {}".format(len(train[train.sentiment == 1])))
 print("number of rows for sentiment 0: {}".format(len(train[train.sentiment == 0])) )<feature_engineering>
example1 = BeautifulSoup(train['review'][0])
 print(train['review'][0])
 print("
")
 print(example1.get_text() )
train['length'] = train['review'].apply(len)
 train.head()<string_transform>
letters_only = re.sub("[^a-zA-Z]", " ", example1.get_text())
 print(letters_only )
def clean_text(raw_text):
raw_text = BeautifulSoup(raw_text, 'lxml' ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", raw_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
return [w for w in words if not w in stops]<feature_engineering>
lower_case = letters_only.lower()
words = lower_case.split()
print(words )
train['clean_review'] = train['review'].apply(clean_text)
 train['length_clean_review'] = train['clean_review'].apply(len)
 train.head()<filter>
words = [w for w in words if not w in stopwords.words("english")]
print(words )
print(train[train['length_clean_review'] == 4]['review'].iloc[0])
 print('------After Cleaning------')
 print(train[train['length_clean_review'] == 4]['clean_review'].iloc[0] )<import_modules>
stops = set(stopwords.words("english"))
def review_to_words(review):
review = BeautifulSoup(review ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review)
 lower_case = letters_only.lower()
words = lower_case.split()
words = [w for w in words if not w in stops]
return " ".join(words )
from sklearn.feature_extraction.text import CountVectorizer<feature_engineering>
review_to_words(train['review'][0] )
bow_transform = CountVectorizer(analyzer=clean_text ).fit(train['review'])
 print(len(bow_transform.vocabulary_))<categorify>
num_reviews = train.shape[0]
train_corpus = []
for i in tqdm_notebook(range(num_reviews)) :
train_corpus.append(review_to_words(train['review'][i]))
review_bow = bow_transform.transform(train['review'] )<train_model>
len(train_corpus[-1].split() )
sparsity =(100.0 * review_bow.nnz /(review_bow.shape[0] * review_bow.shape[1]))
print('sparsity: {}'.format(sparsity))<categorify>
vectorizer = CountVectorizer(analyzer = 'word', tokenizer = None,
preprocessor = None, stop_words = None,
max_features = 8000 )
tfidf_transformer = TfidfTransformer().fit(review_bow)
 tfidf1 = tfidf_transformer.transform(bow1)
 print(tfidf1 )<categorify>
vectorizer.fit(train_corpus )
review_tfidf = tfidf_transformer.transform(review_bow)
 print(review_tfidf.shape )<split>
X_train = vectorizer.transform(train_corpus ).toarray()
y_train = train.sentiment.values.reshape(-1,1)
 print(X_train.shape, y_train.shape )
X_train, X_test, y_train, y_test = train_test_split(train['review'], train['sentiment'], test_size=0.22, random_state=101)
 len(X_train), len(X_test), len(X_train)+ len(X_test )<compute_test_metric>
print(len(train_corpus[0]))
print(np.sum(X_train, axis = 1))
def pred(predicted,compare):
cm = pd.crosstab(compare,predicted)
 TN = cm.iloc[0,0]
FN = cm.iloc[1,0]
TP = cm.iloc[1,1]
FP = cm.iloc[0,1]
print("CONFUSION MATRIX ------->> ")
 print(cm)
 print()
print('Classification paradox :------->>')
 print('Accuracy :- ', round(((TP+TN)*100)/(TP+TN+FP+FN),2))
print()
print('False Negative Rate :- ',round(( FN*100)/(FN+TP),2))
print()
print('False Postive Rate :- ',round(( FP*100)/(FP+TN),2))
print()
print(classification_report(compare,predicted))<predict_on_test>
X_train, X_val, y_train, y_val = train_test_split(X_train , y_train.reshape(-1,),
test_size = 0.2,
stratify = y_train.reshape(-1,),
random_state = 42)
 print("Training Data = ", X_train.shape, y_train.shape)
 print("Validation Date = ", X_val.shape, y_val.shape )
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', LogisticRegression(random_state=101)) ,
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )<predict_on_test>
test = pd.read_csv(".. /input/testData.tsv", delimiter = "\t",
header = 0, quoting = 3)
 print(test.shape)
 test.head()
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )<predict_on_test>
num_test_reviews = test.shape[0]
test_corpus = []
for i in tqdm_notebook(range(num_test_reviews)) :
test_corpus.append(review_to_words(test['review'][i]))
print(len(test))
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', MultinomialNB()),
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )<predict_on_test>
clfrNB = naive_bayes.MultinomialNB()
clfrNB.fit(X_train, y_train.reshape(-1,))
y_pred = clfrNB.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrNB.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrNB.score(X_train, y_train))
print("Validation Accuracy", clfrNB.score(X_val, y_val))
print(cm_train)
 print(cm_val )
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )<train_model>
y_test = clfrNB.predict(X_test)
 print(y_test.shape)
 y_test
pipeline = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', RandomForestClassifier(n_estimators = 500)) ,
])
 pipeline.fit(X_train,y_train)
 predictions = pipeline.predict(X_train)
 pred(predictions,y_train )<predict_on_test>
sample = pd.read_csv(".. /input/sampleSubmission.csv", header = 0)
 sample.head()
predictions = pipeline.predict(X_test)
 pred(predictions,y_test )<predict_on_test>
sub = pd.DataFrame()
sub['id'] = sample['id'].values
sub['sentiment'] = y_test
sub.to_csv("naiveBayesPred.csv", index = False )
pipeline_logit = Pipeline([ 
('bow', CountVectorizer(analyzer=clean_text)) , 
('tfidf', TfidfTransformer()), 
('classifier', LogisticRegression(random_state=101)) ,
])
 pipeline_logit.fit(train['review'],train['sentiment'])
 test['sentiment'] = pipeline_logit.predict(test['review'] )<save_to_csv>
clfrRMC = RandomForestClassifier(n_estimators= 500, criterion = "gini",
max_features = "auto", max_depth = 8,
min_samples_split = 2, random_state = 42)
 clfrRMC.fit(X_train, y_train.reshape(-1,))
y_pred = clfrRMC.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrRMC.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrRMC.score(X_train, y_train))
print("Validation Accuracy = ", clfrRMC.score(X_val, y_val))
print(cm_train)
 print(cm_val )
output.to_csv("output.csv", index=False, quoting=3 )<import_modules>
y_test = clfrRMC.predict(X_test)
 print(y_test.shape)
 print(y_test )
print(os.listdir(".. /input")) 
<load_from_csv>
sub = pd.DataFrame()
sub['id'] = sample['id']
sub['sentiment'] = y_test
sub.to_csv("randomForestPred.csv", index = False)
 print(sub.shape)
 sub.head()
raw_train_data_labeled = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 raw_train_data_unlabeled = pd.read_csv(".. /input/word2vec-nlp-tutorial/unlabeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 raw_test_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv", header=0, delimiter="\t", quoting=3 )<load_from_csv>
clfrLR = LogisticRegression(penalty = 'l2', C = 0.005)
 clfrLR.fit(X_train, y_train)
 y_pred = clfrLR.predict(X_train)
 cm_train = confusion_matrix(y_train, y_pred)
 y_pred = clfrLR.predict(X_val)
 cm_val = confusion_matrix(y_val, y_pred)
 print("Training Accuracy = ", clfrLR.score(X_train, y_train))
print("Validation Accuracy = ", clfrLR.score(X_val, y_val))
print(cm_train)
 print(cm_val )
imdb_data = pd.read_csv('.. /input/imdb-review-dataset/imdb_master.csv',encoding="latin-1")
 imdb_data.head() 
<drop_column>
y_test = clfrLR.predict(X_test)
 print(y_test.shape)
 print(y_test )
imdb_data = imdb_data.drop(["Unnamed: 0","type","file"],axis=1 )<categorify>
sub = pd.DataFrame()
sub['id'] = sample['id']
sub['sentiment'] = y_test
sub.to_csv("LogisticPrediction.csv", index = False)
 print(sub.shape)
 sub.head()