Kaggle competition: Bag of Words Meets Bags of Popcorn
df_total['year'] = df_total['datetime'].dt.year
df_total['month'] = df_total['datetime'].dt.month
df_total['day'] = df_total['datetime'].dt.day
df_total['dayofweek'] = df_total['datetime'].dt.dayofweek
df_total['hour'] = df_total['datetime'].dt.hour
df_total.info()<split>
train = pd.read_csv("/kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv", header=0, delimiter="\t", quoting=3)
 train.shape
train.head()
train['sentiment'].value_counts()
train, valid = train_test_split(df, random_state=42)
 train.shape, valid.shape<define_variables>
xtrain, xvalid, ytrain, yvalid = train_test_split(train.review.values, train.sentiment.values,
random_state=42,
test_size=0.2, shuffle=True)
 xtrain
removed = ['casual', 'registered', 'count', 'datetime']
feats = [col for col in train.columns if col not in removed]<choose_model_class>
print(train["review"][0])
 print(BeautifulSoup(train["review"][0] ).get_text())
 def prep_remove_html(text):
return BeautifulSoup(text ).get_text().lower()
rf = RandomForestRegressor(random_state=42 )<train_model>
def prep_only_words(text):
return re.sub("[^a-zA-Z]", " ", text ).split()
rf.fit(train[feats], train['count'] )<predict_on_test>
def prep_remove_stopwords(word_list):
return [w for w in word_list if not w in stopwords.words("english")]
preds = rf.predict(valid[feats] )<compute_test_metric>
def preprocess(text):
return " ".join(prep_remove_stopwords(prep_only_words(prep_remove_html(text))))
mean_squared_error(valid['count'], preds)**(1/2 )<predict_on_test>
test['count'] = np.exp(rf.predict(test[feats]))<save_to_csv>
vectorizer = TfidfVectorizer(max_features=5000,
ngram_range=(1, 3),
stop_words = stopwords.words('english'),
preprocessor = prep_remove_html,
strip_accents='unicode')
 vectorizer.fit(list(xtrain)+ list(xvalid))
test[['datetime', 'count']].to_csv('rf.csv', index=False )<train_model>
xtrain_features = vectorizer.transform(xtrain)
 xvalid_features = vectorizer.transform(xvalid)
rf2 = RandomForestRegressor(random_state=42, n_estimators=200, n_jobs=-1, min_samples_leaf=4)
 rf2.fit(train[feats], train['count'])
 preds2 = rf2.predict(valid[feats])
 mean_squared_error(valid['count'], preds2)**(1/2 )<save_to_csv>
clf_lr = LogisticRegression()
clf_lr.fit(xtrain_features, ytrain)
 clf_lr.score(xvalid_features, yvalid)
test['count'] = np.exp(rf2.predict(test[feats]))
test[['datetime', 'count']].to_csv('rf2.csv', index=False )<filter>
clf_nb = BernoulliNB()
clf_nb.fit(xtrain_features, ytrain)
 clf_nb.score(xvalid_features, yvalid)
train = df[df['day'] <= 15]
valid = df[df['day'] > 15]
train.shape, valid.shape<compute_train_metric>
clf_rf = RandomForestClassifier(n_estimators = 100)
 clf_rf.fit(xtrain_features, ytrain)
 clf_rf.score(xvalid_features, yvalid )
rf3 = RandomForestRegressor(random_state=42, n_estimators=200, n_jobs=-1, min_samples_leaf=4)
 rf3.fit(train[feats], train['count'])
 preds3 = rf3.predict(valid[feats])
 mean_squared_error(valid['count'], preds3)**(1/2 )<save_to_csv>
test = pd.read_csv("/kaggle/input/word2vec-nlp-tutorial/testData.tsv", header=0, delimiter="\t", quoting=3)
 test_features = vectorizer.transform(test.review.values)
 predictions = clf_lr.predict(test_features)
 output = pd.DataFrame(data={"id":test["id"], "sentiment":predictions})
 output.to_csv("tfidf-logreg.csv", index=False, quoting=3 )