Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<groupby>
training = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip", header=0, \
delimiter="\t", quoting=3)
 print(training.shape)
 y_train = training['sentiment']
x_train = training.drop(["sentiment"], axis=1)
 training.head()
group_year_month = df.groupby(['year','month'] )<groupby>
import nltk
import numpy as np
import matplotlib.pyplot as plt
average_year_month = group_year_month['count'].mean()<groupby>
stopwords_list = set(stopwords.words("english"))
print(f'List of stopwords:
{stopwords_list}
')
 no_stopwords = ["not","don't",'aren','don','ain',"aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn', "hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't",
'ma', 'mightn', "mightn't", 'mustn', "mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't",
"won't", 'wouldn', "wouldn't"]
for no_stopword in no_stopwords:
stopwords_list.remove(no_stopword)
 print(f'Final list of stopwords:
{stopwords_list}' )
group_workingday_hour = df.groupby(['workingday','hour'])
 average_workingday_hour = group_workingday_hour['count'].mean()<save_to_csv>
lemmatizer = WordNetLemmatizer()
def lemma_stem_text(words_list):
text = [lemmatizer.lemmatize(token.lower())for token in words_list]
text = [lemmatizer.lemmatize(token.lower() , "v")for token in text]
return text
word_example = "feet"
print(f'The word "{word_example}" is transformed to "{lemma_stem_text([word_example])[0]}"' )
df.to_csv('bike_all.csv',index=True,index_label='datetime',columns=columns )<randomize_order>
re_negation = re.compile("n't ")
 def negation_abbreviated_to_standard(sent):
sent = re_negation.sub(" not ", sent)
 return sent
word_example = "I aren't "
print(f'The sentence "{word_example}" is transformed to "{negation_abbreviated_to_standard(word_example)}"' )
np.random.seed(5)
 l = list(df.index)
 np.random.shuffle(l)
 df = df.loc[l]<define_variables>
def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review ).get_text()
review_text = negation_abbreviated_to_standard(review_text)
 letters_numbers_only = re.sub("[^a-zA-Z_0-9]", " ", review_text)
 words = np.char.lower(letters_numbers_only.split())
 meaningful_words = [w for w in words if not w in stopwords_list]
lemma_words = lemma_stem_text(meaningful_words)
 return(" ".join(lemma_words))
rows = df.shape[0]
train = int (.7 * rows)
 test = rows-train<save_to_csv>
clean_review = review_to_words(x_train["review"][0])
 print(f'Text of original review:
{x_train["review"][0]}
')
 print(f'Sentiment review: {y_train[0]}
')
 print(f'Text of cleaned review:
{clean_review}' )
df.iloc[:train].to_csv('bike_train.csv'
,index=False,header=False
,columns=columns )<save_to_csv>
reviews = x_train['review']
cleaned_train_reviews = []
for i in reviews:
cleaned_train_reviews.append(review_to_words(i))
df.iloc[train:].to_csv('bike_validation.csv'
,index=False,header=False
,columns=columns )<save_to_csv>
vectorizer = TfidfVectorizer(max_features=20000, ngram_range =(1,2))
train_data_features = vectorizer.fit_transform(cleaned_train_reviews)
 print(train_data_features.shape )
df_test.to_csv('bike_test.csv',index=True,index_label='datetime' )<save_to_csv>
model = LogisticRegression(random_state=0, solver='lbfgs',
multi_class='multinomial')
 model.fit(train_data_features, y_train)
with open('bike_train_column_list.txt','w')as f:
f.write(','.join(columns))<install_modules>
test = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv.zip", header=0, delimiter="\t", \
quoting=3)
 print(test.shape)
 num_reviews = len(test["review"])
 clean_test_reviews = []
print("Cleaning and parsing the test set movie reviews...
 ")
 for i in range(0,num_reviews):
clean_review = review_to_words(test["review"][i])
 clean_test_reviews.append(clean_review)
 test_data_features = vectorizer.transform(clean_test_reviews)
 result = model.predict(test_data_features)
 output = pd.DataFrame(data={"id":test["id"], "sentiment":result} )
!pip install xgboost==0.90<import_modules>
output.to_csv("submission.csv", index=False, quoting=3 )
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error, mean_absolute_error
import xgboost as xgb<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train_data_features, y_train, test_size=0.2, random_state=42 )
column_list_file = 'bike_train_column_list.txt'
train_file = 'bike_train.csv'
validation_file = 'bike_validation.csv'
test_file = 'bike_test.csv'<load_from_csv>
forest = RandomForestClassifier(n_estimators = 500, max_depth = None, min_samples_split=2, min_samples_leaf =1,
bootstrap = True, random_state=0)
 forest = forest.fit(X_train, y_train)
 print(forest.score(X_test, y_test))
df_train = pd.read_csv(train_file,names=columns)
 df_validation = pd.read_csv(validation_file,names=columns )<prepare_x_and_y>
logistic = LogisticRegression(random_state=0, solver='lbfgs',
multi_class='multinomial')
 logistic = logistic.fit(X_train, y_train)
 print(logistic.score(X_test, y_test))
X_train = df_train.iloc[:,1:]
y_train = df_train.iloc[:,0].ravel()
X_validation = df_validation.iloc[:,1:]
y_validation = df_validation.iloc[:,0].ravel()<choose_model_class>
y_pred_forest = forest.predict(X_test)
 y_pred_logistic = logistic.predict(X_test )
regressor = xgb.XGBRegressor(max_depth=5,n_estimators=150 )<train_model>
confusion_matrix_forest = confusion_matrix(y_test, y_pred_forest, labels=[1,0])
 confusion_matrix_forest
<compute_train_metric><EOS> 
confusion_matrix_logistic = confusion_matrix(y_test, y_pred_logistic, labels=[1,0])
 confusion_matrix_logistic