Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<drop_column>
train = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv", header=0, \
delimiter="\t", quoting=3)
 train = train.drop(['id'], axis=1)
 train.head()
df=df.drop(columns=['month', 'day'] )<categorify>
df2 = pd.read_csv('.. /input/imdb-review-dataset/imdb_master.csv',encoding="latin-1")
season_encoded = pd.get_dummies(df['season'],prefix= 'season')
 df = pd.concat(( df,season_encoded), axis=1)
 df = df.drop(columns = 'season' )<feature_engineering>
df2 = df2.drop(['Unnamed: 0','type','file'],axis=1)
 df2.columns = ["review","sentiment"]
df2.head()
df['holiday'] = df['holiday']<feature_engineering>
df2 = df2[df2.sentiment != 'unsup']
df2['sentiment'] = df2['sentiment'].map({'pos': 1, 'neg': 0})
 df2.head()
df['workingday'] = df['workingday']<categorify>
training = pd.concat([train, df2] ).reset_index(drop=True)
 training.shape
weather_encoded = pd.get_dummies(df['weather'],prefix= 'weather')
 df = pd.concat(( df,weather_encoded), axis=1)
 df = df.drop(columns = 'weather' )<sort_values>
def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review, ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
meaningful_words = [w for w in words if not w in stops]
return(" ".join(meaningful_words))
skew = df.apply(lambda x: skew(x))
skew.sort_values(ascending = False )<filter>
num_reviews = training["review"].size
clean_train_reviews = []
print("Cleaning and parsing the training set movie reviews...
 ")
 clean_train_reviews = []
for i in range(0, num_reviews):
if(( i+1)%10000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_train_reviews.append(review_to_words(training["review"][i]))
print("Done" )
skew = skew[abs(skew)> 0.5]
skew<import_modules>
print("Creating the bag of words...
 ")
 vectorizer = CountVectorizer(analyzer = "word", \
tokenizer = None, \
preprocessor = None, \
stop_words = None, \
max_features = 6000)
 train_data_features = vectorizer.fit_transform(clean_train_reviews)
 train_data_features = train_data_features.toarray()
print("Done")
from statsmodels.stats.outliers_influence import variance_inflation_factor<create_dataframe>
x_train,x_test,y_train,y_test=train_test_split(train_data_features,training["sentiment"],test_size=0.2,random_state=0)
 print("Splitting Done" )
vif = pd.DataFrame()
vif['Features'] = vif_data.columns
vif['vif'] = [variance_inflation_factor(
 vif_data.values, i)for i in range(vif_data.shape[1])]
vif.sort_values(by='vif',ascending=False )<train_model>
print("Training the random forest model...")
 forest = RandomForestClassifier(n_estimators = 100)
 forest = forest.fit(x_train, y_train)
 print("Done")
pca=PCA(n_components=1)
 pca.fit(df[['temp', 'atemp']] )<set_options>
result = forest.predict(x_test)
 print("Predictions Done" )
pca.explained_variance_ratio_<categorify>
accuracy=accuracy_score(y_test,result)
 accuracy
df['pca']=pca.fit_transform(df[['temp','atemp']] )<compute_test_metric>
cm=confusion_matrix(y_test,result)
 cm
stats.pearsonr(train['temp'],target )<feature_engineering>
f1=f1_score(y_test,result)
 f1
df.loc[df['windspeed']==0, 'windspeed']=df['windspeed'].mean()<drop_column>
test = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv", header=0, delimiter="\t", \
quoting=3)
 print(test.shape)
 num_reviews = len(test["review"])
 clean_test_reviews = []
print("Cleaning and parsing the test set movie reviews...
 ")
 for i in range(0,num_reviews):
if(( i+1)% 5000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_review = review_to_words(test["review"][i])
 clean_test_reviews.append(clean_review)
 test_data_features = vectorizer.transform(clean_test_reviews)
 test_data_features = test_data_features.toarray()
result = forest.predict(test_data_features)
 output = pd.DataFrame(data={"id":test["id"], "sentiment":result})
 output.to_csv("Bag_of_Words_model.csv", index=False, quoting=3)
 print("Done" )