Kaggle competition: Bag of Words Meets Bags of Popcorn
<SOS> metric: AUC Kaggle data source: bag-of-words-meets-bags-of-popcorn<define_variables>
warnings.filterwarnings('ignore' )
training_rounds = range(len(eval_result['validation_0']['rmse']))<load_from_csv>
import numpy as np
import pandas as pd
df = pd.read_csv(validation_file,names=columns )<predict_on_test>
nltk.download('stopwords')
result = regressor.predict(X_test )<feature_engineering>
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report,accuracy_score,confusion_matrix
df['count_predicted'] = result<filter>
train_lab=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/labeledTrainData.tsv',delimiter='\t',quoting=3)
 train_unlab=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/unlabeledTrainData.tsv',error_bad_lines=False,delimiter='\t')
 test=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/testData.tsv',delimiter='\t',quoting=3)
 submission=pd.read_csv('/kaggle/input/word2vec-nlp-tutorial/sampleSubmission.csv' )
df[df['count_predicted'] < 0]<count_values>
train_lab=train_lab.drop(columns=['id'] )
def adjust_count(x):
if x < 0:
return 0
else:
return x<categorify>
def review_word(raw_review):
review=BeautifulSoup(raw_review ).get_text()
review=re.sub('[^a-zA-Z]',' ',review)
 review=review.lower()
review=review.split()
ps=PorterStemmer()
review=[ps.stem(word)for word in review if not word in set(stopwords.words('english')) ]
review=' '.join(review)
 return review
df['count_predicted'] = df['count_predicted'].map(adjust_count )<filter>
corpus=[]
for i in range(0,25000):
corpus.append(review_word(train_lab['review'][i]))
df[df['count_predicted'] < 0]<feature_engineering>
tv=TfidfVectorizer(max_features=5000,analyzer='word' )
df['count'] = df['count'].map(np.expm1)
 df['count_predicted'] = df['count_predicted'].map(np.expm1 )<count_values>
X=tv.fit_transform(corpus ).toarray()
y=train_lab['sentiment']
value_counts =(residuals > 0 ).value_counts(sort=False)
 print(' Under Estimation: {0:0.2f}'.format(value_counts[True]/len(residuals)))
 print(' Over Estimation: {0:0.2f}'.format(value_counts[False]/len(residuals)) )<compute_test_metric>
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0 )
print("RMSE: {0:0.2f}".format(mean_squared_error(df['count'],df['count_predicted'])**.5))<compute_test_metric>
classifier_nb=GaussianNB()
classifier_nb.fit(X_train,y_train)
 y_pred_nb=classifier_nb.predict(X_test )
def compute_rmsle(y_true, y_pred):
if type(y_true)!= np.ndarray:
y_true = np.array(y_true)
 if type(y_pred)!= np.ndarray:
y_pred = np.array(y_pred)
 return(np.average(( np.log1p(y_pred)- np.log1p(y_true)) **2)**.5 )<compute_test_metric>
report=classification_report(y_pred_nb,y_test)
 print('the classification report is
{}'.format(report))
print('RMSLE')
 print(compute_rmsle(100,50),
compute_rmsle(1000,500),
compute_rmsle(10000,5000))<compute_test_metric>
classifier_rf=RandomForestClassifier(random_state=0)
 classifier_rf.fit(X_train,y_train)
 y_pred_rf=classifier_rf.predict(X_test )
print('RMSLE')
 print(compute_rmsle(100,25),
compute_rmsle(1000,250),
compute_rmsle(10000,2500))<compute_test_metric>
report=classification_report(y_pred_rf,y_test)
 print('the classification report is
{}'.format(report))
print('RMSE')
 print(mean_squared_error([100],[50])**.5,
mean_squared_error([1000],[500])**.5,
mean_squared_error([10000],[5000])**.5 )<compute_test_metric>
classifier_lr=LogisticRegression(random_state=0)
 classifier_lr.fit(X_train,y_train)
 y_pred_lr=classifier_lr.predict(X_test )
print('RMSE')
 print(mean_squared_error([100],[25])**.5,
mean_squared_error([1000],[250])**.5,
mean_squared_error([10000],[2500])**.5 )<compute_test_metric>
report=classification_report(y_pred_lr,y_test)
 print('the classification report is
{}'.format(report))
print("RMSLE: {0}".format(compute_rmsle(df['count'],df['count_predicted'])) )<load_from_csv>
classifier_svc=LinearSVC(random_state=0)
 classifier_svc.fit(X_train,y_train)
 y_pred_svc=classifier_svc.predict(X_test )
df_test = pd.read_csv(test_file,parse_dates=['datetime'] )<prepare_x_and_y>
report=classification_report(y_pred_svc,y_test)
 print('the classification report is
{}'.format(report))
X_test = df_test.iloc[:,1:]<predict_on_test>
tv1=TfidfVectorizer(max_features=5000,analyzer='word',ngram_range=(1,2))
result = regressor.predict(X_test )<prepare_output>
X1=tv1.fit_transform(corpus ).toarray()
y1=train_lab['sentiment']
np.expm1(result )<feature_engineering>
X_train1,X_test1,y_train1,y_test1=train_test_split(X1,y1,test_size=0.2,random_state=0 )
df_test["count"] = np.expm1(result )<filter>
classifier_lr1=LogisticRegression(random_state=0)
 classifier_lr1.fit(X_train1,y_train1)
 y_pred_lr1=classifier_lr1.predict(X_test1 )
df_test[df_test["count"] < 0]<save_to_csv>
report=classification_report(y_pred_lr1,y_test1)
 print('the classification report is
{}'.format(report))
df_test[['datetime','count']].to_csv('predicted_count_log.csv',index=False )<load_from_csv>
corpus_test=[]
for i in range(0,25000):
corpus_test.append(review_word(test['review'][i]))
df = pd.read_csv('/kaggle/input/bike-sharing-demand/train.csv')
 test = pd.read_csv('/kaggle/input/bike-sharing-demand/test.csv')
 df.shape, test.shape<feature_engineering>
test=tv.transform(corpus_test ).toarray()
df['count'] = np.log(df['count'] )<concatenate>
test=classifier_lr.predict(test)
 test=pd.DataFrame(test )
df_total = df.append(test )<data_type_conversions>
submission['sentiment']=test
df_total['datetime'] = pd.to_datetime(df_total['datetime'] )<feature_engineering>
submission.to_csv('Submission.csv',index=False )