Kaggle competition: Bag of Words Meets Bags of Popcorn
sub.to_csv('submission.csv', index=False )<save_to_csv>
import json
import re
import nltk
from bs4 import BeautifulSoup
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
from tensorflow.python.keras.preprocessing.text import Tokenizer
import zipfile
sub.to_csv('submission.csv', index=False )<feature_engineering>
%matplotlib inline
train = pd.read_csv(".. /input/train.csv", parse_dates = ["datetime"])
 test = pd.read_csv(".. /input/test.csv", parse_dates = ["datetime"])
 train["year"] = train["datetime"].dt.year
train["hour"] = train["datetime"].dt.hour
train["dayofweek"] = train["datetime"].dt.dayofweek
test["year"] = test["datetime"].dt.year
test["hour"] = test["datetime"].dt.hour
test["dayofweek"] = test["datetime"].dt.dayofweek<count_values>
nltk.download('stopwords')
train['temp'].value_counts().sort_index()<prepare_x_and_y>
raw_train_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip", delimiter='\t')
 raw_train_data.head()
y_casual = np.log1p(train.casual)
 y_registered = np.log1p(train.registered)
 train.drop(["datetime", "windspeed", "casual", "registered", "count"], 1, inplace=True)
 test.drop(["datetime", "windspeed", ], 1, inplace=True )<train_model>
raw_train_len = raw_train_data["review"].apply(len)
 raw_train_len
hyperparameters = { 'colsample_bytree': 0.725, 'learning_rate': 0.013,
'num_leaves': 56, 'reg_alpha': 0.754, 'reg_lambda': 0.071,
'subsample': 0.523, 'n_estimators': 1093}
model = lgb.LGBMRegressor(**hyperparameters)
 model.fit(train, y_casual)
 preds1 = model.predict(test)
 hyperparameters = { 'colsample_bytree': 0.639, 'learning_rate': 0.011,
'num_leaves': 30, 'reg_alpha': 0.351, 'reg_lambda': 0.587,
'subsample': 0.916, 'n_estimators': 2166}
model = lgb.LGBMRegressor(**hyperparameters,)
 model.fit(train, y_registered)
 preds2 = model.predict(test)
 submission=pd.read_csv(".. /input/sampleSubmission.csv")
 submission["count"] = np.expm1(preds1)+ np.expm1(preds2)
<prepare_x_and_y>
print('number of + : {}'.format(raw_train_data['sentiment'].value_counts() [1]))
print('number of - : {}'.format(raw_train_data['sentiment'].value_counts() [0]))
submission.iloc[1258:1269, 1]= submission.iloc[1258:1269, 1]*0.5
submission.iloc[4492:4515, 1]= submission.iloc[4492:4515, 1]*0.5
submission.iloc[6308:6330, 1]= submission.iloc[6308:6330, 1]*0.5
submission.iloc[3041:3063, 1]= submission.iloc[3041:3063, 1]*0.5
submission.iloc[6332:6354, 1]= submission.iloc[6332:6354, 1]*0.5
submission.iloc[3065:3087, 1]= submission.iloc[3065:3087, 1]*0.5
submission.iloc[5992:6015, 1]= submission.iloc[5992:6015, 1]*0.5
submission.iloc[2771:2794, 1]= submission.iloc[2771:2794, 1]*0.5<filter>
train_word_cnt = raw_train_data['review'].apply(lambda x:len(x.split(' ')))
 train_word_cnt
submission.iloc[6332:6354]<save_to_csv>
r_qmarks = np.mean(raw_train_data['review'].apply(lambda x: '?' in x))
r_fullstops = np.mean(raw_train_data['review'].apply(lambda x: '.' in x))
r_capitals = np.mean(raw_train_data['review'].apply(lambda x: max(y.isupper() for y in x)))
 r_numbers = np.mean(raw_train_data['review'].apply(lambda x: max(y.isdigit() for y in x)))
 print('물음표가 있는 리뷰 : {:.2f}%'.format(r_qmarks * 100))
print('마침표가 있는 리뷰 : {:.2f}%'.format(r_fullstops * 100))
print('대문자가 있는 리뷰 : {:.2f}%'.format(r_capitals * 100))
print('숫자가 있는 리뷰 : {:.2f}%'.format(r_numbers * 100))
submission.to_csv("lgb.csv", index=False )<load_from_csv>
review = raw_train_data['review'][0]
review_text = BeautifulSoup(review,"html5lib" ).get_text()
review_text = re.sub("[^a-zA-Z]"," ",review_text )
train = pd.read_csv(".. /input/bike-sharing-demand/train.csv")
 test = pd.read_csv(".. /input/bike-sharing-demand/test.csv")
 train.head()<feature_engineering>
stop_words = set(stopwords.words('english'))
review_text = review_text.lower()
words = review_text.split()
words = [w for w in words if not w in stop_words]
for df in [train, test]:
df["datetime"] = pd.DatetimeIndex(df["datetime"])
 df["hour"] = [x.hour for x in df["datetime"]]
df["weekday"] = [x.dayofweek for x in df["datetime"]]
df["month"] = [x.month for x in df["datetime"]]
df["year"] = [x.year for x in df["datetime"]]
df['year_season'] = df['year'].astype(str)+ "_" + df['season'].astype(str)
 df["year"] = df["year"].map({2011:1, 2012:0})
 df.drop('datetime',axis=1,inplace=True )<feature_engineering>
clean_review = ' '.join(words)
 print(clean_review )
df_list = {"train":None, "test" : None}
for name, df in zip(df_list.keys() ,[train, test]):
df['windspeed'] = np.log(df['windspeed']+1)
 df["weekday_working"] = df["weekday"]*df["workingday"]
df["weekday_holiday"] = df["weekday"]*df["holiday"]
df['casual_workhour'] = df[['hour', 'workingday']].apply(lambda x: int(x['workingday'] == 0 and 10 <= x['hour'] <= 19), axis=1)
 df['casual_holi_hour'] = df[['hour', 'holiday']].apply(lambda x: int(x['holiday'] == 1 and 9 <= x['hour'] <= 22), axis=1)
 df['register_workhour'] = df[['hour', 'workingday']].apply(
 lambda x:int(( x['workingday'] == 1 and(6 <= x['hour'] <= 8 or 17 <= x['hour'] <= 20))
or(x['workingday'] == 0 and 10 <= x['hour'] <= 15)) , axis=1)
 df['register_holi_hour'] = df[['hour', 'holiday']].apply(
 lambda x:int(x['holiday'] == 0 and(7 <= x['hour'] <= 8 or 17 <= x['hour'] <= 18)) , axis=1)
 df.drop('atemp',axis=1,inplace=True)
 by_season = train.groupby('year_season')[['count']].median()
by_season.columns = ['count_season']
train1 = train.join(by_season, on='year_season' ).drop('year_season',axis=1)
 test1 = test.join(by_season, on='year_season' ).drop('year_season',axis=1 )<prepare_x_and_y>
def preprocess(review,remove_stopwords = False):
review_text = BeautifulSoup(review,"html5lib" ).get_text()
review_text = re.sub("[^a-zA-Z]"," ",review_text)
 words = review_text.lower().split()
if remove_stopwords:
stop_words = set(stopwords.words('english'))
words = [w for w in words if not w in stop_words]
clean_review = ' '.join(words)
 return clean_review
y_list = ["casual","registered","count"]
train_x = train1[[col for col in train1.columns if col not in ['casual','registered', 'count']]]
train_y = np.log(train1[y_list]+1 )<split>
clean_train_reviews = []
for review in raw_train_data['review']:
clean_train_reviews.append(preprocess(review,remove_stopwords = True))
folds = KFold(n_splits = 5, shuffle = True, random_state = 123)
 rms1,rms2 = [],[]
models1,models2 = [], []
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train1)) :
x_train, y_train = train_x.ix[trn_idx], train_y.ix[trn_idx]
x_val, y_val = train_x.ix[val_idx], train_y.ix[val_idx]
lgb_param = {'boosting_type':'gbdt',
'num_leaves': 45,
'max_depth': 30,
'learning_rate': 0.01,
'bagging_fraction' : 0.9,
'bagging_freq': 20,
'colsample_bytree': 0.9,
'metric': 'rmse',
'min_child_weight': 1,
'min_child_samples': 10,
'zero_as_missing': True,
'objective': 'regression',
}
train_set1 = lgb.Dataset(x_train, y_train["registered"], silent=False)
 valid_set1 = lgb.Dataset(x_val, y_val["registered"], silent=False)
 lgb_model1 = lgb.train(params = lgb_param, train_set = train_set1 , num_boost_round=5000, early_stopping_rounds=100,verbose_eval=500, valid_sets=valid_set1)
 train_set2 = lgb.Dataset(x_train, y_train["casual"], silent=False)
 valid_set2 = lgb.Dataset(x_val, y_val["casual"], silent=False)
 lgb_model2 = lgb.train(params = lgb_param, train_set = train_set2 , num_boost_round=5000, early_stopping_rounds=100,verbose_eval=500, valid_sets=valid_set2)
 models1.append(lgb_model1)
 models2.append(lgb_model2 )<predict_on_test>
clean_train_df = pd.DataFrame({'id':raw_train_data['id'], 'review':clean_train_reviews, 'sentiment':raw_train_data['sentiment']} )
preds = []
for model in models1:
regi_pred = model.predict(test1)
 preds.append(regi_pred)
 fin_casual = np.mean(preds, axis=0)
 preds = []
for model in models2:
casual_pred = model.predict(test1)
 preds.append(casual_pred)
 fin_regi = np.mean(preds, axis=0)
 count_pred1 = np.exp(fin_casual)+ np.exp(fin_regi)- 2<split>
tokenizer = Tokenizer(oov_token='<UNK>')
 tokenizer.fit_on_texts(clean_train_reviews)
 text_sequences = tokenizer.texts_to_sequences(clean_train_reviews )
def lgb_cv(num_leaves, learning_rate, n_estimators, reg_alpha, reg_lambda, min_split_gain, min_child_weight,min_child_samples, colsample_bytree, x_data=None, y_data=None, n_splits=5, output='score'):
score = 0
kf = KFold(n_splits=n_splits)
 models = []
for train_index, valid_index in kf.split(x_data):
x_train, y_train = x_data.iloc[train_index], y_data[train_index]
x_valid, y_valid = x_data.iloc[valid_index], y_data[valid_index]
model = lgb.LGBMRegressor(
 num_leaves = int(num_leaves),
learning_rate = learning_rate,
n_estimators = int(n_estimators),
reg_alpha = reg_alpha,
reg_lambda = reg_lambda,
min_split_gain= min_split_gain,
min_child_weight = min_child_weight,
min_child_samples = int(min_child_samples),
colsample_bytree = np.clip(colsample_bytree, 0, 1), 
)
 model.fit(x_train, y_train)
 models.append(model)
 pred = model.predict(x_valid)
 true = y_valid
score -= mean_squared_error(true, pred)/n_splits
if output == 'score':
return score
if output == 'model':
return models<categorify>
word_vocab = tokenizer.word_index
print(word_vocab)
func_fixed1 = partial(lgb_cv, x_data=train_x, y_data=train_y["casual"], n_splits=5, output='score')
 func_fixed2 = partial(lgb_cv, x_data=train_x, y_data=train_y["registered"], n_splits=5, output='score')
 lgbBO = BayesianOptimization(
 func_fixed1,
{
'num_leaves':(30, 100),
'learning_rate':(0.001, 0.015),
'n_estimators':(1000, 3000),
'reg_alpha':(0.0001, 1),
'reg_lambda':(0.0001, 1),
'min_split_gain' :(0.001, 0.1),
'min_child_weight' :(0.001, 0.1),
'min_child_samples' :(10,25),
'colsample_bytree':(0.85, 1.0),
},
random_state=4321 
)
 lgbBO.maximize(init_points=5, n_iter=20)
 lgbB1 = BayesianOptimization(
 func_fixed2,
{
'num_leaves':(30, 100),
'learning_rate':(0.001, 0.015),
'n_estimators':(1000, 3000),
'reg_alpha':(0.0001, 1),
'reg_lambda':(0.0001, 1),
'min_split_gain' :(0.001, 0.1),
'min_child_weight' :(0.001, 0.1),
'min_child_samples' :(10,25),
'colsample_bytree':(0.85, 1.0),
},
random_state=4321 
)
 lgbB1.maximize(init_points=5, n_iter=20 )<train_on_grid>
data_configs = {}
data_configs['vocab'] = word_vocab
data_configs['vocab_size'] = len(word_vocab)+ 1
params1 = lgbBO.max['params']
params2 = lgbB1.max['params']
lgb_models1 = lgb_cv(
 params1['num_leaves'],
params1['learning_rate'],
params1['n_estimators'],
params1['reg_alpha'],
params1['reg_lambda'],
params1['min_split_gain'],
params1['min_child_weight'],
params1['min_child_samples'],
params1['colsample_bytree'],
x_data=train_x, y_data=train_y["casual"], n_splits=5, output='model')
 lgb_models2 = lgb_cv(
 params2['num_leaves'],
params2['learning_rate'],
params2['n_estimators'],
params2['reg_alpha'],
params2['reg_lambda'],
params2['min_split_gain'],
params2['min_child_weight'],
params2['min_child_samples'],
params2['colsample_bytree'],
x_data=train_x, y_data=train_y["registered"], n_splits=5, output='model')
 preds = []
for model in lgb_models1:
pred = model.predict(test1)
 preds.append(pred)
 casual_pred = np.mean(preds, axis=0)
 preds = []
for model in lgb_models2:
pred = model.predict(test1)
 preds.append(pred)
 registered_pred = np.mean(preds, axis=0)
 count_pred2 = np.exp(casual_pred)+ np.exp(registered_pred)- 2<train_model>
MAX_SEQUENCE_LENGTH = 174
train_inputs = pad_sequences(text_sequences,maxlen=MAX_SEQUENCE_LENGTH, padding='post')
 print('shape of train data: ', train_inputs.shape )
preds = {}
regs = {"gbdt": GradientBoostingRegressor(random_state=0),
"rf": RandomForestRegressor(random_state=0, n_jobs=-1)}
for name, reg in regs.items() :
if name == 'gbdt':
reg.set_params(n_estimators=1500, min_samples_leaf=6)
 elif name == 'rf':
reg.set_params(n_estimators=1500, min_samples_leaf=2)
 reg.fit(train_x, train_y['casual'])
 pred_casual = reg.predict(test1)
 pred_casual = np.exp(pred_casual)- 1
pred_casual[pred_casual < 0] = 0
if name == 'gbdt':
reg.set_params(n_estimators=1500, min_samples_leaf=6)
 elif name == 'rf':
reg.set_params(n_estimators=1500, min_samples_leaf=2)
 reg.fit(train_x, train_y['registered'])
 pred_registered = reg.predict(test1)
 pred_registered = np.exp(pred_registered)- 1
pred_registered[pred_registered < 0] = 0
preds[name] = pred_casual + pred_registered<save_to_csv>
DATA_IN_PATH = './data_in/'
TRAIN_INPUT_DATA = 'train_input.npy'
TRAIN_LABEL_DATA = 'train_label.npy'
TRAIN_CLEAN_DATA = 'train_clean.csv'
DATA_CONFIGS = 'data_configs.json'
if not os.path.exists(DATA_IN_PATH):
os.makedirs(DATA_IN_PATH )
pred_mean =(count_pred1 + count_pred2 + preds['gbdt'] + preds['rf'])/4
sample = pd.read_csv(".. /input/bike-sharing-demand/sampleSubmission.csv")
 sample["count"] = pred_mean
sample.to_csv("sample.csv",index=False )<load_from_csv>
np.save(open(DATA_IN_PATH + TRAIN_INPUT_DATA, 'wb'), train_inputs)
 np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA, 'wb'), train_labels)
 clean_train_df.to_csv(DATA_IN_PATH + TRAIN_CLEAN_DATA, index = False)
 json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'), ensure_ascii=False )
!pip install bs4
train = pd.read_csv(".. /input/word2vec-nlp-tutorial/labeledTrainData.tsv.zip", header=0, \
delimiter="\t", quoting=3)
 train = train.drop(['id'], axis=1)
 train.head()<load_from_csv>
test_data = pd.read_csv(".. /input/word2vec-nlp-tutorial/testData.tsv.zip", delimiter='\t')
 test_data.head()
df2 = pd.read_csv('.. /input/imdb-review-dataset/imdb_master.csv',encoding="latin-1")
<drop_column>
clean_test_reviews = []
for review in test_data['review']:
clean_test_reviews.append(preprocess(review, remove_stopwords = True))
clean_test_df = pd.DataFrame({'review': clean_test_reviews, 'id': test_data['id']})
 test_id = np.array(test_data['id'])
 text_sequences = tokenizer.texts_to_sequences(clean_test_reviews)
 test_inputs = pad_sequences(text_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post' )
df2 = df2.drop(['Unnamed: 0','type','file'],axis=1)
 df2.columns = ["review","sentiment"]
df2.head()<categorify>
TEST_INPUT_DATA = 'test_input.npy'
TEST_CLEAN_DATA = 'test_clean.csv'
TEST_ID_DATA = 'test_id.npy'
np.save(open(DATA_IN_PATH + TEST_INPUT_DATA, 'wb'), test_inputs)
 np.save(open(DATA_IN_PATH + TEST_ID_DATA, 'wb'), test_id)
 clean_test_df.to_csv(DATA_IN_PATH + TEST_CLEAN_DATA, index = False )
df2 = df2[df2.sentiment != 'unsup']
df2['sentiment'] = df2['sentiment'].map({'pos': 1, 'neg': 0})
 df2.head()<concatenate>
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, GlobalAveragePooling1D, Dense, LSTM, Dropout, Bidirectional
training = pd.concat([train, df2] ).reset_index(drop=True)
 training.shape<categorify>
vocab_size = len(word_vocab)+1
vocab_size
def review_to_words(raw_review):
review_text = BeautifulSoup(raw_review, ).get_text()
letters_only = re.sub("[^a-zA-Z]", " ", review_text)
 words = letters_only.lower().split()
stops = set(stopwords.words("english"))
meaningful_words = [w for w in words if not w in stops]
return(" ".join(meaningful_words))<define_variables>
def small_model() :
model = Sequential()
model.add(Embedding(vocab_size, 16))
model.add(GlobalAveragePooling1D())
 model.add(Dense(16, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 return model
num_reviews = training["review"].size
clean_train_reviews = []
print("Cleaning and parsing the training set movie reviews...
 ")
 clean_train_reviews = []
for i in range(0, num_reviews):
if(( i+1)%10000 == 0):
print("Review %d of %d
" %(i+1, num_reviews))
clean_train_reviews.append(review_to_words(training["review"][i]))
print("Done" )<feature_engineering>
model = small_model()
print("Creating the bag of words...
 ")
 vectorizer = CountVectorizer(analyzer = "word", \
tokenizer = None, \
preprocessor = None, \
stop_words = None, \
max_features = 6000)
 train_data_features = vectorizer.fit_transform(clean_train_reviews)
 train_data_features = train_data_features.toarray()
print("Done")
<split>
history = model.fit(train_inputs,
train_labels,
epochs=10,
batch_size=256,
validation_split = 0.3 )
x_train,x_test,y_train,y_test=train_test_split(train_data_features,training["sentiment"],test_size=0.2,random_state=0)
 print("Splitting Done" )<train_model>
del model
del history
print("Training the random forest model...")
 forest = RandomForestClassifier(n_estimators = 100)
 forest = forest.fit(x_train, y_train)
 print("Done" )<predict_on_test>
def big_model() :
model = Sequential()
model.add(Embedding(vocab_size, 16))
model.add(Bidirectional(LSTM(64, recurrent_dropout=0.1)))
 model.add(Dropout(0.5))
model.add(Dense(32, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(1, activation = 'sigmoid'))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 return model
result = forest.predict(x_test)
 print("Predictions Done" )<compute_test_metric>
model = big_model()
accuracy=accuracy_score(y_test,result)
 accuracy<compute_test_metric>
history = model.fit(train_inputs,
train_labels,
epochs=10,
batch_size=256,
validation_split = 0.3 )
cm=confusion_matrix(y_test,result)
 cm<compute_test_metric>
model.save('my_model.h5' )
f1=f1_score(y_test,result)
 f1<load_from_csv>
%%time
test_label = model.predict_classes(test_inputs )
<load_pretrained><EOS> 
def submit(predictions):
test_data['sentiment'] = predictions
test_data.to_csv('submission.csv', index=False, columns=['id','sentiment'])
 submit(test_label )