Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<categorify>
folds = 3
debug = False
num_rounds = 1000
en_model = spacy.load('en', disable=['parser', 'ner'])
 def lemmatization(texts):
output = []
for i in texts:
s = [token.lemma_ for token in en_model(i)]
output.append(' '.join(s))
return output<categorify>
py.init_notebook_mode(connected=True)
train['lemmatized_text'] = lemmatization(train['stopwords_cleaned'])
 test['lemmatized_text'] = lemmatization(test['stopwords_cleaned'] )<split>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
x_train, x_test, y_train, y_test = train_test_split(train['stopwords_cleaned'], train['target'],
test_size = 0.2, random_state = 1 )<choose_model_class>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
hub_layer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',
input_shape = [],
output_shape = [512],
dtype = tf.string,
trainable = True)
 model = tf.keras.models.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation = 'relu'))
model.add(tf.keras.layers.Dense(32, activation = 'relu'))
model.add(tf.keras.layers.Dense(1, activation = 'sigmoid'))
model.summary()<choose_model_class>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics = ['accuracy'] )<train_model>
building_meta_df[building_meta_df.site_id == 0]
model.fit(x_train,
y_train,
epochs = 1,
validation_data =(x_test, y_test))<predict_on_test>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
pred = model.predict_classes(test['stopwords_cleaned'] )<save_to_csv>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )<save_to_csv>
set_localtime(weather_train_df )
submission.to_csv("submission.csv", index = False )<import_modules>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import string
import re
import nltk
from nltk.corpus import stopwords
from nltk import FreqDist, word_tokenize
from nltk.stem import WordNetLemmatizer,PorterStemmer
from sklearn import model_selection
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.linear_model import LogisticRegression<load_from_csv>
preprocess(train_df )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 sub_sample = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 train.head()<count_values>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group
train['target'].value_counts()<feature_engineering>
weather_train_df.isna().sum()
temp = train.groupby('target' ).count() ['text'].reset_index()
temp['label'] = temp['target'].apply(lambda x : 'Disaster Tweet' if x==1 else 'Non Disaster Tweet')
 temp<count_missing_values>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
train.isnull().sum()<count_missing_values>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
train.isnull().sum()<count_missing_values>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
test.isnull().sum()<count_missing_values>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
test.isnull().sum()<count_unique_values>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
train.keyword.nunique() ,test.keyword.nunique()<count_unique_values>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed',]
train.location.nunique() ,test.location.nunique()<categorify>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
train['location'].replace({'United States':'USA',
'New York':'USA',
"London":'UK',
"Los Angeles, CA":'USA',
"Washington, D.C.":'USA',
"California":'USA',
"Chicago, IL":'USA',
"Chicago":'USA',
"New York, NY":'USA',
"California, USA":'USA',
"FLorida":'USA',
"Nigeria":'Africa',
"Kenya":'Africa',
"Everywhere":'Worldwide',
"San Francisco":'USA',
"Florida":'USA',
"United Kingdom":'UK',
"Los Angeles":'USA',
"Toronto":'Canada',
"San Francisco, CA":'USA',
"NYC":'USA',
"Seattle":'USA',
"Earth":'Worldwide',
"Ireland":'UK',
"London, England":'UK',
"New York City":'USA',
"Texas":'USA',
"London, UK":'UK',
"Atlanta, GA":'USA',
"Mumbai":"India"},inplace=True)
 sns.barplot(y=train['location'].value_counts() [:5].index,x=train['location'].value_counts() [:5] )<data_type_conversions>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
for col in ['keyword', 'location']:
train[col] = train[col].fillna(f'no_{col}')
 for col in ['keyword', 'location']:
test[col] = test[col].fillna(f'no_{col}' )<feature_engineering>
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 text = re.sub(r'<.*?>',' ' ,text)
 return text
train['text'] = train['text'].apply(lambda x: clean_text(x))
test['text'] = test['text'].apply(lambda x: clean_text(x))
train['text'].head()<feature_engineering>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()
tokenizer = nltk.tokenize.RegexpTokenizer(r'\w+')
 train['text'] = train['text'].apply(lambda x: tokenizer.tokenize(x))
test['text'] = test['text'].apply(lambda x: tokenizer.tokenize(x))
train['text'].head() 
<feature_engineering>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()
def remove_stopwords(text):

 words = [w for w in text if w not in stopwords.words('english')]
return words
train['text'] = train['text'].apply(lambda x : remove_stopwords(x))
test['text'] = test['text'].apply(lambda x : remove_stopwords(x))<drop_column>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )<feature_engineering>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()
def stemming(words):
ps=PorterStemmer()
return [ps.stem(word)for word in words]
train['text']=train['text'].apply(lambda x: stemming(x))
test['text']=test['text'].apply(lambda x: stemming(x))<categorify>
oof_total = oof_total / len(train_df )
def lemmatizing(words):
lemmatizer =WordNetLemmatizer()
return [lemmatizer.lemmatize(word)for word in words]
train['text']=train['text'].apply(lambda x: lemmatizing(x))
test['text']=test['text'].apply(lambda x: lemmatizing(x))<feature_engineering>
del train_df, weather_train_df, building_meta_df
gc.collect()
def final_text(words):
return ' '.join(words)
 train['text']=train['text'].apply(lambda x:final_text(x))
test['text']=test['text'].apply(lambda x:final_text(x))
train.head(10 )<feature_engineering>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
count_vectorizer = CountVectorizer()
train_vectors = count_vectorizer.fit_transform(train['text'])
 test_vectors = count_vectorizer.transform(test["text"])
<feature_engineering>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
sample_submission = reduce_mem_usage(sample_submission )
tfidf = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, 2))
train_vectors = tfidf.fit_transform(train['text'])
 test_vectors = tfidf.transform(test["text"] )<compute_train_metric>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
clf = LogisticRegression(C=0.9,max_iter=1000,penalty='l2')
 scores = model_selection.cross_val_score(clf, train_vectors, train["target"], cv=7, scoring="f1")
 scores<train_model>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
clf.fit(train_vectors, train["target"] )<save_to_csv>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 sample_submission.to_csv("submission.csv", index=False )<import_modules>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )