Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import gc, math
from pandas.tseries.holiday import USFederalHolidayCalendar as calendar
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold
from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder
%matplotlib inline
def top_3_accuracy(x,y): return top_k_categorical_accuracy(x,y, 3)
 gc.enable()
def get_available_gpus() :
local_device_protos = device_lib.list_local_devices()
return [x.name for x in local_device_protos if x.device_type == 'GPU'] 
<compute_test_metric>
sns.set(rc={'figure.figsize':(11,8)})
 sns.set(style="whitegrid" )
def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<categorify>
%%time
metadata_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')
 train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv', parse_dates=['timestamp'])
 test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv', parse_dates=['timestamp'])
 weather_train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv', parse_dates=['timestamp'])
 weather_test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv', parse_dates=['timestamp'] )
def _stack_it(raw_strokes):

 stroke_vec = literal_eval(raw_strokes)
 in_strokes = [(xi,yi,i)
 for i,(x,y)in enumerate(stroke_vec)
 for xi,yi in zip(x,y)]
c_strokes = np.stack(in_strokes)
 c_strokes[:,2] = [1]+np.diff(c_strokes[:,2] ).tolist()
c_strokes[:,2] += 1
return pad_sequences(c_strokes.swapaxes(0, 1),
maxlen=STROKE_COUNT,
padding='post' ).swapaxes(0, 1)
<define_variables>
weather = pd.concat([weather_train_df,weather_test_df],ignore_index=True)
 weather_key = ['site_id', 'timestamp']
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df
DP_DIR = '.. /input/shuffle-csv-50k'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
<load_from_csv>
weather_train_df = timestamp_align(weather_train_df)
 weather_test_df = timestamp_align(weather_test_df )
def image_generator_xd(batchsize, ks):
while True:
for k in np.random.permutation(ks):
filename = os.path.join(DP_DIR, 'train_k{}.csv.gz.gz'.format(k))
for df in pd.read_csv(filename, chunksize=batchsize):
df['drawing'] = df['drawing'].map(_stack_it)
 x2 = np.stack(df['drawing'], 0)
 y = keras.utils.to_categorical(df.y, num_classes=NCATS)
 yield x2, y
def df_to_image_array_xd(df):
df['drawing'] = df['drawing'].map(_stack_it)
 x2 = np.stack(df['drawing'], 0)
 return x2<define_variables>
del weather
del df_2d
del temp_skeleton
del site_ids_offsets
train_datagen = image_generator_xd(batchsize=batchsize, ks=range(NCSVS - 2))
val_datagen = image_generator_xd(batchsize=batchsize, ks=range(NCSVS - 2, NCSVS))<choose_model_class>
weather_train_df.isna().sum()
stroke_read_model = Sequential()
stroke_read_model.add(BatchNormalization(input_shape =(None,)+(3,)))
 stroke_read_model.add(Conv1D(256,(5,), activation = 'relu'))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Conv1D(256,(5,), activation = 'relu'))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Conv1D(256,(3,), activation = 'relu'))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Bidirectional(LSTM(128, dropout = 0.3, recurrent_dropout= 0.3, return_sequences = True)))
 stroke_read_model.add(Bidirectional(LSTM(128,dropout = 0.3, recurrent_dropout= 0.3, return_sequences = True)))
 stroke_read_model.add(Bidirectional(LSTM(128,dropout = 0.3, recurrent_dropout= 0.3, return_sequences = False)))
 stroke_read_model.add(Dense(512, activation = 'relu'))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Dense(NCATS, activation = 'softmax'))
stroke_read_model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy', top_3_accuracy])
 stroke_read_model.summary()<choose_model_class>
weather_test_df.isna().sum()
weight_path="{}_weights.best.hdf5".format('stroke_lstm_bidirectional_relu')
 checkpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=4,
verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=3)
 callbacks_list = [checkpoint, early, reduceLROnPlat]<train_model>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
hist = stroke_read_model.fit_generator(train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=val_datagen, validation_steps = val_steps,
callbacks = callbacks_list)
 clear_output()<save_to_csv>
weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
hist_df = pd.DataFrame(hist.history)
 hist_df.to_csv('hist_training.csv')
 hist_df.index = np.arange(1, len(hist_df)+1)
 fig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))
axs[0].plot(hist_df.val_top_3_accuracy, lw=5, label='Validation Accuracy')
 axs[0].plot(hist_df.top_3_accuracy, lw=5, label='Training Accuracy')
 axs[0].set_ylabel('Accuracy')
 axs[0].set_xlabel('Epoch')
 axs[0].grid()
axs[0].legend(loc=0)
 axs[1].plot(hist_df.val_loss, lw=5, label='Validation MLogLoss')
 axs[1].plot(hist_df.loss, lw=5, label='Training MLogLoss')
 axs[1].set_ylabel('MLogLoss')
 axs[1].set_xlabel('Epoch')
 axs[1].grid()
axs[1].legend(loc=0)
 fig.savefig('hist.png', dpi=300)
 plt.show() ;<load_from_csv>
weather_train_df.isna().sum()
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 lstm_results = stroke_read_model.evaluate(x_valid, y_valid, batch_size = 4096)
 print('Accuracy: %2.1f%%, Top 3 Accuracy %2.1f%%' %(100*lstm_results[1], 100*lstm_results[2])) 
<load_from_csv>
weather_test_df.isna().sum()
sub_df = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
sub_df['drawing'] = sub_df['drawing'].map(_stack_it)
 sub_vec = np.stack(sub_df['drawing'].values, 0)
 sub_pred = stroke_read_model.predict(sub_vec, verbose=True, batch_size=4096)
<feature_engineering>
train_df['meter_reading'] = np.log1p(train_df['meter_reading'] )
top3 = preds2catids(sub_pred)
 cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 sub_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
sub_df.head()<save_to_csv>
le = LabelEncoder()
metadata_df['primary_use'] = le.fit_transform(metadata_df['primary_use'] )
sub_df[['key_id', 'word']].to_csv('lstm_relu_datagen.csv', index=False )<set_options>
metadata_df = reduce_mem_usage(metadata_df)
 train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df)
 weather_train_df = reduce_mem_usage(weather_train_df)
 weather_test_df = reduce_mem_usage(weather_test_df )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
print(f'Training data shape: {train_df.shape}')
 print(f'Weather training shape: {weather_train_df.shape}')
 print(f'Weather training shape: {weather_test_df.shape}')
 print(f'Weather testing shape: {metadata_df.shape}')
 print(f'Test data shape: {test_df.shape}' )
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
%%time
full_train_df = train_df.merge(metadata_df, on='building_id', how='left')
 full_train_df = full_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left' )
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
full_train_df = full_train_df.loc[~(full_train_df['air_temperature'].isnull() & full_train_df['cloud_coverage'].isnull() & full_train_df['dew_temperature'].isnull() & full_train_df['precip_depth_1_hr'].isnull() & full_train_df['sea_level_pressure'].isnull() & full_train_df['wind_direction'].isnull() & full_train_df['wind_speed'].isnull() & full_train_df['offset'].isnull())]
STEPS = 800
EPOCHS = 18
size = 64
batchsize = 680<choose_model_class>
del train_df
del weather_train_df
gc.collect()
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_from_csv>
%%time
full_test_df = test_df.merge(metadata_df, on='building_id', how='left')
 full_test_df = full_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left' )
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
del metadata_df
del weather_test_df
del test_df
gc.collect()
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<train_model>
print(f'Average meter reading: {full_train_df.meter_reading.mean() } kWh' )
callbacks = [
ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=5,
min_delta=0.005, mode='max', cooldown=3, verbose=1)
 ]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<train_model>
pd.DataFrame(full_train_df.isna().sum().sort_values(ascending=False), columns=['NaN Count'] )
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<train_model>
def mean_without_overflow_fast(col):
col /= len(col)
 return col.mean() * len(col )
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<predict_on_test>
missing_values =(100-full_train_df.count() / len(full_train_df)* 100 ).sort_values(ascending=False )
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
%%time
missing_features = full_train_df.loc[:, missing_values > 0.0]
missing_features = missing_features.apply(mean_without_overflow_fast )
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
for key in full_train_df.loc[:, missing_values > 0.0].keys() :
if key == 'year_built' or key == 'floor_count':
full_train_df[key].fillna(math.floor(missing_features[key]), inplace=True)
 full_test_df[key].fillna(math.floor(missing_features[key]), inplace=True)
 else:
full_train_df[key].fillna(missing_features[key], inplace=True)
 full_test_df[key].fillna(missing_features[key], inplace=True )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
full_train_df["timestamp"] = pd.to_datetime(full_train_df["timestamp"])
 full_test_df["timestamp"] = pd.to_datetime(full_test_df["timestamp"] )
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
def transform(df):
df['hour'] = np.uint8(df['timestamp'].dt.hour)
 df['day'] = np.uint8(df['timestamp'].dt.day)
 df['weekday'] = np.uint8(df['timestamp'].dt.weekday)
 df['month'] = np.uint8(df['timestamp'].dt.month)
 df['year'] = np.uint8(df['timestamp'].dt.year-1900)
 df['square_feet'] = np.log(df['square_feet'])
 return df
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<set_options>
full_train_df = transform(full_train_df)
 full_test_df = transform(full_test_df )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
def encode_cyclic_feature(df, col, max_vals):
df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
 del df[col]
return df
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
dates_range = pd.date_range(start='2015-12-31', end='2019-01-01')
 us_holidays = calendar().holidays(start=dates_range.min() , end=dates_range.max())
 full_train_df['is_holiday'] =(full_train_df['timestamp'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 full_test_df['is_holiday'] =(full_test_df['timestamp'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8 )
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
full_train_df.loc[(full_train_df['weekday'] == 5)|(full_train_df['weekday'] == 6), 'is_holiday'] = 1
full_test_df.loc[(full_test_df['weekday'])== 5 |(full_test_df['weekday'] == 6), 'is_holiday'] = 1
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680<choose_model_class>
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_from_csv>
full_train_df = full_train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
full_test_df = full_test_df.drop(['timestamp'], axis=1)
 full_train_df = full_train_df.drop(['timestamp'], axis=1)
 print(f'Shape of training dataset: {full_train_df.shape}')
 print(f'Shape of testing dataset: {full_test_df.shape}' )
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<prepare_x_and_y>
full_train_df = reduce_mem_usage(full_train_df)
 full_test_df = reduce_mem_usage(full_test_df)
 gc.collect()
%%timeit
x, y = next(train_datagen )<train_on_grid>
beaufort = [(0, 0, 0.3),(1, 0.3, 1.6),(2, 1.6, 3.4),(3, 3.4, 5.5),(4, 5.5, 8),(5, 8, 10.8),(6, 10.8, 13.9), 
(7, 13.9, 17.2),(8, 17.2, 20.8),(9, 20.8, 24.5),(10, 24.5, 28.5),(11, 28.5, 33),(12, 33, 200)]
for item in beaufort:
full_train_df.loc[(full_train_df['wind_speed']>=item[1])&(full_train_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<predict_on_test>
full_train_df['group'] = full_train_df['month']
full_train_df['group'].replace(( 1,2,3,4,5,6), 1,inplace=True)
 full_train_df['group'].replace(( 7,8,9,10,11,12), 2, inplace=True )
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
full_train_df = encode_cyclic_feature(full_train_df, 'weekday', 7)
 full_train_df = encode_cyclic_feature(full_train_df, 'hour', 24)
 full_train_df = encode_cyclic_feature(full_train_df, 'day', 31)
 full_train_df = encode_cyclic_feature(full_train_df, 'month', 12 )
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
full_train_df = reduce_mem_usage(full_train_df )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
categoricals = ['site_id', 'building_id', 'primary_use', 'meter', 'wind_direction', 'is_holiday']
drop_cols = ['sea_level_pressure', 'wind_speed']
numericals = ['square_feet', 'year_built', 'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'floor_count', 'beaufort_scale', 'weekday_sin', 'day_sin', 'hour_sin', 'month_sin']
feat_cols = categoricals + numericals
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
full_train_df = reduce_mem_usage(full_train_df)
 gc.collect()
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<save_to_csv>
target = full_train_df["meter_reading"]
del full_train_df["meter_reading"]
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
full_train_df.drop(drop_cols, axis=1)
 gc.collect()
df_test = pd.read_csv('.. /input/test.csv')
 print('Total number of question pairs for testing: {}'.format(len(df_test)))
<feature_engineering>
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse'},
'subsample': 0.4,
'subsample_freq': 1,
'learning_rate': 0.25,
'num_leaves': 31,
'feature_fraction': 0.8,
'lambda_l1': 1,
'lambda_l2': 1
}
folds = 2
seed = 666
kf = GroupKFold(n_splits=folds)
 models = []
for train_index, val_index in kf.split(full_train_df, full_train_df['building_id'], groups=full_train_df['group']):
train_X = full_train_df[feat_cols].iloc[train_index]
val_X = full_train_df[feat_cols].iloc[val_index]
train_y = target.iloc[train_index]
val_y = target.iloc[val_index]
lgb_train = lgb.Dataset(train_X, train_y, categorical_feature=categoricals)
 lgb_eval = lgb.Dataset(val_X, val_y, categorical_feature=categoricals)
 gbm = lgb.train(params,
lgb_train,
num_boost_round=500,
valid_sets=(lgb_train, lgb_eval),
early_stopping_rounds=100,
verbose_eval = 100)
 models.append(gbm )
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
del full_train_df, train_X, val_X, lgb_train, lgb_eval, train_y, val_y, target
gc.collect()
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
for item in beaufort:
full_test_df.loc[(full_test_df['wind_speed']>=item[1])&(full_test_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
full_test_df = encode_cyclic_feature(full_test_df, 'weekday', 7)
 full_test_df = encode_cyclic_feature(full_test_df, 'hour', 24)
 full_test_df = encode_cyclic_feature(full_test_df, 'day', 31)
 full_test_df = encode_cyclic_feature(full_test_df, 'month', 12 )
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
full_test_df = full_test_df[feat_cols]
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
full_test_df = reduce_mem_usage(full_test_df )
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values<concatenate>
i=0
res=[]
step_size = 50000
for j in tqdm(range(int(np.ceil(full_test_df.shape[0]/50000)))) :
res.append(np.expm1(sum([model.predict(full_test_df.iloc[i:i+step_size])for model in models])/folds))
i+=step_size
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
res = np.concatenate(res )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = res
submission.loc[submission['meter_reading']<0, 'meter_reading'] = 0
submission.to_csv('submission_fe_lgbm.csv', index=False)
 submission