Kaggle competition: ASHRAE - Great Energy Predictor III
cols3 = []
FE = ['Census_OSVersion', 'Census_OSBuildRevision', 'Census_InternalBatteryNumberOfCharges', 'AvSigVersion', 'Lag1']
for col in FE:
cols3 += encode_FE(df_train, col)
 encode_FE(df_test, col)
 FE2 = ['CountryIdentifier', 'Census_InternalBatteryNumberOfCharges']
for col in FE2:
cols3 += encode_FE2(df_train, df_test, col )<define_variables>
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'
CE = ['CountryIdentifier', 'SkuEdition', 'Firewall', 'Census_ProcessorCoreCount', 'Census_OSUILocaleIdentifier', 'Census_FlightRing']<drop_column>
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
cols = [x for x in df_train.columns if x not in ['HasDetections']+CE+cols3+cols6+cols8]
cols2 = CE; ct = 1
for col in cols.copy() :
rate = df_train[col].value_counts(normalize=True, dropna=False ).values[0]
if rate > 0.98:
del df_train[col]
del df_test[col]
cols.remove(col)
 ct += 1
rmv3=['Census_OSSkuName', 'OsVer', 'Census_OSArchitecture', 'Census_OSInstallLanguageIdentifier']
rmv4=['SMode']
for col in rmv3+rmv4:
del df_train[col]
del df_test[col]
cols.remove(col)
 ct +=1
print('Removed',ct,'variables')
 x=gc.collect()<categorify>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
print('Factorizing...')
 for col in cols+cols2+cols6:
factor_data(df_train, df_test, col)
 print('Relaxing data...')
 for col in cols+cols2: relax_data(df_train, df_test, col)
 print('Optimizing memory...')
 for col in cols+cols2+cols6:
reduce_memory(df_train, col)
 reduce_memory(df_test, col)
 categorize(df_train, df_test, cols2)
 print('Number of variables is',len(cols+cols2+cols3+cols6+cols8))
display_memory(df_train, df_test )<choose_model_class>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
pred_val = np.zeros(len(df_test))
folds = StratifiedKFold(n_splits=5, shuffle=True)
 ct = 0
for idxT, idxV in folds.split(df_train[cols+cols2+cols3+cols6], df_train['HasDetections']):
ct += 1; print('
df_trainA = df_train.loc[idxT]
df_trainB = df_train.loc[idxV]
model = lgb.LGBMClassifier(n_estimators=10000, colsample_bytree=0.5, objective='binary', num_leaves=2048,
max_depth=-1, learning_rate=0.04)
 h=model.fit(df_trainA[cols+cols2+cols3+cols6+cols8], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols+cols2+cols3+cols6+cols8], df_trainB['HasDetections'])], verbose=200,
early_stopping_rounds=100)
 del df_trainA, df_trainB; x=gc.collect()
idx = 0; ct2 = 1; chunk = 1000000
print('Predicting test...')
 while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] += model.predict_proba(df_test.iloc[idx][cols+cols2+cols3+cols6+cols8])[:,1]
ct2 += 1; idx = idx2<feature_engineering>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 root_median = Path('.. /input/site-meter-median-bland-by-leak')
 df_groupNum_median = pd.read_pickle(root_median/'df_groupNum_median.pickle')
del df_train; x=gc.collect()
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',
usecols=['MachineIdentifier','AvSigVersion'], nrows=len(pred_val))
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_test['HasDetections'] = pred_val / 5.0
df_test['X'] = df_test['Date'] - datetime(2018,11,20,4,0)
 df_test['X'] = df_test['X'].map(lambda x: x.total_seconds() /86400)
 df_test['X'].fillna(0,inplace=True)
 s = 5.813888
df_test['F'] = 1.0
df_test['F'] = 1 - df_test['X']/s
df_test.loc[df_test['X']<=0,'F'] = 1.0
df_test.loc[df_test['X']>s,'F'] = 0
df_test['HasDetections'] *= df_test['F']<save_to_csv>
building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates() , on='building_id' )
df_test[['MachineIdentifier','HasDetections']].to_csv('submission.csv', index=False )<install_modules>
building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int' )
!git clone https://github.com/guoday/ctrNet-tool.git
!cp -r ctrNet-tool/*./
!rm -r ctrNet-tool data.git
!ls -all<import_modules>
train_df = train_df [ train_df['building_id'] != 1099 ]
import ctrNet
import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from src import misc_utils as utils
import os
import gc
import random<define_variables>
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
print('Loading Train and Test Data.
 ')
 train = pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 test['HasDetections']=[0]*len(test )<split>
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )
def make_bucket(data,num=10):
data.sort()
bins=[]
for i in range(num):
bins.append(data[int(len(data)*(i+1)//num)-1])
 return bins
float_features=['Census_SystemVolumeTotalCapacity','Census_PrimaryDiskTotalCapacity']
for f in float_features:
train[f]=train[f].fillna(1e10)
 test[f]=test[f].fillna(1e10)
 data=list(train[f])+list(test[f])
 bins=make_bucket(data,num=50)
 train[f]=np.digitize(train[f],bins=bins)
 test[f]=np.digitize(test[f],bins=bins)
 train, dev,_,_ = train_test_split(train,train['HasDetections'],test_size=0.02, random_state=2019)
 features=train.columns.tolist() [1:-1]<choose_model_class>
set_local(weather_train_df)
 add_holiyday(weather_train_df )
hparam=tf.contrib.training.HParams(
 model='fm',
norm=True,
batch_norm_decay=0.9,
hidden_size=[128,128],
k=16,
hash_ids=int(2e5),
batch_size=1024,
optimizer="adam",
learning_rate=0.0008,
num_display_steps=1000,
num_eval_steps=1000,
epoch=1,
metric='auc',
init_method='uniform',
init_value=0.1,
feature_nums=len(features),
kfold=10)
 utils.print_hparams(hparam )<randomize_order>
train_df_black = train_df.copy()
train_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')
 train_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df_black['black_count']=0
for bid in train_df_black.building_id.unique() :
df = train_df_black[train_df_black.building_id==bid]
for meter in df.meter.unique() :
dfm = df[df.meter == meter]
b =(dfm.meter_reading == 0 ).astype(int)
 train_df_black.loc[(train_df_black.building_id==bid)&(train_df_black.meter == meter), 'black_count'] = b.groupby(( ~b.astype(bool)).cumsum() ).cumsum()
index=set(range(train.shape[0]))
K_fold=[]
for i in range(hparam.kfold):
if i == hparam.kfold-1:
tmp=index
else:
tmp=random.sample(index,int(1.0/hparam.kfold*train.shape[0]))
index=index-set(tmp)
 print("Number:",len(tmp))
K_fold.append(tmp)
 for i in range(hparam.kfold):
print("Fold",i)
 dev_index=K_fold[i]
dev_index=random.sample(dev_index,int(0.1*len(dev_index)))
 train_index=[]
for j in range(hparam.kfold):
if j!=i:
train_index+=K_fold[j]
model=ctrNet.build_model(hparam)
 model.train(train_data=(train.iloc[train_index][features],train.iloc[train_index]['HasDetections']),\
dev_data=(train.iloc[dev_index][features],train.iloc[dev_index]['HasDetections']))
print("Training Done! Inference...")
 if i==0:
preds=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold
else:
preds+=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold<save_to_csv>
train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])
 train_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1 )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] = preds
print(submission['HasDetections'].head())
 submission.to_csv('nffm_submission.csv', index=False )<install_modules>
del train_df_black
gc.collect()
!git clone https://github.com/guoday/ctrNet-tool.git
!cp -r ctrNet-tool/*./
!rm -r ctrNet-tool data.git
!ls -all<define_variables>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
random.seed(2019)
 np.random.seed(2019 )<define_variables>
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
print('Loading Train and Test Data.
 ')
 train = pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 test['HasDetections']=[0]*len(test )<categorify>
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )
def make_bucket(data,num=10):
data.sort()
bins=[]
for i in range(num):
bins.append(data[int(len(data)*(i+1)//num)-1])
 return bins
float_features=['Census_SystemVolumeTotalCapacity','Census_PrimaryDiskTotalCapacity']
for f in float_features:
train[f]=train[f].fillna(1e10)
 test[f]=test[f].fillna(1e10)
 data=list(train[f])+list(test[f])
 bins=make_bucket(data,num=50)
 train[f]=np.digitize(train[f],bins=bins)
 test[f]=np.digitize(test[f],bins=bins)
 features=train.columns.tolist() [1:-1]<choose_model_class>
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()
hparam=tf.contrib.training.HParams(
 model='nffm',
norm=True,
batch_norm_decay=0.9,
hidden_size=[128,128],
k=8,
hash_ids=int(2e5),
batch_size=1024,
optimizer="adam",
learning_rate=0.001,
num_display_steps=1000,
num_eval_steps=1000,
epoch=1,
metric='auc',
init_method='uniform',
init_value=0.1,
feature_nums=len(features),
kfold=5)
 utils.print_hparams(hparam )<randomize_order>
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931
index=set(range(train.shape[0]))
K_fold=[]
for i in range(hparam.kfold):
if i == hparam.kfold-1:
tmp=index
else:
tmp=random.sample(index,int(1.0/hparam.kfold*train.shape[0]))
index=index-set(tmp)
 print("Number:",len(tmp))
K_fold.append(tmp)
 for i in range(hparam.kfold):
print("Fold",i)
 dev_index=K_fold[i]
dev_index=random.sample(dev_index,int(0.1*len(dev_index)))
 train_index=[]
for j in range(hparam.kfold):
if j!=i:
train_index+=K_fold[j]
model=ctrNet.build_model(hparam)
 model.train(train_data=(train.iloc[train_index][features],train.iloc[train_index]['HasDetections']),\
dev_data=(train.iloc[dev_index][features],train.iloc[dev_index]['HasDetections']))
print("Training Done! Inference...")
 if i==0:
preds=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold
else:
preds+=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold<save_to_csv>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] = preds
print(submission['HasDetections'].head())
 submission.to_csv('nffm_submission.csv', index=False )<import_modules>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["day"] = df["timestamp"].dt.day
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
import pandas as pd
import numpy as np
import xgboost as xgb<load_from_csv>
preprocess(train_df )
train = pd.read_csv(".. /input/train.csv", parse_dates=['Original_Quote_Date'])
 test = pd.read_csv(".. /input/test.csv", parse_dates=['Original_Quote_Date'] )<prepare_x_and_y>
if use_ucf:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
x_cols = train.columns
x_cols = x_cols.drop(['QuoteNumber','QuoteConversion_Flag'])
 print(x_cols )<split>
X_train, X_test, y_train, y_test = train_test_split(
 train[x_cols], train['QuoteConversion_Flag'], test_size=0.5, stratify=train['QuoteConversion_Flag'], random_state=42 )<count_missing_values>
weather_train_df.isna().sum()
train_nulls = X_train.isnull().sum()
train_null_cols = list(train_nulls[train_nulls > 0].index)
 print(train_null_cols )<count_missing_values>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
test_nulls = test.isnull().sum()
test_null_cols = list(test_nulls[test_nulls > 0].index)
 print(test_null_cols )<categorify>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
X_train[train_null_cols] = X_train[train_null_cols].fillna(train[train_null_cols].mode().iloc[0] )<data_type_conversions>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
test[test_null_cols] = test[test_null_cols].fillna(test[test_null_cols].mode().iloc[0] )<feature_engineering>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
def feature_engineering(data):
data['Original_Quote_Date_year'] = data['Original_Quote_Date'].dt.year
data['Original_Quote_Date_month'] = data['Original_Quote_Date'].dt.month
data['Original_Quote_Date_weekday'] = data['Original_Quote_Date'].dt.weekday
return data<feature_engineering>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
X_train = feature_engineering(X_train )<feature_engineering>
year_map = building_meta_df.year_built.value_counts()
building_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)
 bid_map = train_df.building_id.value_counts()
train_df['bid_cnt'] = train_df.building_id.map(bid_map )
test = feature_engineering(test )<drop_column>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
X_train = X_train.drop('Original_Quote_Date', axis=1)
 test = test.drop('Original_Quote_Date', axis=1 )<categorify>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
for c in train.columns:
if train[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(np.unique(list(X_train[c].values)+ list(test[c].values)))
 X_train[c] = lbl.transform(list(X_train[c].values))
test[c] = lbl.transform(list(test[c].values))<choose_model_class>
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )
clf = xgb.XGBClassifier()
params = {
"learning_rate": [0.1, 0.001],
"max_depth": [3, 6],
"n_estimators": [100, 200, 300],
"objective": ["binary:logistic"]
}
random_search = RandomizedSearchCV(estimator=clf,
param_distributions=params,
scoring='roc_auc',
refit=True,
n_iter=3,
verbose=1)
 random_search<prepare_x_and_y>
add_sg(weather_train_df )
X_train = X_train.values
y_train = y_train.values<train_model>
category_cols = ['building_id', 'site_id', 'primary_use', 'IsHoliday', 'groupNum_train']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
'year_cnt', 'bid_cnt',
'dew_smooth', 'air_smooth',
'dew_diff', 'air_diff',
'dew_diff2', 'air_diff2'
] + list(df_groupNum_median.drop('timestamp',axis=1 ).columns )
random_search.fit(X_train, y_train )<find_best_params>
train_df = train_df.merge(building_meta_df, on=['building_id','meter'], how='left')
 train_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df = reduce_mem_usage(train_df, use_float16=True)
 del weather_train_df
gc.collect()
print(random_search.best_params_)
 print(random_search.best_score_)
 print(random_search.best_estimator_ )<predict_on_test>
def create_X_y(train_df, groupNum_train):
target_train_df = train_df[train_df['groupNum_train'] == groupNum_train].copy()
target_train_df = target_train_df.merge(df_groupNum_median, on=['timestamp'], how='left')
 target_train_df['group_median_'+str(groupNum_train)] = np.nan
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
predictions = random_search.predict(test[x_cols] )<create_dataframe>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 50
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
submission = pd.DataFrame()
submission['QuoteNumber'] = test['QuoteNumber']
submission['QuoteConversion_Flag'] = predictions<save_to_csv>
seed = 666
shuffle = False
kf = StratifiedKFold(n_splits=folds )
PREDICTIONS_FILENAME_PREFIX = 'predictions_'
PREDICTIONS_FILENAME = PREDICTIONS_FILENAME_PREFIX + time.strftime('%Y%m%d-%H%M%S')+ '.csv'
print(PREDICTIONS_FILENAME)
 submission.to_csv(PREDICTIONS_FILENAME, index = False )<set_options>
for groupNum_train in building_meta_df['groupNum_train'].unique() :
X_train, y_train = create_X_y(train_df, groupNum_train=groupNum_train)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('groupNum_train', groupNum_train, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 exec('models' +str(groupNum_train)+ '=[]')
 train_df_site = train_df[train_df['groupNum_train']==groupNum_train].copy()
for train_idx, valid_idx in kf.split(train_df_site, train_df_site['building_id']):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df_site.iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
exec('models' +str(groupNum_train)+ '.append([mindex, model])')
 gc.collect()
if debug:
break
try:
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 plt.show()
except:
pass
del X_train, y_train
gc.collect()
print('-------------------------------------------------------------' )
print('torch version:', torch.__version__)
 warnings.filterwarnings('ignore' )<define_variables>
del train_df
gc.collect()
DATA_DIR = '.. /input/bengaliai-cv19'
SUBMISSION_CSV_FILE = 'submission.csv'
MYFILE_DIR = '.. /input/myfile'<compute_train_metric>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 test_df = test_df.merge(building_meta_df[['building_id','meter','groupNum_train']], on=['building_id','meter'], how='left')
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )
def do_predict(net, input):
def logit_to_probability(logit):
probability=[]
for l in logit:
p = F.softmax(l,1)
 probability.append(p)
 return probability
num_ensemble = len(net)
 for i in range(num_ensemble):
net[i].eval()
probability=[0,0,0]
for i in range(num_ensemble):
logit = net[i](input)
 prob = logit_to_probability(logit)
 probability = [p+q for p,q in zip(probability,prob)]
probability = [p/num_ensemble for p in probability]
predict = [torch.argmax(p,-1)for p in probability]
predict = [p.data.cpu().numpy() for p in predict]
predict = np.array(predict ).T
predict = predict.reshape(-1)
 return predict<define_variables>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )
DESENET_CHECKPOINT_FILE = [
'.. /input/hengmodels/swa_fold1_no_bn_model.pth',
]
TASK_NAME = [ 'grapheme_root', 'vowel_diacritic', 'consonant_diacritic' ]<load_pretrained>
def create_X(test_df, groupNum_train):
target_test_df = test_df[test_df['groupNum_train'] == groupNum_train].copy()
target_test_df = target_test_df.merge(df_groupNum_median, on=['timestamp'], how='left')
 target_test_df = target_test_df.merge(building_meta_df, on=['building_id','meter','groupNum_train'], how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 target_test_df['group_median_'+str(groupNum_train)] = np.nan
X_test = target_test_df[feature_cols + category_cols]
return X_test
net = []
for checkpoint_file in DESENET_CHECKPOINT_FILE:
n = DenseNet().cuda()
n.load_state_dict(torch.load(checkpoint_file, map_location=lambda storage, loc: storage),strict=True)
 net.append(n )<load_from_csv>
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )
def run_make_submission_csv() :
row_id=[]
target=[]
batch_size= 32
print('
start here !!!!')
 for i in range(4):
start_timer = timer()
df = pd.read_parquet(DATA_DIR+'/test_image_data_%d.parquet'%i, engine='pyarrow')
 print('pd.read_parquet() = %s'%(time_to_str(( timer() - start_timer),'sec')))
 start_timer = timer()
num_test = len(df)
 for b in range(0,num_test,batch_size):
if b%1000==0:
print('test_image_data_%d.parquet @%06d, %s'%(i,b,time_to_str(( timer() - start_timer),'sec')))
 B = min(num_test,b+batch_size)-b
image = df.iloc[b:b+B, range(1,32332+1)].values
image_id = df.iloc[b:b+B, 0].values
image = image.reshape(B,1,137, 236)
 image = np.tile(image,(1,3,1,1))
image = image.astype(np.float32)/255
input = torch.from_numpy(image ).float().cuda()
predict = do_predict(net, input)
 image_id = np.tile(image_id.reshape(B,1),(1,3,)) + ['_'] + TASK_NAME
image_id = image_id.reshape(-1)
 row_id.append(image_id)
 target.append(predict)
 print('')
 row_id = np.concatenate(row_id)
 target = np.concatenate(target)
 df = pd.DataFrame(zip(row_id, target), columns=['row_id', 'target'])
 df.to_csv(SUBMISSION_CSV_FILE, index=False)
<find_best_params>
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118
run_make_submission_csv()<install_modules>
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null 
<set_options>
if not debug:
sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f' )
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )<define_variables>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates() , on='building_id', how='left' )
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'<define_variables>
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )<load_from_csv>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
<categorify>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
<categorify><EOS> 
print('LV score=', leak_score )