Kaggle competition: ASHRAE - Great Energy Predictor III
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )<save_to_csv>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<split>
basePath = '/kaggle/input/ashrae-energy-prediction/'
building_metadata = pd.read_csv(basePath+'building_metadata.csv')
 weather_train = pd.read_csv(basePath+'weather_train.csv')
 raw_train = pd.read_csv(basePath+'train.csv')
 print(building_metadata.head(10))
print(weather_train.head(10))
print(raw_train.head(10))
RS = 12357
ROUNDS = 315
print("Started")
 np.random.seed(RS)
 input_folder = '.. /input/'
def train_xgb(X, y, params):
print("Will train XGB for {} rounds, RandomSeed: {}".format(ROUNDS, RS))
x, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=RS)
 xg_train = xgb.DMatrix(x, label=y_train)
 xg_val = xgb.DMatrix(X_val, label=y_val)
 watchlist = [(xg_train, 'train'),(xg_val, 'eval')]
return xgb.train(params, xg_train, ROUNDS, watchlist)
 def predict_xgb(clr, X_test):
return clr.predict(xgb.DMatrix(X_test))
def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 i = 0
for feat in features:
outfile.write('{0}\t{1}\tq
'.format(i, feat))
i = i + 1
outfile.close()<load_from_csv>
del curSiteData
gc.collect()
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.11
params['max_depth'] = 5
params['silent'] = 1
params['seed'] = RS
df_train = pd.read_csv(input_folder + 'train.csv')
 df_test = pd.read_csv(input_folder + 'test.csv')
 print("Original data: X_train: {}, X_test: {}".format(df_train.shape, df_test.shape)) 
<feature_engineering>
weather_train = weather_train.drop('hour',axis=1 )
print("Features processing")
 def get_weight(count, eps=10000, min_count=2):
return 0 if count < min_count else 1 /(count + eps)
 train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist() ).astype(str)
 words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
stops = set(stopwords.words("english")) 
<string_transform>
offsetBase = weather_train[['site_id','timestamp','air_temperature']]
offsetBase['timestamp'] = pd.to_datetime(offsetBase['timestamp'])
 offsetBase['temp_rank'] = offsetBase.groupby(['site_id', offsetBase.timestamp.dt.date])['air_temperature'].rank('average' )
def word_shares(row):
q1 = set(str(row['question1'] ).lower().split())
 q1words = q1.difference(stops)
 if len(q1words)== 0:
return '0:0:0:0:0'
q2 = set(str(row['question2'] ).lower().split())
 q2words = q2.difference(stops)
 if len(q2words)== 0:
return '0:0:0:0:0'
q1stops = q1.intersection(stops)
 q2stops = q2.intersection(stops)
 shared_words = q1words.intersection(q2words)
 shared_weights = [weights.get(w, 0)for w in shared_words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R1 = np.sum(shared_weights)/ np.sum(total_weights)
 R2 = len(shared_words)/(len(q1words)+ len(q2words))
R31 = len(q1stops)/ len(q1words)
 R32 = len(q2stops)/ len(q2words)
 return '{}:{}:{}:{}:{}'.format(R1, R2, len(shared_words), R31, R32 )<feature_engineering>
weather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])
 def alignTimestamp(df):
df['offset'] = df.site_id.map(site_offset)
 df['timestamp_updated'] = df['timestamp'] - pd.to_timedelta(df['offset'],unit='H')
 df['timestamp'] = df['timestamp_updated']
df = df.drop('timestamp_updated',axis=1)
 return df
weather_train = alignTimestamp(weather_train )
df = pd.concat([df_train, df_test])
 df['word_shares'] = df.apply(word_shares, axis=1, raw=True)
 x = pd.DataFrame()
x['word_match'] = df['word_shares'].apply(lambda x: float(x.split(':')[0]))
x['tfidf_word_match'] = df['word_shares'].apply(lambda x: float(x.split(':')[1]))
x['shared_count'] = df['word_shares'].apply(lambda x: float(x.split(':')[2]))
x['stops1_ratio'] = df['word_shares'].apply(lambda x: float(x.split(':')[3]))
x['stops2_ratio'] = df['word_shares'].apply(lambda x: float(x.split(':')[4]))
x['diff_stops_r'] = x['stops1_ratio'] - x['stops2_ratio']
x['len_q1'] = df['question1'].apply(lambda x: len(str(x)))
 x['len_q2'] = df['question2'].apply(lambda x: len(str(x)))
 x['diff_len'] = x['len_q1'] - x['len_q2']
x['len_char_q1'] = df['question1'].apply(lambda x: len(str(x ).replace(' ', '')))
 x['len_char_q2'] = df['question2'].apply(lambda x: len(str(x ).replace(' ', '')))
 x['diff_len_char'] = x['len_char_q1'] - x['len_char_q2']
x['len_word_q1'] = df['question1'].apply(lambda x: len(str(x ).split()))
x['len_word_q2'] = df['question2'].apply(lambda x: len(str(x ).split()))
x['diff_len_word'] = x['len_word_q1'] - x['len_word_q2']
x['avg_world_len1'] = x['len_char_q1'] / x['len_word_q1']
x['avg_world_len2'] = x['len_char_q2'] / x['len_word_q2']
x['diff_avg_word'] = x['avg_world_len1'] - x['avg_world_len2']
x['exactly_same'] =(df['question1'] == df['question2'] ).astype(int)
 x['duplicated'] = df.duplicated(['question1', 'question2'] ).astype(int )<prepare_x_and_y>
weather_na_cols = ["wind_direction","wind_speed"]
for i,i_col in enumerate(weather_na_cols):
weather_train[i_col].fillna(weather_train[i_col].mean() ,inplace=True )
feature_names = list(x.columns.values)
 create_feature_map(feature_names)
 print("Features: {}".format(feature_names))
x_train = x[:df_train.shape[0]]
x_test = x[df_train.shape[0]:]
y_train = df_train['is_duplicate'].values
del x, df_train<categorify>
raw_train['timestamp'] = pd.to_datetime(raw_train['timestamp'])
 raw_train['dayofweek'] = raw_train['timestamp'].dt.dayofweek
raw_train['weekday'] = raw_train['timestamp'].dt.weekday
raw_train['hour'] = raw_train['timestamp'].dt.hour
raw_train['day'] = raw_train['timestamp'].dt.day
raw_train['month'] = raw_train['timestamp'].dt.month
if 1:
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
print("Oversampling started for proportion: {}".format(len(pos_train)/(len(pos_train)+ len(neg_train))))
p = 0.175
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -= 1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print("Oversampling done, new proportion: {}".format(len(pos_train)/(len(pos_train)+ len(neg_train))))
x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<train_model>
del raw_train
gc.collect()
print("Training data: X_train: {}, Y_train: {}, X_test: {}".format(x_train.shape, len(y_train), x_test.shape))
ROUNDS = 500
clr = train_xgb(x_train, y_train, params)
 preds = predict_xgb(clr, x_test)
 print(preds.shape )<save_to_csv>
raw_train = pd.read_csv(basePath+'train.csv' )
print("Translate for output type...")
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = preds
sub.to_csv("xgb_seed{}_n{}.csv".format(RS, ROUNDS), index=False)
 print("Features importances...")
 importance = clr.get_fscore(fmap='xgb.fmap')
 importance = sorted(importance.items() , key=operator.itemgetter(1))
ft = pd.DataFrame(importance, columns=['feature', 'fscore'])
 ft.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(10, 25))
plt.gcf().savefig('features_importance.png')
 print('Clear.' )<set_options>
raw_train_building_info = pd.merge(raw_train, building_metadata,left_on = "building_id", right_on = "building_id", how = "left")
 groupedYearBuiltTrainData = raw_train_building_info.groupby(['year_built'] ).mean().reset_index()
ax = sns.lineplot(x="year_built", y="meter_reading", data=groupedDOWTrainData_buildingInfo[['year_built','meter_reading']])
 ax.set_title('Average Meter reading by year')
 del groupedYearBuiltTrainData
warnings.filterwarnings("ignore")
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
del raw_train_building_info
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float32',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int16',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float64',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float32',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float32',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float64',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float64',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float32',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float32',
'Census_InternalPrimaryDisplayResolutionVertical': 'float32',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float64',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<load_from_csv>
raw_train = raw_train [ raw_train['building_id'] != 1099 ]
raw_train = raw_train.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
%%time
train = pd.read_csv('/kaggle/input/train.csv', dtype= dtypes ,nrows = 2000000)
 test = pd.read_csv('/kaggle/input/test.csv' , dtype = dtypes)
 train = reduce_mem_usage(train)
 test = reduce_mem_usage(test)
 score = []<define_variables>
raw_train['timestamp'] = pd.to_datetime(raw_train['timestamp'])
 raw_train['is_holiday'] =(raw_train['timestamp'].dt.date.isin(us_holidays)).astype(np.int8 )
print('Number of columns in train data is ->',train.shape[1])
 float_list = []
int_list = []
category_list = []
for f in train.columns:
if train[f].dtype =='int64' or train[f].dtype =='int8' or train[f].dtype =='int16' or train[f].dtype =='int32':
int_list.append(f)
 elif train[f].dtype =='float64' or train[f].dtype =='float32' or train[f].dtype =='float16':
float_list.append(f)
 else:
category_list.append(f)
 print('Number of columns in train data of Float Type is ->', len(float_list))
print('Number of columns in train data of Int Type is ->', len(int_list))
print('Number of columns in train data of Category Type is ->', len(category_list))<count_values>
building_traindata = pd.merge(raw_train,building_metadata,left_on = "building_id", right_on = "building_id", how = "left")
 raw_train_data = pd.merge(building_traindata,weather_train,left_on = ["site_id", "timestamp"], right_on = ["site_id", "timestamp"])
 na_columns = raw_train_data.isna().sum()
print(na_columns/raw_train_data.shape[0] )
print(train['HasDetections'].value_counts() /train.shape[0]*100 )<count_unique_values>
raw_train_data['dayofweek'] = raw_train_data['timestamp'].dt.dayofweek.astype(np.uint8)
 raw_train_data['weekday'] = raw_train_data['timestamp'].dt.weekday.astype(np.uint8)
 raw_train_data['hour'] = raw_train_data['timestamp'].dt.hour.astype(np.uint8)
 raw_train_data['day'] = raw_train_data['timestamp'].dt.day.astype(np.uint8)
 raw_train_data['month'] = raw_train_data['timestamp'].dt.month.astype(np.uint8 )
print('Creating data info of train data and test data.....')
 stats = []
for f in train.columns:
stats.append(( f ,train[f].nunique() , train[f].isnull().sum() *100/train.shape[0] ,
train[f].value_counts(normalize = True , dropna = True ).values[0]*100 ,
train[f].value_counts(normalize = True , dropna = False ).values[0]*100 ,
train[f].dtype))
stats_df = pd.DataFrame(stats , columns= ['Columns', 'Number of Unique Values' , 'Percentage of Data missing in train' ,
'Percentage of Biggest Category of dropna True' ,
'Percentage of Biggest Category of dropna False' ,
'Columns DataType'])
 display(stats_df.sort_values('Percentage of Data missing in train', ascending=False))<count_missing_values>
beaufort_vals = [(0,0,0.4),(1,0.5,1.5),(2,1.6,3.3),(3,3.4,5.5),(4,5.6,7.9),(5,8.0,10.7),(6,10.8,13.8),(7,13.9,17.1), 
(8,17.2,20.7),(9,20.8,24.4),(10,24.5,28.4),(11,28.5,32.6),(12,32.7,400)]
col_temp = ['EngineVersion','AppVersion','AvSigVersion', 'OsBuildLab' , 'OsVer' , 'Census_OSVersion']
for f in col_temp:
print('Null Values in column ',f,' is ->',train[f].isnull().sum() )<data_type_conversions>
def degToCompass(num):
val=int(( num/22.5)+.5)
 arr=[i for i in range(0,16)]
return arr[(val % 16)]
def add_category_1(data):
data['EngineVersion_0'] = data['EngineVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['EngineVersion_1'] = data['EngineVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['EngineVersion_2'] = data['EngineVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['EngineVersion_3'] = data['EngineVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['AppVersion_0'] = data['AppVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['AppVersion_1'] = data['AppVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['AppVersion_2'] = data['AppVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['AppVersion_3'] = data['AppVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['AvSigVersion_0'] = data['AvSigVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['AvSigVersion_1'] = data['AvSigVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['AvSigVersion_2'] = data['AvSigVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['AvSigVersion_3'] = data['AvSigVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['OsBuildLab_0'] = data['OsBuildLab'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['OsBuildLab_1'] = data['OsBuildLab'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['OsBuildLab_2'] = data['OsBuildLab'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['OsBuildLab_3'] = data['OsBuildLab'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['OsVer_0'] = data['OsVer'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['OsVer_1'] = data['OsVer'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['OsVer_2'] = data['OsVer'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['OsVer_3'] = data['OsVer'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['Census_OSVersion_0'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['Census_OSVersion_1'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['Census_OSVersion_2'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['Census_OSVersion_3'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 return data<count_values>
raw_train_data['wind_direction'] = raw_train_data['wind_direction'].apply(degToCompass )
remove_list = []
for f in train.columns:
if train[f].isnull().sum() *100/train.shape[0] > 98 :
remove_list.append(f)
 for f in train.columns:
if train[f].value_counts().values[0]*100/train.shape[0] > 98:
remove_list.append(f)
 print(remove_list )<drop_column>
for i_scale in beaufort_vals:
raw_train_data.loc[(raw_train_data['wind_speed']>=i_scale[1])&(raw_train_data['wind_speed']<=i_scale[2]),'beaufort_scale'] = i_scale[0]
train.drop(columns = remove_list , inplace = True)
 test.drop(columns = remove_list , inplace = True )<define_variables>
categoricalCols = ["site_id", "building_id", "primary_use", "hour", "weekday", "meter", "wind_direction","is_holiday"]
numericCols = ["square_feet", "year_built", "air_temperature", "cloud_coverage",
"dew_temperature", 'precip_depth_1_hr', 'floor_count','beaufort_scale']
cat_col = []
for f in train.columns:
if str(train[f].dtype)== 'category':
cat_col.append(f)
 print(len(cat_col))
print(cat_col )<categorify>
le = LabelEncoder()
le.fit(raw_train_data['primary_use'])
 raw_train_data['primary_use'] = le.transform(raw_train_data['primary_use'] )
to_encode = []
for f in cat_col:
if train[f].nunique() >10:
to_encode.append(f)
 print(to_encode )<drop_column>
featuresToUse = numericCols+categoricalCols
train_data = raw_train_data.loc[:,featuresToUse]
train_data['square_feet'] = np.log(train_data['square_feet'])
 train_data['year_built'] = train_data['year_built']-1900
train_data = reduce_mem_usage(train_data)
 target_data = raw_train_data['meter_reading']
to_encode.remove('MachineIdentifier')
 cat_col.remove('MachineIdentifier' )<categorify>
del weather_train,raw_train_data
gc.collect()
def frequency_encoder(variable):
df = pd.concat([ train[variable] , test[variable] ] ).value_counts().reset_index()
df = df.merge(df[variable].value_counts().reset_index().reset_index().rename(columns = {'level_0': 'rank_encode' , 'index':variable , variable:'count'}), on =variable , how='left')[['index' , 'rank_encode']]
df.set_index('index' , inplace = True)
 return df.to_dict() ['rank_encode']<categorify>
n_folds=2
cv = KFold(n_splits=n_folds, random_state=666, shuffle=False)
 def computeRMSE(clf,index):
y_predict = clf.predict(train_data.iloc[index])
 y_predict[y_predict<0]=0
return np.sqrt(mean_squared_error(target_data.iloc[index], y_predict))
%%time
for f in tqdm_notebook(to_encode):
dictionary = frequency_encoder(f)
 train[f+'_ranking'] = train[f].apply(lambda x: dictionary.get(x , np.nan)).astype('category')
 test[f+'_ranking'] = test[f].apply(lambda x : dictionary.get(x , np.nan)).astype('category')
<categorify>
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse'},
'subsample': 0.25,
'subsample_freq': 1,
'learning_rate': 0.4,
'num_leaves': 20,
'feature_fraction': 0.9,
'lambda_l1': 1,
'lambda_l2': 1,
'max_bin':64,
'early_stopping_rounds':200
}
%%time
for f in tqdm_notebook(cat_col):
lb = LabelEncoder()
lb.fit(list(train[f].values)+ list(test[f].values))
train[f] = lb.transform(list(train[f].values))
test[f] = lb.transform(list(test[f].values))<prepare_x_and_y>
scores_lgb = []
feature_importances = pd.DataFrame()
feature_importances['feature'] = train_data.columns
models = []
for(train, val), i in zip(cv.split(train_data, train_data['building_id']), range(n_folds)) :
lgb_train = lgbm.Dataset(train_data.iloc[train,:], target_data.iloc[train],categorical_feature=categoricalCols)
 lgb_val = lgbm.Dataset(train_data.iloc[val,:], target_data.iloc[val],categorical_feature=categoricalCols)
 lgb_best = lgbm.train(
 params,
lgb_train,
num_boost_round=2000,
valid_sets=(lgb_train, lgb_val),
verbose_eval = 100 
)
 feature_importances['fold_{}'.format(i + 1)] = lgb_best.feature_importance()
rmse_train = computeRMSE(lgb_best,train)
 rmse_val = computeRMSE(lgb_best,val)
 scores_lgb.append(( rmse_train, rmse_val))
models.append(lgb_best )
y = train['HasDetections']
train.drop(columns=['HasDetections' , 'MachineIdentifier' ] , inplace = True)
 test_index = list(test['MachineIdentifier'])
 test.drop(columns = ['MachineIdentifier'] , inplace = True)
 train.sort_values('AvSigVersion')
 n_folds = 3
folds = StratifiedKFold(n_splits = n_folds , shuffle = True , random_state = 98)
<drop_column>
del train_data, target_data
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<init_hyperparams>
del lgb_train,lgb_val,train, val
params = {'num_leaves': 256,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 5,
'learning_rate': 0.05,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 5,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"lambda_l1": 0.15,
"lambda_l2": 0.15,
"random_state": 42,
"verbosity": -1}<split>
raw_test = pd.read_csv(basePath+'test.csv')
 raw_test['timestamp'] = pd.to_datetime(raw_test['timestamp'])
 raw_test['is_holiday'] =(raw_test['timestamp'].dt.date.isin(us_holidays)).astype(np.int8)
 building_testdata = pd.merge(raw_test,building_metadata,left_on = ["building_id"], right_on = ["building_id"],how='left' )
test_1 = test[ : int(test.shape[0]/3)]
test_2 = test[int(test.shape[0]/3): int(test.shape[0]/3*2)]
test_3 = test[int(test.shape[0]/3*2): ]<set_options>
del building_metadata,raw_test
gc.collect()
building_testdata = reduce_mem_usage(building_testdata )
del test
gc.collect()<compute_test_metric>
weather_test = pd.read_csv(basePath+'weather_test.csv')
 weather_test['timestamp'] = pd.to_datetime(weather_test['timestamp'])
 weather_test = alignTimestamp(weather_test)
 for i,i_col in enumerate(weather_na_cols):
weather_test[i_col].fillna(weather_test[i_col].mean() ,inplace=True)
 raw_test_data = pd.merge(building_testdata,weather_test,left_on = ["site_id", "timestamp"], right_on = ["site_id", "timestamp"],how = 'left')
 del weather_test,building_testdata
gc.collect()
def fast_auc(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(preds, dtrain):
return fast_auc(dtrain, preds)
 train_col = train.columns
feature_importance = pd.DataFrame()
pred_1 = np.zeros(test_1.shape[0])
 pred_2 = np.zeros(test_2.shape[0])
 pred_3 = np.zeros(test_3.shape[0])
 feature_importance['columns'] = train.columns
for n_fold ,(train_index , valid_index)in enumerate(folds.split(train , y)) :
gc.collect()
print('Fold ',n_fold , ' Started at ',time.ctime())
 X_train , X_valid = train.iloc[train_index] , train.iloc[valid_index]
y_train , y_valid = y.iloc[train_index] , y.iloc[valid_index]
train_data = lgb.Dataset(X_train[train_col] , label = y_train)
 valid_data = lgb.Dataset(X_valid[train_col] , label = y_valid)
 model = lgb.train(params , train_data , num_boost_round = 1000 , valid_sets = [train_data , valid_data] , verbose_eval =-1, early_stopping_rounds = 200)
 pred = model.predict(X_valid , num_iteration=model.best_iteration)
 pred = list(pred)
 pred_1+=model.predict(test_1[train_col])/n_folds
pred_2+=model.predict(test_2[train_col])/n_folds
pred_3+=model.predict(test_3[train_col])/n_folds
score.append(( r2_score(pred , y_valid), mean_squared_error(pred , y_valid), eval_auc(pred , y_valid)))
 feature_importance['fold_'+str(n_fold+1)] = model.feature_importance() 
<feature_engineering>
na_columns = raw_test_data.isna().sum()
print(na_columns/raw_test_data.shape[0])
 for i,i_col in enumerate(weather_na_cols):
raw_test_data[i_col].fillna(raw_test_data[i_col].mean() ,inplace=True )
display(score)
 feature_importance['avg'] =(feature_importance['fold_1']+feature_importance['fold_2']+feature_importance['fold_3'])/3
feature_importance.head() 
<save_to_csv>
raw_test_data['wind_direction'] = raw_test_data['wind_direction'].apply(degToCompass)
 for i_scale in beaufort_vals:
raw_test_data.loc[(raw_test_data['wind_speed']>=i_scale[1])&(raw_test_data['wind_speed']<=i_scale[2]),'beaufort_scale'] = i_scale[0]
submission = pd.DataFrame()
submission['MachineIdentifier'] = test_index
submission['HasDetections'] = pred
submission.to_csv('lgb_first.csv' , index = False )<set_options>
raw_test_data['primary_use'] = le.transform(raw_test_data['primary_use'])
 raw_test_data = reduce_mem_usage(raw_test_data )
try:
cdll.LoadLibrary("libc.so.6")
 libc = CDLL("libc.so.6")
 libc.malloc_trim(0)
 except(OSError, AttributeError):
libc = None
__old_del = getattr(pd.DataFrame, '__del__', None)
 def __new_del(self):
if __old_del:
__old_del(self)
 libc.malloc_trim(0)
 if libc:
print('Applying monkeypatch for pd.DataFrame.__del__', file=sys.stderr)
 pd.DataFrame.__del__ = __new_del
else:
print('Skipping monkeypatch for pd.DataFrame.__del__: libc or malloc_trim() not found', file=sys.stderr )<load_from_csv>
raw_test_data['dayofweek'] = raw_test_data['timestamp'].dt.dayofweek.astype(np.uint8)
 raw_test_data['weekday'] = raw_test_data['timestamp'].dt.weekday.astype(np.uint8)
 raw_test_data['hour'] = raw_test_data['timestamp'].dt.hour.astype(np.uint8)
 raw_test_data['day'] = raw_test_data['timestamp'].dt.day.astype(np.uint8)
 raw_test_data['month'] = raw_test_data['timestamp'].dt.month.astype(np.uint8)
 raw_test_data = raw_test_data.drop(['timestamp'],axis=1 )
Debug = False
def load(x):
ignore = ['MachineIdentifier']
if x in ignore: return False
else: return True
if Debug:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load,nrows=10000)
 else:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
if Debug:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load,nrows=10000)
 else:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
 print('Loaded',len(df_train),'rows of TRAIN and',len(df_test),'rows of TEST' )<categorify>
raw_test_data['square_feet'] = np.log(raw_test_data['square_feet'])
 raw_test_data['year_built'] = raw_test_data['year_built']-1900
raw_test_data = raw_test_data[featuresToUse]
raw_test_data,NAList = reduce_mem_usage(raw_test_data)
 gc.collect()
def encode_FE(df,col):
vc = df[col].value_counts(dropna=False, normalize=True ).to_dict()
nm = col+'_FE'
df[nm] = df[col].map(vc)
 df[nm] = df[nm].astype('float32')
 return [nm]
def encode_FE2(df1, df2, col):
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=False, normalize=True ).to_dict()
nm = col+'_FE2'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 return [nm]
def factor_data(df_train, df_test, col):
df_comb = pd.concat([df_train[col],df_test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 df_comb += 1
df_comb = np.where(df_comb==0, df_comb.max() +1, df_comb)
 df_train[col] = df_comb[:len(df_train)]
df_test[col] = df_comb[len(df_train):]
del df_comb
def reduce_memory(df,col):
mx = df[col].max()
if mx<256:
df[col] = df[col].astype('uint8')
 elif mx<65536:
df[col] = df[col].astype('uint16')
 else:
df[col] = df[col].astype('uint32')
 def relax_data(df_train, df_test, col):
cv1 = pd.DataFrame(df_train[col].value_counts().reset_index().rename({col:'train'},axis=1))
cv2 = pd.DataFrame(df_test[col].value_counts().reset_index().rename({col:'test'},axis=1))
cv3 = pd.merge(cv1,cv2,on='index',how='outer')
 factor = len(df_test)/len(df_train)
 cv3['train'].fillna(0,inplace=True)
 cv3['test'].fillna(0,inplace=True)
 cv3['remove'] = False
cv3['remove'] = cv3['remove'] |(cv3['train'] < len(df_train)/10000)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] < cv3['test']/3)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] > 3*cv3['test'])
 cv3['new'] = cv3.apply(lambda x: x['index'] if x['remove']==False else 0,axis=1)
 cv3['new'],_ = cv3['new'].factorize(sort=True)
 cv3.set_index('index',inplace=True)
 cc = cv3['new'].to_dict()
df_train[col] = df_train[col].map(cc)
 reduce_memory(df_train,col)
 df_test[col] = df_test[col].map(cc)
 reduce_memory(df_test,col)
 def display_memory(df_train, df_test):
print(len(df_train),'rows of training data use',df_train.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 print(len(df_test),'rows of test data use',df_test.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 def categorize(df_train, df_test, cols):
for col in cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )<feature_engineering>
i = 0
stepSize = 50000
predictions = []
for j in tqdm(range(int(np.ceil(raw_test_data.shape[0]/stepSize)))) :
predictions.append(np.expm1(sum([model.predict(raw_test_data.iloc[i:i+stepSize])for model in models])/n_folds))
i+=stepSize
predictions = np.concatenate(predictions )
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_train['DateAS'] = df_train['AvSigVersion'].map(datedictAS)
 df_test['DateAS'] = df_test['AvSigVersion'].map(datedictAS)
 datedictOS = np.load('.. /input/malware-timestamps-2/OSVersionTimestamps.npy')[() ]
df_train['DateOS'] = df_train['Census_OSVersion'].map(datedictOS)
 df_test['DateOS'] = df_test['Census_OSVersion'].map(datedictOS)
 df_train['AppVersion2'] = df_train['AppVersion'].map(lambda x: np.int(x.split('.')[1]))
df_test['AppVersion2'] = df_test['AppVersion'].map(lambda x: np.int(x.split('.')[1]))
df_train['Lag1'] = df_train['DateAS'] - df_train['DateOS']
df_train['Lag1'] = df_train['Lag1'].map(lambda x: x.days//7)
 df_test['Lag1'] = df_test['DateAS'] - df_test['DateOS']
df_test['Lag1'] = df_test['Lag1'].map(lambda x: x.days//7)
 df_train['Lag5'] = datetime(2018,7,26)- df_train['DateAS']
df_train['Lag5'] = df_train['Lag5'].map(lambda x: x.days//1)
 df_train.loc[ df_train['Lag5']<0, 'Lag5' ] = 0
df_test['Lag5'] = datetime(2018,9,27)- df_test['DateAS']
df_test['Lag5'] = df_test['Lag5'].map(lambda x: x.days//1)
 df_test.loc[ df_test['Lag5']<0, 'Lag5' ] = 0
df_train['Lag5'] = df_train['Lag5'].astype('float32')
 df_test['Lag5'] = df_test['Lag5'].astype('float32')
 df_train['driveA'] = df_train['Census_SystemVolumeTotalCapacity'].astype('float')/df_train['Census_PrimaryDiskTotalCapacity'].astype('float')
 df_test['driveA'] = df_test['Census_SystemVolumeTotalCapacity'].astype('float')/df_test['Census_PrimaryDiskTotalCapacity'].astype('float')
 df_train['driveA'] = df_train['driveA'].astype('float32')
 df_test['driveA'] = df_test['driveA'].astype('float32')
 df_train['driveB'] = df_train['Census_PrimaryDiskTotalCapacity'].astype('float')- df_train['Census_SystemVolumeTotalCapacity'].astype('float')
 df_test['driveB'] = df_test['Census_PrimaryDiskTotalCapacity'].astype('float')- df_test['Census_SystemVolumeTotalCapacity'].astype('float')
 df_train['driveB'] = df_train['driveB'].astype('float32')
 df_test['driveB'] = df_test['driveB'].astype('float32')
 cols6=['Lag1']
cols8=['Lag5','driveB','driveA']
del df_train['DateAS'], df_train['DateOS']
del df_test['DateAS'], df_test['DateOS']
del datedictAS, datedictOS
x=gc.collect()<categorify>
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = predictions
submission.loc[submission['meter_reading']<0, 'meter_reading'] = 0
submission.to_csv('submission.csv', index=False)
 submission