Kaggle competition: House Prices - Advanced Regression Techniques
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242)
 print('ok' )<train_model>
warnings.filterwarnings('ignore')
 pd.set_option('display.float_format', lambda x: '%.3f' % x)
 %matplotlib inline
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.05
params['max_depth'] = 4
params['min_child_weight'] = 3
params['gamma'] = 0
params['subsample'] = 0.9
params['colsample_bytree'] = 0.9
params['nthread'] = 4
params['seed'] = 50
params['scale_pos_weight=1'] = 1
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 450, watchlist, early_stopping_rounds=50, verbose_eval=10)
 print('ok' )<save_to_csv>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = quora_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False)
 print('ok' )<set_options>
whole_df = pd.concat([train, test] )
gc.enable()
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
train_path='/kaggle/input/microsoft-malware-prediction/train.csv'
test_path='/kaggle/input/microsoft-malware-prediction/test.csv'
train = pd.read_csv(train_path, dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv(test_path, dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 gc.collect()<count_missing_values>
train_salePrice = train.loc[:, 'SalePrice']
train_ID = train.loc[:, 'Id']
test_ID = test.loc[:, 'Id']
train.isna().sum() [train.isna().sum() > 0] * 100 / len(train )<categorify>
whole_df.set_index('Id', drop = True, inplace = True )
y_train = train['HasDetections']
del train['HasDetections']
for ids, usecol in enumerate(train.columns.tolist() [1:-1]):
print(ids, usecol)
 train[usecol] = train[usecol].astype('str')
 test[usecol] = test[usecol].astype('str')
 le = LabelEncoder().fit(
 np.unique(train[usecol].unique().tolist() +
test[usecol].unique().tolist()))
train[usecol] = le.transform(train[usecol])+1
test[usecol] = le.transform(test[usecol])+1
agg_tr =(train 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Train'}, axis=1))
agg_te =(test 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Test'}, axis=1))
agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer' ).replace(np.nan, 0)
 agg = agg[(agg['Train'] > 1000)].reset_index(drop=True)
 agg['Total'] = agg['Train'] + agg['Test']
agg = agg[(agg['Train'] / agg['Total'] > 0.2)&(agg['Train'] / agg['Total'] < 0.8)]
agg[usecol+'Copy'] = agg[usecol]
train[usecol] =(pd.merge(train[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
test[usecol] =(pd.merge(test[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
del le, agg_tr, agg_te, agg
gc.collect()<train_model>
whole_df.drop(['GarageArea', '1stFlrSF', 'TotRmsAbvGrd', 'GarageYrBlt', 'Utilities','MiscVal'], axis = 1, inplace = True )
del train['MachineIdentifier'], test['MachineIdentifier']
gc.collect()
skf = StratifiedKFold(n_splits=6, shuffle=True, random_state=42)
 skf.get_n_splits(train.index, y_train)
 test_result = np.zeros(len(test))
for train_index, test_index in skf.split(train.index, y_train):
X_fit, X_val = train[train.index.isin(train_index)], train[train.index.isin(test_index)]
y_fit, y_val = y_train[y_train.index.isin(train_index)], y_train[y_train.index.isin(test_index)]
model = lgb.LGBMClassifier(max_depth=-1,
n_estimators=30000,
learning_rate=0.1,
colsample_bytree=0.2,
objective='binary',
n_jobs=-1)
 model.fit(X_fit, y_fit, eval_metric='auc', eval_set=[(X_val, y_val)],
verbose=100, early_stopping_rounds=100)
 test_result += model.predict_proba(test)[:,1]
del X_fit, X_val, y_fit, y_val
gc.collect()
to_submit = pd.read_csv('/kaggle/input/microsoft-malware-prediction/sample_submission.csv')
 to_submit['HasDetections'] = test_result / 6
to_submit.to_csv('fin.csv', index=False )<define_variables>
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis =1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 miss_cols = missing.iloc[:20].index
missing.iloc[:20]
Debug = False
MM = ['AvSigVersion','Census_OSVersion','Census_OSBuildRevision','AppVersion','EngineVersion']
NE = ['Census_SystemVolumeTotalCapacity','Census_PrimaryDiskTotalCapacity']
OHE = [ 'RtpStateBitfield','DefaultBrowsersIdentifier', 'AVProductStatesIdentifier',
'AVProductsInstalled', 'AVProductsEnabled', 'CountryIdentifier', 'CityIdentifier',
'GeoNameIdentifier', 'LocaleEnglishNameIdentifier', 'Processor', 'OsBuild', 'OsSuite',
'SmartScreen','Census_MDC2FormFactor', 'Census_OEMNameIdentifier',
'Census_ProcessorCoreCount', 'Census_ProcessorModelIdentifier',
'Census_OSUILocaleIdentifier', 'Census_PrimaryDiskTypeName',
'Census_HasOpticalDiskDrive', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName',
'Census_InternalPrimaryDiagonalDisplaySizeInInches',
'Census_InternalPrimaryDisplayResolutionHorizontal',
'Census_InternalPrimaryDisplayResolutionVertical',
'Census_PowerPlatformRoleName', 'Census_InternalBatteryType',
'Census_InternalBatteryNumberOfCharges', 'Census_OSEdition', 'Census_GenuineStateName',
'Census_ActivationChannel', 'Census_FirmwareManufacturerIdentifier', 'Census_IsTouchEnabled',
'Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable', 'Wdft_IsGamer',
'Wdft_RegionIdentifier', 'OsBuildLab', 'OrganizationIdentifier','Platform',
'Census_OEMModelIdentifier', 'IsProtected', 'IeVerIdentifier','Firewall',
'Census_ProcessorManufacturerIdentifier','Census_OSInstallTypeName',
'Census_OSWUAutoUpdateOptionsName','Census_IsFlightingInternal',
'Census_FlightRing','Census_ThresholdOptIn','Census_FirmwareVersionIdentifier',
'Census_IsSecureBootEnabled','Census_IsWIMBootEnabled']
XX = ['SMode','IsBeta', 'OsVer', 'OsPlatformSubRelease', 'SkuEdition', 'AutoSampleOptIn', 'PuaMode',
'UacLuaenable', 'Census_ProcessorClass', 'Census_OSArchitecture', 'Census_OSBranch',
'Census_OSBuildNumber', 'Census_OSSkuName', 'Census_OSInstallLanguageIdentifier',
'Census_IsPortableOperatingSystem', 'Census_IsFlightsDisabled', 'Census_IsVirtualDevice',
'IsSxsPassiveMode','ProductName','HasTpm','Census_DeviceFamily']
XXX = ['MachineIdentifier']
dtypes = {}
for x in OHE+NE+MM: dtypes[x] = 'category'
dtypes['HasDetections'] = 'int8'
if Debug:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', usecols=dtypes.keys() , dtype=dtypes,nrows=10000)
 else:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', usecols=dtypes.keys() , dtype=dtypes)
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
print('Loaded',len(df_train),'rows of TRAIN.CSV!')
 df_train = df_train.sample(frac=1)
 df_train.reset_index(drop=True,inplace=True)
 if Debug:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=list(dtypes.keys())[0:-1], dtype=dtypes,nrows=10000)
 else:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=list(dtypes.keys())[0:-1], dtype=dtypes)
 print('Loaded',len(df_test),'rows of TEST.CSV!' )<categorify>
whole_df.loc[:, "Alley"] = whole_df.loc[:, "Alley"].fillna("None")
 whole_df.loc[:, "BedroomAbvGr"] = whole_df.loc[:, "BedroomAbvGr"].fillna(0)
 whole_df.loc[:, "BsmtQual"] = whole_df.loc[:, "BsmtQual"].fillna("None")
 whole_df.loc[:, "BsmtCond"] = whole_df.loc[:, "BsmtCond"].fillna("None")
 whole_df.loc[:, "BsmtExposure"] = whole_df.loc[:, "BsmtExposure"].fillna("None")
 whole_df.loc[:, "BsmtFinType1"] = whole_df.loc[:, "BsmtFinType1"].fillna("None")
 whole_df.loc[:, "BsmtFinType2"] = whole_df.loc[:, "BsmtFinType2"].fillna("None")
 whole_df.loc[:, "BsmtFullBath"] = whole_df.loc[:, "BsmtFullBath"].fillna(0)
 whole_df.loc[:, "BsmtHalfBath"] = whole_df.loc[:, "BsmtHalfBath"].fillna(0)
 whole_df.loc[:, "BsmtUnfSF"] = whole_df.loc[:, "BsmtUnfSF"].fillna(0)
 whole_df.loc[:, "CentralAir"] = whole_df.loc[:, "CentralAir"].fillna("N")
 whole_df.loc[:, "Condition1"] = whole_df.loc[:, "Condition1"].fillna("Norm")
 whole_df.loc[:, "Condition2"] = whole_df.loc[:, "Condition2"].fillna("Norm")
 whole_df.loc[:, "EnclosedPorch"] = whole_df.loc[:, "EnclosedPorch"].fillna(0)
 whole_df.loc[:, "ExterCond"] = whole_df.loc[:, "ExterCond"].fillna("TA")
 whole_df.loc[:, "ExterQual"] = whole_df.loc[:, "ExterQual"].fillna("TA")
 whole_df.loc[:, "Fence"] = whole_df.loc[:, "Fence"].fillna("None")
 whole_df.loc[:, "FireplaceQu"] = whole_df.loc[:, "FireplaceQu"].fillna("None")
 whole_df.loc[:, "Fireplaces"] = whole_df.loc[:, "Fireplaces"].fillna(0)
 whole_df.loc[:, "Functional"] = whole_df.loc[:, "Functional"].fillna("Typ")
 whole_df.loc[:, "GarageType"] = whole_df.loc[:, "GarageType"].fillna("None")
 whole_df.loc[:, "GarageFinish"] = whole_df.loc[:, "GarageFinish"].fillna("None")
 whole_df.loc[:, "GarageQual"] = whole_df.loc[:, "GarageQual"].fillna("None")
 whole_df.loc[:, "GarageCond"] = whole_df.loc[:, "GarageCond"].fillna("None")
 whole_df.loc[:, "GarageCars"] = whole_df.loc[:, "GarageCars"].fillna(0)
 whole_df.loc[:, "HalfBath"] = whole_df.loc[:, "HalfBath"].fillna(0)
 whole_df.loc[:, "HeatingQC"] = whole_df.loc[:, "HeatingQC"].fillna("TA")
 whole_df.loc[:, "KitchenAbvGr"] = whole_df.loc[:, "KitchenAbvGr"].fillna(0)
 whole_df.loc[:, "KitchenQual"] = whole_df.loc[:, "KitchenQual"].fillna("TA")
 whole_df.loc[:, "LotFrontage"] = whole_df.loc[:, "LotFrontage"].fillna(0)
 whole_df.loc[:, "LotShape"] = whole_df.loc[:, "LotShape"].fillna("Reg")
 whole_df.loc[:, "MasVnrType"] = whole_df.loc[:, "MasVnrType"].fillna("None")
 whole_df.loc[:, "MasVnrArea"] = whole_df.loc[:, "MasVnrArea"].fillna(0)
 whole_df.loc[:, "MiscFeature"] = whole_df.loc[:, "MiscFeature"].fillna("None")
 whole_df.loc[:, "MSZoning"] = whole_df.loc[:, "MSZoning"].fillna(whole_df.loc[:, "MSZoning"].mode() [0])
 whole_df.loc[:, "OpenPorchSF"] = whole_df.loc[:, "OpenPorchSF"].fillna(0)
 whole_df.loc[:, "PavedDrive"] = whole_df.loc[:, "PavedDrive"].fillna("N")
 whole_df.loc[:, "PoolQC"] = whole_df.loc[:, "PoolQC"].fillna("None")
 whole_df.loc[:, "PoolArea"] = whole_df.loc[:, "PoolArea"].fillna(0)
 whole_df.loc[:, "SaleCondition"] = whole_df.loc[:, "SaleCondition"].fillna("Normal")
 whole_df.loc[:, "ScreenPorch"] = whole_df.loc[:, "ScreenPorch"].fillna(0)
 whole_df.loc[:, "WoodDeckSF"] = whole_df.loc[:, "WoodDeckSF"].fillna(0 )
def factor_data(df_train, df_test, col):
df_comb = pd.concat([df_train[col],df_test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 df_comb += 1
df_comb = np.where(df_comb==0, df_comb.max() +1, df_comb)
 df_train[col] = df_comb[:len(df_train)]
df_test[col] = df_comb[len(df_train):]
del df_comb
mx = max(df_train[col].max() ,df_test[col].max())+1
return mx
def reduce_memory(df,col):
mx = df[col].max()
if mx<256:
df[col] = df[col].astype('uint8')
 elif mx<65536:
df[col] = df[col].astype('uint16')
 else:
df[col] = df[col].astype('uint32')
 def encode_FE_lg(df,col,verbose=1):
ln = 1/df[col].nunique()
vc =(df[col].value_counts(dropna=False, normalize=True)+ln ).map(math.log ).to_dict()
nm = col+'_FE_lg'
df[nm] = df[col].map(vc)
 df[nm] -= df[nm].min()
df[nm] = df[nm]/df[nm].max()
df[nm] = df[nm].astype('float32')
 if verbose==1:
print('FE encoded',col)
 return [nm]
def encode_CE(df, col, filter, zscore, tar='HasDetections', m=0.5, verbose=1):
cv = pd.DataFrame(df[col].value_counts(dropna=False)).reset_index()
cv4 = df.groupby(col)[tar].mean().reset_index().rename({tar:'rate',col:'index'},axis=1)
 d1 = set(cv['index'].unique())
 cv = pd.merge(cv,cv4,on='index',how='left')
 if(len(cv[ cv['index'].isna() ])!=0):
cv.loc[ cv['index'].isna() ,'rate' ] = df.loc[ df[col].isna() ,tar ].mean()
cv = cv[ cv[col]>(filter * len(df)) ]
cv['ratec'] =(df[tar].sum() - cv['rate']*cv[col])/(len(df)-cv[col])
 cv['sd'] = zscore * 0.5 / cv[col].map(lambda x: math.sqrt(x))
cv = cv[(abs(cv['rate']-m)>=cv['sd'])|(abs(cv['ratec']-1+m)>=cv['sd'])]
d2 = set(cv['index'].unique())
 d = list(d1 - d2)
 if(df[col].dtype.name=='category'):
if(not 0 in df[col].cat.categories):
df[col].cat.add_categories(0,inplace=True)
 else:
print('
df.loc[ df[col].isin(d),col ] = 0
if verbose==1:
print('CE encoded',col,'-',len(d2),'values.Removed',len(d),'values')
 mx = df[col].nunique()
return [mx,d2]
def encode_CE_test(df,col,d):
if(df[col].dtype.name=='category'):
if(not 0 in df[col].cat.categories):
df[col].cat.add_categories(0,inplace=True)
 else:
print('
df.loc[ ~df[col].isin(d),col ] = 0
mx = df[col].nunique()
return [mx,d]<data_type_conversions>
whole_df.drop('SalePrice', axis = 1, inplace = True )
def makeNew(df,verbose=1,add=0,TS=True,data=0):
old = df.columns
df['AppVersion2'] = df['AppVersion'].apply(lambda x: x.split('.')[1] ).astype('category')
 if TS:
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df['DateAS'] = df['AvSigVersion'].map(datedictAS)
 datedictOS = np.load('.. /input/malware-timestamps-2/OSVersionTimestamps.npy')[() ]
df['DateOS'] = df['Census_OSVersion'].map(datedictOS)
 df['Lag1'] = df['DateAS'] - df['DateOS']
df['Lag1'] = df['Lag1'].map(lambda x: x.days//7)
 df['Lag1'] = df['Lag1']/52.0
df['Lag1'] = df['Lag1'].astype('float32')
 df['Lag1'].fillna(0,inplace=True)
 if data!=0:
if data==1:
df['Lag5'] = datetime(2018,7,26)- df['DateAS']
elif data==2:
df['Lag5'] = datetime(2018,9,27)- df['DateAS']
elif data==3:
df['Lag5'] = datetime(2018,10,27)- df['DateAS']
df['Lag5'] = df['Lag5'].map(lambda x: x.days//1)
 df.loc[ df['Lag5']<0, 'Lag5' ] = 0
df['Lag5'] = df['Lag5']/365.0
df['Lag5'] = df['Lag5'].astype('float32')
 df['Lag5'].fillna(0,inplace=True)
 del df['DateAS'], df['DateOS']
del datedictAS, datedictOS
x=gc.collect()
for col in NE:
nm = col+'_NE'
df[nm] = df[col].astype('float32')
 df[nm] /= np.std(df[nm])
 new = list(set(df.columns)-set(old))
ret = []
for x in new:
if str(df[x].dtype)=='category':
if add==1: OHE.append(x)
 else:
ret.append(x)
 df[x].fillna(df[x].mean() ,inplace=True)
 if verbose==1:
print('Engineered',len(new),'new features!')
 return ret<categorify>
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 miss_cols = missing.iloc[:20].index
missing.iloc[:20]
FE = []
for col in df_train.columns:
if col=='HasDetections': continue
if df_train[col].nunique() >10:
FE.append(col)
 NUM = makeNew(df_train,verbose=0,add=1,data=1)
 makeNew(df_test,verbose=0,data=2)
 print('Engineered '+str(len(NUM)) +' variables(including NE)')
 ct = len(NUM)+1; cnew = ct
for x in FE:
NUM += encode_FE_lg(df_train,x,verbose=0)
 encode_FE_lg(df_test,x,verbose=0)
 ct += 1
print('Frequency encoded '+str(len(NUM)-cnew)+' variables')
 inps={}; tt = 0
for col in OHE:
factor_data(df_train,df_test,col)
 d = encode_CE(df_train,col,0.001,1)[1]
encode_CE_test(df_test,col,d)
 inps[col] = factor_data(df_train,df_test,col)
 tt += inps[col]
reduce_memory(df_train,col)
 reduce_memory(df_test,col)
 ct += 1
for x in np.unique(NE+MM):
del df_train[x]
if x!='AvSigVersion': del df_test[x]
x = gc.collect()
mm = round(df_train.memory_usage(deep=True ).sum() / 1024**2)
 mm2 = round(df_test.memory_usage(deep=True ).sum() / 1024**2)
 print('Encoded '+str(len(NUM)) +' non-CE variables and '+str(len(OHE)) +' CE containing '+str(tt)+' unique values into '+str(mm)+' Mb memory')
 print('Test memory is '+str(mm2)+' Mb' )<train_model>
for i in miss_cols[:7]:
whole_df.loc[:, i] = whole_df.loc[:, i].fillna(whole_df.loc[:, i].mode() [0] )
class printAUC(callbacks.Callback):
def __init__(self, X_train, y_train, inps, fes, X_val, y_val, k, ee):
super(printAUC, self ).__init__()
self.bestAUC = 0
self.X_train = X_train
self.y_train = y_train
self.inps = inps
self.fes = fes
self.X_val = X_val
self.y_val = y_val
self.k = k
self.ee = ee
def on_epoch_end(self, epoch, logs={}):
pred = self.model.predict([self.X_train[col] for col in self.inps] + [self.X_train[self.fes]])
 aucTR = roc_auc_score(self.y_train, pred)
 pred = self.model.predict([self.X_val[col] for col in self.inps] + [self.X_val[self.fes]])
 auc = roc_auc_score(self.y_val, pred)
 print("Train AUC: " + str(round(aucTR,5)) +" - Validation AUC: " + str(round(auc,5)))
 if(self.bestAUC < auc):
self.bestAUC = auc
self.model.save("bestNet"+str(self.k)+".h5", overwrite=True)
 return<define_variables>
total = whole_df.isnull().sum().sort_values(ascending = False)
 percent =(whole_df.isnull().sum() /whole_df.isnull().count() ).sort_values()
missing = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])
 print('Total Missing Values')
 missing.iloc[:20]
groups = [ ['CountryIdentifier','CityIdentifier','OrganizationIdentifier','GeoNameIdentifier',
'LocaleEnglishNameIdentifier','Census_OSInstallLanguageIdentifier','Census_OSUILocaleIdentifier',
'Wdft_RegionIdentifier'],
['DefaultBrowsersIdentifier', 'AVProductStatesIdentifier', 'AVProductsInstalled', 'AVProductsEnabled',
'IsProtected', 'SMode', 'IeVerIdentifier', 'SmartScreen', 'Firewall','Census_IsSecureBootEnabled',
'Census_IsWIMBootEnabled','Wdft_IsGamer','Census_OSWUAutoUpdateOptionsName','Census_GenuineStateName',
'AppVersion2'],
['Processor','Census_MDC2FormFactor','Census_DeviceFamily','Census_ProcessorCoreCount','Census_ProcessorClass',
'Census_PrimaryDiskTypeName','Census_HasOpticalDiskDrive','Census_TotalPhysicalRAM','Census_ChassisTypeName',
'Census_InternalPrimaryDiagonalDisplaySizeInInches', 'Census_InternalPrimaryDisplayResolutionHorizontal',
'Census_InternalPrimaryDisplayResolutionVertical', 'Census_PowerPlatformRoleName', 'Census_InternalBatteryType',
'Census_InternalBatteryNumberOfCharges','Census_IsTouchEnabled','Census_IsPenCapable',
'Census_IsAlwaysOnAlwaysConnectedCapable'],
['Census_OEMNameIdentifier', 'Census_OEMModelIdentifier', 'Census_ProcessorManufacturerIdentifier',
'Census_ProcessorModelIdentifier','Census_FirmwareManufacturerIdentifier', 'Census_FirmwareVersionIdentifier']
]<prepare_x_and_y>
whole_df.isnull().values.any()
df_train_Y = df_train['HasDetections']
del df_train['HasDetections']
x=gc.collect()
chunk = len(df_train)//5
idx = range(chunk*0,chunk//2)
 idx2 = range(chunk//2,chunk)
 idx3 = range(chunk,chunk*3)
 idx4 = range(chunk*3,chunk*5)
 X_val1 = df_train.loc[idx]
Y_val1 = df_train_Y.loc[idx]
X_val2 = df_train.loc[idx2]
Y_val2 = df_train_Y.loc[idx2]
X_train1 = df_train.loc[idx3]
Y_train1 = df_train_Y.loc[idx3]
X_train2 = df_train.loc[idx4]
Y_train2 = df_train_Y.loc[idx4]
del df_train, df_train_Y
x=gc.collect()<categorify>
ord_fields = ['MSSubClass','ExterQual','LotShape','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1',
'BsmtFinType2','HeatingQC','Functional','FireplaceQu','KitchenQual', 'GarageFinish',
'GarageQual','GarageCond','PoolQC','Fence']
orders = [
['20','30','40','45','50','60','70','75','80','85', '90','120','150','160','180','190'],
['Fa','TA','Gd','Ex'],
['Reg','IR1' ,'IR2','IR3'],
['None','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','No','Mn','Av','Gd'],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['Po','Fa','TA','Gd','Ex'],
['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],
['None','Po','Fa','TA','Gd','Ex'],
['Fa','TA','Gd','Ex'],
['None','Unf','RFn','Fin'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Fa','Gd','Ex'],
['None','MnWw','GdWo','MnPrv','GdPrv']]
for i in range(len(orders)) :
ord_en = OrdinalEncoder(categories = {0 : orders[i]})
 whole_df.loc[:, ord_fields[i]] = ord_en.fit_transform(whole_df.loc[:, ord_fields[i]].values.reshape(-1,1))
ins = []; outs = {}
for k in inps.keys() :
x = Input(shape=(1,))
ins.append(x)
 y = np.int(inps[k])
 x = Embedding(y, y, input_length=1 )(x)
 x = Reshape(target_shape=(y,))(x)
 outs[k]=x
all = set(inps.keys())
 used = []
outs2 = []
for k in groups:
g = [outs[x] for x in set(k ).intersection(all)]
used += list(set(k ).intersection(all))
x = concatenate(g)
 s = sum([inps[x] for x in set(k ).intersection(all)])
 x = Dense(s//2,kernel_initializer='he_uniform' )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 outs2.append(x)
 g = [outs[x] for x in all-set(used)]
x = concatenate(g)
 s = sum([inps[x] for x in all-set(used)])
 x = Dense(s//2,kernel_initializer='he_uniform' )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 outs2.append(x)
 x = Input(shape=(len(NUM),))
ins.append(x)
 x = Dense(len(NUM)//2,kernel_initializer='he_uniform' )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 x = concatenate(outs2+[x])
 x = Dense(100,kernel_initializer='he_uniform' )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 x = Dense(100,kernel_initializer='he_uniform' )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 x = Dense(100,kernel_initializer='he_uniform' )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Activation('elu' )(x)
 x = Dense(1,activation='sigmoid' )(x)
 model = Model(inputs=ins, outputs=x)
 model.compile(optimizer=Adam(lr=0.01), loss='binary_crossentropy', metrics=['accuracy'])
<train_model>
cat_df.nunique()
epochs=10
batch=256
for k in range(epochs):
model.fit([X_train1[col] for col in OHE] + [X_train1[NUM]],Y_train1,
batch_size=batch, epochs = 1, verbose=2, callbacks=[
printAUC(X_train1, Y_train1, OHE, NUM, X_val1, Y_val1, 0, k)],
validation_data =([X_val1[col] for col in OHE] + [X_val1[NUM]],Y_val1))
model.fit([X_train2[col] for col in OHE] + [X_train2[NUM]],Y_train2,
batch_size=batch, epochs = 1, verbose=2, callbacks=[
printAUC(X_train2, Y_train2, OHE, NUM, X_val2, Y_val2, 0, k)],
validation_data =([X_val2[col] for col in OHE] + [X_val2[NUM]],Y_val2))
X_train1['HasDetections'] = Y_train1
X_train1 = X_train1.sample(frac=1)
 Y_train1 = X_train1['HasDetections']
del X_train1['HasDetections']
X_train2['HasDetections'] = Y_train2
X_train2 = X_train2.sample(frac=1)
 Y_train2 = X_train2['HasDetections']
del X_train2['HasDetections']
x=gc.collect()<drop_column>
cat_cols = cat_df.columns
for one in cat_cols:
lab_enc = LabelEncoder()
lab_enc.fit(list(whole_df[one].values))
whole_df[one] = lab_enc.transform(list(whole_df[one].values))
del model
del X_train1, Y_train1, X_val1, Y_val1
del X_train2, Y_train2, X_val2, Y_val2
del ins, outs, outs2, x
x = gc.collect()<predict_on_test>
lam = 0.15
train_log_SP = boxcox1p(train_salePrice, lam )
pred = np.zeros(( len(df_test),1))
print('Predicting test...')
 model = load_model('bestNet0.h5')
 idx = 0; chunk = 1000000
if Debug: chunk = 5000
ct2 = 1;
while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred[idx] += model.predict([df_test.iloc[idx][col] for col in OHE] + [df_test.iloc[idx][NUM]])
 print(' part '+str(ct2)+' done')
 ct2 += 1
idx = idx2
del model
x = gc.collect()<feature_engineering>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
for feat in skewed_features:
whole_df[feat] = boxcox1p(whole_df[feat], lam )
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_test['HasDetections'] = pred
df_test['X'] = df_test['Date'] - datetime(2018,11,20,4,0)
 df_test['X'] = df_test['X'].map(lambda x: x.total_seconds() /86400)
 df_test['X'].fillna(0,inplace=True)
 s = 5.813888
df_test['F'] = 1.0
df_test['F'] = 1 - df_test['X']/s
df_test.loc[df_test['X']<=0,'F'] = 1.0
df_test.loc[df_test['X']>s,'F'] = 0
df_test['HasDetections'] *= df_test['F']
pred = df_test['HasDetections']<save_to_csv>
ntrain = len(train.index)
 ntrain
print('Writing submission file...')
 if Debug:
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv', nrows=10000)
 else:
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submit['HasDetections'] = pred
submit.to_csv('submission.csv', index=False)
 print('Done!' )<load_from_csv>
train = whole_df.iloc[ : ntrain]
test = whole_df.iloc[ntrain : ]
y = train_log_SP
X = train
dtypes = {}
dtypes['MachineIdentifier'] = 'str'
dtypes['AvSigVersion'] = 'category'
dtypes['HasDetections'] = 'int8'
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', usecols=list(dtypes.keys()), dtype=dtypes)
 print('Loaded',len(df_train),'rows of train.CSV!')
 df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=list(dtypes.keys())[0:-1], dtype=dtypes)
 print('Loaded',len(df_test),'rows of test.CSV!')
 df_test2 = pd.read_csv('.. /input/malware-detection-stacking-2-0/submission_kfold.csv')
 print('Loaded',len(df_test),'rows of submission_kfold.csv!')
 datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_train['Date'] = df_train['AvSigVersion'].map(datedictAS)
 df_test2 = pd.merge(df_test2, df_test, on='MachineIdentifier', how='left')
 df_test2['AvSigVersion2'] = df_test2['AvSigVersion'].map(lambda x: np.int(x.split('.')[1]))<feature_engineering>
from sklearn.model_selection import train_test_split
df_test2.loc[(df_test2['AvSigVersion2']==275)|(df_test2['AvSigVersion2']==273),'HasDetections'] *= 0.8<feature_engineering>
from sklearn.model_selection import train_test_split
df_test2.loc[ df_test2['Date']>datetime(2018,11,21,0,0),'HasDetections'] *= 0.4
df_test2.loc[ df_test2['Date']>datetime(2018,11,24,0,0),'HasDetections'] *= 0.4<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0 )
df_test2[['MachineIdentifier','HasDetections']].to_csv('PrivateLeaderboard.csv', index=False )<import_modules>
from sklearn.linear_model import LinearRegression
import pandas as pd, numpy as np, os, gc
import matplotlib.pyplot as plt
from datetime import datetime
from datetime import timedelta<load_from_csv>
lm = LinearRegression()
dtypes = {}
dtypes['MachineIdentifier'] = 'str'
dtypes['AvSigVersion'] = 'category'
dtypes['HasDetections'] = 'int8'
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', usecols=list(dtypes.keys()), dtype=dtypes)
 print('Loaded',len(df_train),'rows of train.CSV!')
 df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=list(dtypes.keys())[0:-1], dtype=dtypes)
 print('Loaded',len(df_test),'rows of test.CSV!')
 df_test2 = pd.read_csv(".. /input/msft-malware-my-best-submission-on-the-private-lb/xdeepfm_submission_CV0.737726_YTS_KnlV32_publicLB0pt688_private_LB0pt662.csv")
 print('Loaded',len(df_test),'rows of super_blend.csv!')
 datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_train['Date'] = df_train['AvSigVersion'].map(datedictAS)
 df_test2 = pd.merge(df_test2, df_test, on='MachineIdentifier', how='left')
 df_test2['AvSigVersion2'] = df_test2['AvSigVersion'].map(lambda x: np.int(x.split('.')[1]))<feature_engineering>
lm.fit(X_train, y_train )
df_test2.loc[(df_test2['AvSigVersion2']==275)|(df_test2['AvSigVersion2']==273),'HasDetections'] *= 0.6<feature_engineering>
predictions = lm.predict(X_test )
df_test2.loc[ df_test2['Date']>datetime(2018,11,21,0,0),'HasDetections'] *= 0.6<save_to_csv>
print('RMSE: ', np.sqrt(metrics.mean_squared_error(y_test,predictions)) )
df_test2[['MachineIdentifier','HasDetections']].to_csv('PrivateLeaderboard.csv', index=False )<load_from_csv>
metrics.explained_variance_score(y_test,predictions )
oof = pd.read_csv(".. /input/microsoft-outputs/val_kfold.csv")['0']
oof_2 = pd.read_csv(".. /input/microsoft-outputs/val_stratfold.csv")['0']
predictions = pd.read_csv(".. /input/microsoft-outputs/output_Kfold.csv")["HasDetections"]
predictions_2 = pd.read_csv(".. /input/microsoft-outputs/output_stratfold.csv")["HasDetections"]<concatenate>
lm_test = LinearRegression()
train_stack = np.vstack([oof,oof_2] ).transpose()
test_stack = np.vstack([predictions, predictions_2] ).transpose()<define_variables>
lm_test.fit(X,y )
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}<load_from_csv>
test_predictions = lm_test.predict(test )
print('Download Train and Test Data.
 ')
 train = reduce_mem_usage(pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', dtype=dtypes, low_memory=True))
train['MachineIdentifier'] = train.index.astype('uint32')
 test = reduce_mem_usage(pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', dtype=dtypes, low_memory=True))
test['MachineIdentifier'] = test.index.astype('uint32' )<prepare_x_and_y>
test_predictions = inv_boxcox1p(test_predictions, lam )
<prepare_x_and_y><EOS> 
solution = pd.DataFrame({"id": test_ID, "SalePrice": test_predictions})
 solution.to_csv("House_Solutions.csv", index = False )