Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<compute_test_metric>
gc.enable()
def score(model, X_train, y_train, X_valid = [], y_valid = []):
rms = sqrt(mean_squared_error(np.square(np.exp(y_train)) , np.square(np.exp(model.predict(X_train)))))
 score = [rms, model.score(X_train, y_train)]
if len(X_valid)!= 0 and len(y_valid)!= 0:
score.append(sqrt(mean_squared_error(np.square(np.exp(y_valid)) , np.square(np.exp(model.predict(X_valid))))))
if model.oob_score:
score.append(model.oob_score_)
 return score
n_train = df_train.shape[0]
def prcs(df, fe=[]):
add_datepart(df, 'Open Date')
 if 'city' in fe:
df = df.drop('City', axis=1)
 if 'outlier' in fe:
df = df.drop(index=16, axis=0)
 if 'MB' in fe:
df['Type'] = df['Type'].replace('MB', 'DT')
 if 'city_group' in fe:
df = df.drop('City Group', axis=1)
 if 'dummies' in fe:
p_cols = [ f'P{n}' for n in range(1,38)]
df = pd.get_dummies(df, columns=p_cols)
 if 'city_group' not in fe:
df = pd.get_dummies(df, columns=['City Group'], drop_first=True)
 df = pd.get_dummies(df, columns=['Type'])
 train_cats(df)
 X, _, _ = proc_df(df, None)
 drop_cols = ['Open Year', 'Open Month', 'Open Week', 'Open Day', 'Open Dayofweek',
'Open Dayofyear', 'Open Is_month_end', 'Open Is_month_start',
'Open Is_quarter_end', 'Open Is_quarter_start', 'Open Is_year_end',
'Open Is_year_start']
X = X.drop(drop_cols, axis=1)
 if 'id' in fe:
X = X.drop('Id', axis=1)
 if 'scale_open' in fe:
X['Open Elapsed'] =(X['Open Elapsed']/1000 ).apply(np.log)
 X_train = X[:n_train]
X_test = X[n_train:]
return X_train, X_test
def train_cv(X, y):
models = []
scores = []
kf = KFold(n_splits=4, random_state=12, shuffle=False)
 for train_index, val_index in kf.split(X):
X_train_ = X.iloc[train_index]
y_train_ = y.iloc[train_index]
X_val_ = X.iloc[val_index]
y_val_ = y.iloc[val_index]
m = RandomForestRegressor(n_jobs=-1, n_estimators=100, max_features=0.5, oob_score=True)
 m.fit(X_train_, y_train_)
 models.append(m)
 scores.append(score(m, X_train_, y_train_, X_val_, y_val_))
return models, np.array(scores ).mean(axis=0)
 def predict(models, X):
f = 1 / len(models)
 pred = 0
for m in models:
pred += f * m.predict(X)
 return pred<prepare_x_and_y>
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
X_train, X_test = prcs(df_joined.copy())
 y_train = df_train['revenue'].copy().apply(np.log )<train_model>
print('Download Train and Test Data.
 ')
 train = reduce_mem_usage(pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True))
train['MachineIdentifier'] = train.index.astype('uint32')
 test = reduce_mem_usage(pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True))
test['MachineIdentifier'] = test.index.astype('uint32')
 gc.collect()
m = RandomForestRegressor(n_jobs=-1, n_estimators=150, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )<save_to_csv>
print('Transform all features to category.
 ')
 for usecol in train.columns.tolist() [1:-1]:
train[usecol] = train[usecol].astype('str')
 test[usecol] = test[usecol].astype('str')
 le = LabelEncoder().fit(
 np.unique(train[usecol].unique().tolist() +
test[usecol].unique().tolist()))
train[usecol] = le.transform(train[usecol])+1
test[usecol] = le.transform(test[usecol])+1
agg_tr =(train 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Train'}, axis=1))
agg_te =(test 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Test'}, axis=1))
agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer' ).replace(np.nan, 0)
 agg = agg[(agg['Train'] > 1000)].reset_index(drop=True)
 agg['Total'] = agg['Train'] + agg['Test']
agg = agg[(agg['Train'] / agg['Total'] > 0.2)&(agg['Train'] / agg['Total'] < 0.8)]
agg[usecol+'Copy'] = agg[usecol]
train[usecol] =(pd.merge(train[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
test[usecol] =(pd.merge(test[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
del le, agg_tr, agg_te, agg, usecol
gc.collect()
y_train = np.array(train['HasDetections'])
 train_ids = train.index
test_ids = test.index
del train['HasDetections'], train['MachineIdentifier'], test['MachineIdentifier']
gc.collect()
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.exp(predict(m, X_test)))
 df_preds.to_csv('submission0.csv', index=True, index_label='Id')
 df_preds.head()<compute_test_metric>
ohe = OneHotEncoder(categories='auto', sparse=True, dtype='uint8' ).fit(train)
 m = 100000
train = vstack([ohe.transform(train[i*m:(i+1)*m])for i in range(train.shape[0] // m + 1)])
 test = vstack([ohe.transform(test[i*m:(i+1)*m])for i in range(test.shape[0] // m + 1)])
 save_npz('train.npz', train, compressed=True)
 save_npz('test.npz', test, compressed=True)
 del ohe, train, test
gc.collect()
models, scores = train_cv(X_train, y_train)
 print(scores )<save_to_csv>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 skf.get_n_splits(train_ids, y_train)
 lgb_test_result_1 = np.zeros(test_ids.shape[0])
 oof_1= np.zeros(train_ids.shape[0])
 counter = 0
print('
LightGBM
')
 for train_index, test_index in skf.split(train_ids, y_train):
print('Fold {}
'.format(counter + 1))
train = load_npz('train.npz')
 X_fit = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
 X_val = vstack([train[test_index[i*m:(i+1)*m]] for i in range(test_index.shape[0] // m + 1)])
 X_fit, X_val = csr_matrix(X_fit, dtype='float32'), csr_matrix(X_val, dtype='float32')
 y_fit, y_val = y_train[train_index], y_train[test_index]
del train
gc.collect()
lgb_model = lgb.LGBMClassifier(max_depth=-1,
n_estimators=30000,
learning_rate=0.05,
num_leaves=2**12-1,
colsample_bytree=0.28,
objective='binary',
n_jobs=-1)
 lgb_model.fit(X_fit, y_fit, eval_metric='auc',
eval_set=[(X_val, y_val)],
verbose=100, early_stopping_rounds=100)
 oof_1[test_index] += lgb_model.predict_proba(X_val)[:,1]
del X_fit, X_val, y_fit, y_val, train_index, test_index
gc.collect()
test = load_npz('test.npz')
 test = csr_matrix(test, dtype='float32')
 lgb_test_result_1 += lgb_model.predict_proba(test)[:,1]
counter += 1
del test
gc.collect()
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.exp(predict(models, X_test)))
 df_preds.to_csv('submission1.csv', index=True, index_label='Id')
 df_preds.head()<prepare_x_and_y>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] =(lgb_test_result_1 / counter)
 submission.to_csv('lgb_submission.csv', index=False )