Kaggle competition: Microsoft Malware Prediction
def missing_value(df):

 number = df.isnull().sum().sort_values(ascending=False)
 number = number[number > 0]
percentage = df.isnull().sum() *100 /df.shape[0]
percentage = percentage[percentage > 0].sort_values(ascending=False)
 return pd.concat([number,percentage],axis=1,keys=["Total","Percentage"])
 missing_value(df_all )<feature_engineering>
gc.enable()
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
train_path='/kaggle/input/microsoft-malware-prediction/train.csv'
test_path='/kaggle/input/microsoft-malware-prediction/test.csv'
train = pd.read_csv(train_path, dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv(test_path, dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 gc.collect()
df_all["OpenMonth"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[0]))
df_all["OpenYear"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[-1]))
df_all["Open Day"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[1]))
df_all = df_all.drop(["Open Date"],axis=1 )<count_values>
train.isna().sum() [train.isna().sum() > 0] * 100 / len(train )
bias_feat = []
for feat in df_all.columns:
counts = df_all[feat].value_counts().iloc[0]
if counts*100 / len(df_all)>99.94:
bias_feat.append(feat)
 bias_feat<categorify>
y_train = train['HasDetections']
del train['HasDetections']
for ids, usecol in enumerate(train.columns.tolist() [1:-1]):
print(ids, usecol)
 train[usecol] = train[usecol].astype('str')
 test[usecol] = test[usecol].astype('str')
 le = LabelEncoder().fit(
 np.unique(train[usecol].unique().tolist() +
test[usecol].unique().tolist()))
train[usecol] = le.transform(train[usecol])+1
test[usecol] = le.transform(test[usecol])+1
agg_tr =(train 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Train'}, axis=1))
agg_te =(test 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Test'}, axis=1))
agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer' ).replace(np.nan, 0)
 agg = agg[(agg['Train'] > 1000)].reset_index(drop=True)
 agg['Total'] = agg['Train'] + agg['Test']
agg = agg[(agg['Train'] / agg['Total'] > 0.2)&(agg['Train'] / agg['Total'] < 0.8)]
agg[usecol+'Copy'] = agg[usecol]
train[usecol] =(pd.merge(train[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
test[usecol] =(pd.merge(test[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
del le, agg_tr, agg_te, agg
gc.collect()
<train_model><EOS> 
del train['MachineIdentifier'], test['MachineIdentifier']
gc.collect()
skf = StratifiedKFold(n_splits=6, shuffle=True, random_state=42)
 skf.get_n_splits(train.index, y_train)
 test_result = np.zeros(len(test))
for train_index, test_index in skf.split(train.index, y_train):
X_fit, X_val = train[train.index.isin(train_index)], train[train.index.isin(test_index)]
y_fit, y_val = y_train[y_train.index.isin(train_index)], y_train[y_train.index.isin(test_index)]
model = lgb.LGBMClassifier(max_depth=-1,
n_estimators=30000,
learning_rate=0.1,
colsample_bytree=0.2,
objective='binary',
n_jobs=-1)
 model.fit(X_fit, y_fit, eval_metric='auc', eval_set=[(X_val, y_val)],
verbose=100, early_stopping_rounds=100)
 test_result += model.predict_proba(test)[:,1]
del X_fit, X_val, y_fit, y_val
gc.collect()
to_submit = pd.read_csv('/kaggle/input/microsoft-malware-prediction/sample_submission.csv')
 to_submit['HasDetections'] = test_result / 6
to_submit.to_csv('fin.csv', index=False )