Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<compute_test_metric>
warnings.filterwarnings("ignore")
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
score = model.evaluate(X_val, y_val, verbose = 0)
 print('Test loss: {}%'.format(score[0] * 100))
print('Test accuracy: {}%'.format(score[1] * 100))
print("MLP Error: %.2f%%" %(100 - score[1] * 100))<choose_model_class>
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float32',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int16',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float64',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float32',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float32',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float64',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float64',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float32',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float32',
'Census_InternalPrimaryDisplayResolutionVertical': 'float32',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float64',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def get_mlpv2() :
return Sequential([
Dense(512, input_dim=num_pixels, activation='relu'),
Dropout(0.3),
Dense(256, activation='relu'),
Dropout(0.2),
Dense(128, activation='relu'),
Dense(num_classes, kernel_initializer='normal', activation='softmax')
 ] )<choose_model_class>
%%time
train = pd.read_csv('/kaggle/input/train.csv', dtype= dtypes ,nrows = 2000000)
 test = pd.read_csv('/kaggle/input/test.csv' , dtype = dtypes)
 train = reduce_mem_usage(train)
 test = reduce_mem_usage(test)
 score = []
model = get_mlpv2()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()<train_model>
print('Number of columns in train data is ->',train.shape[1])
 float_list = []
int_list = []
category_list = []
for f in train.columns:
if train[f].dtype =='int64' or train[f].dtype =='int8' or train[f].dtype =='int16' or train[f].dtype =='int32':
int_list.append(f)
 elif train[f].dtype =='float64' or train[f].dtype =='float32' or train[f].dtype =='float16':
float_list.append(f)
 else:
category_list.append(f)
 print('Number of columns in train data of Float Type is ->', len(float_list))
print('Number of columns in train data of Int Type is ->', len(int_list))
print('Number of columns in train data of Category Type is ->', len(category_list))
learning_history = model.fit(X_train, y_train,
batch_size = 1024, epochs = 40, verbose = 2,
validation_data=(X_val, y_val)) ;<compute_test_metric>
print(train['HasDetections'].value_counts() /train.shape[0]*100 )
score = model.evaluate(X_val, y_val, verbose = 0)
 print('Test loss: {}%'.format(score[0] * 100))
print('Test accuracy: {}%'.format(score[1] * 100))
print("MLP Error: %.2f%%" %(100 - score[1] * 100))<choose_model_class>
print('Creating data info of train data and test data.....')
 stats = []
for f in train.columns:
stats.append(( f ,train[f].nunique() , train[f].isnull().sum() *100/train.shape[0] ,
train[f].value_counts(normalize = True , dropna = True ).values[0]*100 ,
train[f].value_counts(normalize = True , dropna = False ).values[0]*100 ,
train[f].dtype))
stats_df = pd.DataFrame(stats , columns= ['Columns', 'Number of Unique Values' , 'Percentage of Data missing in train' ,
'Percentage of Biggest Category of dropna True' ,
'Percentage of Biggest Category of dropna False' ,
'Columns DataType'])
 display(stats_df.sort_values('Percentage of Data missing in train', ascending=False))
def get_triplecnn() :
return Sequential([
Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape = input_shape),
Conv2D(32, kernel_size=(3, 3), activation='relu'),
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(64, kernel_size=(3, 3), activation='relu'),
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(128, kernel_size=(3, 3), activation='relu'),
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Flatten() ,
Dense(256, activation='relu'),
Dropout(0.5),
Dense(num_classes, activation = "softmax")
 ] )<train_model>
col_temp = ['EngineVersion','AppVersion','AvSigVersion', 'OsBuildLab' , 'OsVer' , 'Census_OSVersion']
for f in col_temp:
print('Null Values in column ',f,' is ->',train[f].isnull().sum() )
learning_history = model.fit(X_train, y_train,
batch_size = 128,
epochs = 50,
verbose = 1,
validation_data =(X_val, y_val))<compute_test_metric>
def add_category_1(data):
data['EngineVersion_0'] = data['EngineVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['EngineVersion_1'] = data['EngineVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['EngineVersion_2'] = data['EngineVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['EngineVersion_3'] = data['EngineVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['AppVersion_0'] = data['AppVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['AppVersion_1'] = data['AppVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['AppVersion_2'] = data['AppVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['AppVersion_3'] = data['AppVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['AvSigVersion_0'] = data['AvSigVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['AvSigVersion_1'] = data['AvSigVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['AvSigVersion_2'] = data['AvSigVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['AvSigVersion_3'] = data['AvSigVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['OsBuildLab_0'] = data['OsBuildLab'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['OsBuildLab_1'] = data['OsBuildLab'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['OsBuildLab_2'] = data['OsBuildLab'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['OsBuildLab_3'] = data['OsBuildLab'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['OsVer_0'] = data['OsVer'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['OsVer_1'] = data['OsVer'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['OsVer_2'] = data['OsVer'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['OsVer_3'] = data['OsVer'].apply(lambda x : x.split('.')[3] ).astype('category')
 data['Census_OSVersion_0'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[0] ).astype('category')
 data['Census_OSVersion_1'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[1] ).astype('category')
 data['Census_OSVersion_2'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[2] ).astype('category')
 data['Census_OSVersion_3'] = data['Census_OSVersion'].apply(lambda x : x.split('.')[3] ).astype('category')
 return data
score = model.evaluate(X_val, y_val, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])
 print("CNN Error: %.2f%%" %(100-score[1]*100))<predict_on_test>
remove_list = []
for f in train.columns:
if train[f].isnull().sum() *100/train.shape[0] > 98 :
remove_list.append(f)
 for f in train.columns:
if train[f].value_counts().values[0]*100/train.shape[0] > 98:
remove_list.append(f)
 print(remove_list )
y_pred = model.predict(X_val )<train_model>
train.drop(columns = remove_list , inplace = True)
 test.drop(columns = remove_list , inplace = True )
train_aug = ImageDataGenerator(
 featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
horizontal_flip = False,
vertical_flip = False,
fill_mode = 'nearest',
rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1)
 train_aug.fit(X_train)
 train_gen = train_aug.flow(X_train, y_train, batch_size=64 )<choose_model_class>
cat_col = []
for f in train.columns:
if str(train[f].dtype)== 'category':
cat_col.append(f)
 print(len(cat_col))
print(cat_col )
def get_newtriplecnn() :
return Sequential([
Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape = input_shape),
Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.25),
Flatten() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dropout(0.5),
Dense(256, activation='relu'),
BatchNormalization() ,
Dropout(0.4),
Dense(64, activation='relu'),
BatchNormalization() ,
Dropout(0.3),
Dense(num_classes, activation = "softmax")
 ] )<choose_model_class>
to_encode = []
for f in cat_col:
if train[f].nunique() >10:
to_encode.append(f)
 print(to_encode )
callbacks1 = [
EarlyStopping(monitor = 'loss', patience = 6),
ReduceLROnPlateau(monitor = 'loss', patience = 3),
ModelCheckpoint('.. /working/model.best.hdf5', save_best_only=True)
 ]<train_model>
to_encode.remove('MachineIdentifier')
 cat_col.remove('MachineIdentifier' )
history = model.fit_generator(( train_gen), epochs = 100,
steps_per_epoch = X_train.shape[0] // 64,
validation_data =(X_val, y_val),
callbacks = callbacks1,
)<load_pretrained>
def frequency_encoder(variable):
df = pd.concat([ train[variable] , test[variable] ] ).value_counts().reset_index()
df = df.merge(df[variable].value_counts().reset_index().reset_index().rename(columns = {'level_0': 'rank_encode' , 'index':variable , variable:'count'}), on =variable , how='left')[['index' , 'rank_encode']]
df.set_index('index' , inplace = True)
 return df.to_dict() ['rank_encode']
model = load_model('.. /working/model.best.hdf5' )<compute_test_metric>
%%time
for f in tqdm_notebook(to_encode):
dictionary = frequency_encoder(f)
 train[f+'_ranking'] = train[f].apply(lambda x: dictionary.get(x , np.nan)).astype('category')
 test[f+'_ranking'] = test[f].apply(lambda x : dictionary.get(x , np.nan)).astype('category')
score = model.evaluate(X_val, y_val, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])
 print("CNN Error: %.2f%%" %(100-score[1]*100))<save_to_csv>
%%time
for f in tqdm_notebook(cat_col):
lb = LabelEncoder()
lb.fit(list(train[f].values)+ list(test[f].values))
train[f] = lb.transform(list(train[f].values))
test[f] = lb.transform(list(test[f].values))
output = model.predict(X_test)
 output = np.argmax(output, axis = 1)
 output = pd.Series(output, name="Label")
 submission = pd.concat([pd.Series(range(1,28001), name = "ImageId"), output], axis = 1)
 submission.to_csv("submission.csv", index=False )<split>
y = train['HasDetections']
train.drop(columns=['HasDetections' , 'MachineIdentifier' ] , inplace = True)
 test_index = list(test['MachineIdentifier'])
 test.drop(columns = ['MachineIdentifier'] , inplace = True)
 train.sort_values('AvSigVersion')
 n_folds = 3
folds = StratifiedKFold(n_splits = n_folds , shuffle = True , random_state = 98)
def load_data(path):
with np.load(path)as f:
x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
return(x_train, y_train),(x_test, y_test)
(x_train1, y_train1),(x_test1, y_test1)= load_data('.. /input/mnist-numpy/mnist.npz' )<categorify>
%%time
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
x_train1 = x_train1 / 255
x_test1 = x_test1 / 255
x_train1 = x_train1.reshape(-1, 28, 28, 1)
 x_test1 = x_test1.reshape(-1, 28, 28, 1)
 y_train1 = y_train1.reshape(y_train1.shape[0], 1)
 y_test1 = y_test1.reshape(y_test1.shape[0], 1 )<concatenate>
params = {'num_leaves': 256,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 5,
'learning_rate': 0.05,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 5,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"lambda_l1": 0.15,
"lambda_l2": 0.15,
"random_state": 42,
"verbosity": -1}
Add_X = np.vstack(( x_train1, x_test1))
Add_y = np.vstack(( y_train1, y_test1))
Add_y = to_categorical(Add_y )<load_from_csv>
test_1 = test[ : int(test.shape[0]/3)]
test_2 = test[int(test.shape[0]/3): int(test.shape[0]/3*2)]
test_3 = test[int(test.shape[0]/3*2): ]
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_train = train.drop(labels = ["label"], axis = 1)
 y_train = train['label']
y_train = to_categorical(y_train)
 X_train /= 255
X_train = X_train.values.reshape(-1, 28, 28, 1 )<train_model>
del test
gc.collect()
add_train_aug = ImageDataGenerator(
 featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
horizontal_flip = False,
vertical_flip = False,
fill_mode = 'nearest',
rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1)
 add_train_aug.fit(Add_X)
 add_train_gen = add_train_aug.flow(Add_X, Add_y, batch_size=64 )<choose_model_class>
def fast_auc(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(preds, dtrain):
return fast_auc(dtrain, preds)
 train_col = train.columns
feature_importance = pd.DataFrame()
pred_1 = np.zeros(test_1.shape[0])
 pred_2 = np.zeros(test_2.shape[0])
 pred_3 = np.zeros(test_3.shape[0])
 feature_importance['columns'] = train.columns
for n_fold ,(train_index , valid_index)in enumerate(folds.split(train , y)) :
gc.collect()
print('Fold ',n_fold , ' Started at ',time.ctime())
 X_train , X_valid = train.iloc[train_index] , train.iloc[valid_index]
y_train , y_valid = y.iloc[train_index] , y.iloc[valid_index]
train_data = lgb.Dataset(X_train[train_col] , label = y_train)
 valid_data = lgb.Dataset(X_valid[train_col] , label = y_valid)
 model = lgb.train(params , train_data , num_boost_round = 1000 , valid_sets = [train_data , valid_data] , verbose_eval =-1, early_stopping_rounds = 200)
 pred = model.predict(X_valid , num_iteration=model.best_iteration)
 pred = list(pred)
 pred_1+=model.predict(test_1[train_col])/n_folds
pred_2+=model.predict(test_2[train_col])/n_folds
pred_3+=model.predict(test_3[train_col])/n_folds
score.append(( r2_score(pred , y_valid), mean_squared_error(pred , y_valid), eval_auc(pred , y_valid)))
 feature_importance['fold_'+str(n_fold+1)] = model.feature_importance()
add_callbacks = [
EarlyStopping(monitor = 'loss', patience = 6),
ReduceLROnPlateau(monitor = 'loss', patience = 3),
ModelCheckpoint('.. /working/additional_model.best.hdf5', save_best_only=True)
 ]<choose_model_class>
display(score)
 feature_importance['avg'] =(feature_importance['fold_1']+feature_importance['fold_2']+feature_importance['fold_3'])/3
feature_importance.head()
<choose_model_class><EOS> 
submission = pd.DataFrame()
submission['MachineIdentifier'] = test_index
submission['HasDetections'] = pred
submission.to_csv('lgb_first.csv' , index = False )