Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<train_model>
try:
cdll.LoadLibrary("libc.so.6")
 libc = CDLL("libc.so.6")
 libc.malloc_trim(0)
 except(OSError, AttributeError):
libc = None
__old_del = getattr(pd.DataFrame, '__del__', None)
 def __new_del(self):
if __old_del:
__old_del(self)
 libc.malloc_trim(0)
 if libc:
print('Applying monkeypatch for pd.DataFrame.__del__', file=sys.stderr)
 pd.DataFrame.__del__ = __new_del
else:
print('Skipping monkeypatch for pd.DataFrame.__del__: libc or malloc_trim() not found', file=sys.stderr )
history = model.fit_generator(( add_train_gen), epochs = 100,
steps_per_epoch = x_train1.shape[0] // 64,
validation_data =(X_val, y_val),
callbacks = add_callbacks,
)<load_pretrained>
Debug = False
def load(x):
ignore = ['MachineIdentifier']
if x in ignore: return False
else: return True
if Debug:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load,nrows=10000)
 else:
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
if Debug:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load,nrows=10000)
 else:
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
 print('Loaded',len(df_train),'rows of TRAIN and',len(df_test),'rows of TEST' )
model = load_model('additional_model.best.hdf5' )<compute_test_metric>
def encode_FE(df,col):
vc = df[col].value_counts(dropna=False, normalize=True ).to_dict()
nm = col+'_FE'
df[nm] = df[col].map(vc)
 df[nm] = df[nm].astype('float32')
 return [nm]
def encode_FE2(df1, df2, col):
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=False, normalize=True ).to_dict()
nm = col+'_FE2'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 return [nm]
def factor_data(df_train, df_test, col):
df_comb = pd.concat([df_train[col],df_test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 df_comb += 1
df_comb = np.where(df_comb==0, df_comb.max() +1, df_comb)
 df_train[col] = df_comb[:len(df_train)]
df_test[col] = df_comb[len(df_train):]
del df_comb
def reduce_memory(df,col):
mx = df[col].max()
if mx<256:
df[col] = df[col].astype('uint8')
 elif mx<65536:
df[col] = df[col].astype('uint16')
 else:
df[col] = df[col].astype('uint32')
 def relax_data(df_train, df_test, col):
cv1 = pd.DataFrame(df_train[col].value_counts().reset_index().rename({col:'train'},axis=1))
cv2 = pd.DataFrame(df_test[col].value_counts().reset_index().rename({col:'test'},axis=1))
cv3 = pd.merge(cv1,cv2,on='index',how='outer')
 factor = len(df_test)/len(df_train)
 cv3['train'].fillna(0,inplace=True)
 cv3['test'].fillna(0,inplace=True)
 cv3['remove'] = False
cv3['remove'] = cv3['remove'] |(cv3['train'] < len(df_train)/10000)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] < cv3['test']/3)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] > 3*cv3['test'])
 cv3['new'] = cv3.apply(lambda x: x['index'] if x['remove']==False else 0,axis=1)
 cv3['new'],_ = cv3['new'].factorize(sort=True)
 cv3.set_index('index',inplace=True)
 cc = cv3['new'].to_dict()
df_train[col] = df_train[col].map(cc)
 reduce_memory(df_train,col)
 df_test[col] = df_test[col].map(cc)
 reduce_memory(df_test,col)
 def display_memory(df_train, df_test):
print(len(df_train),'rows of training data use',df_train.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 print(len(df_test),'rows of test data use',df_test.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 def categorize(df_train, df_test, cols):
for col in cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )
score = model.evaluate(X_val, y_val, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])
 print("CNN Error: %.2f%%" %(100-score[1]*100))<save_to_csv>
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_train['DateAS'] = df_train['AvSigVersion'].map(datedictAS)
 df_test['DateAS'] = df_test['AvSigVersion'].map(datedictAS)
 datedictOS = np.load('.. /input/malware-timestamps-2/OSVersionTimestamps.npy')[() ]
df_train['DateOS'] = df_train['Census_OSVersion'].map(datedictOS)
 df_test['DateOS'] = df_test['Census_OSVersion'].map(datedictOS)
 df_train['AppVersion2'] = df_train['AppVersion'].map(lambda x: np.int(x.split('.')[1]))
df_test['AppVersion2'] = df_test['AppVersion'].map(lambda x: np.int(x.split('.')[1]))
df_train['Lag1'] = df_train['DateAS'] - df_train['DateOS']
df_train['Lag1'] = df_train['Lag1'].map(lambda x: x.days//7)
 df_test['Lag1'] = df_test['DateAS'] - df_test['DateOS']
df_test['Lag1'] = df_test['Lag1'].map(lambda x: x.days//7)
 df_train['Lag5'] = datetime(2018,7,26)- df_train['DateAS']
df_train['Lag5'] = df_train['Lag5'].map(lambda x: x.days//1)
 df_train.loc[ df_train['Lag5']<0, 'Lag5' ] = 0
df_test['Lag5'] = datetime(2018,9,27)- df_test['DateAS']
df_test['Lag5'] = df_test['Lag5'].map(lambda x: x.days//1)
 df_test.loc[ df_test['Lag5']<0, 'Lag5' ] = 0
df_train['Lag5'] = df_train['Lag5'].astype('float32')
 df_test['Lag5'] = df_test['Lag5'].astype('float32')
 df_train['driveA'] = df_train['Census_SystemVolumeTotalCapacity'].astype('float')/df_train['Census_PrimaryDiskTotalCapacity'].astype('float')
 df_test['driveA'] = df_test['Census_SystemVolumeTotalCapacity'].astype('float')/df_test['Census_PrimaryDiskTotalCapacity'].astype('float')
 df_train['driveA'] = df_train['driveA'].astype('float32')
 df_test['driveA'] = df_test['driveA'].astype('float32')
 df_train['driveB'] = df_train['Census_PrimaryDiskTotalCapacity'].astype('float')- df_train['Census_SystemVolumeTotalCapacity'].astype('float')
 df_test['driveB'] = df_test['Census_PrimaryDiskTotalCapacity'].astype('float')- df_test['Census_SystemVolumeTotalCapacity'].astype('float')
 df_train['driveB'] = df_train['driveB'].astype('float32')
 df_test['driveB'] = df_test['driveB'].astype('float32')
 cols6=['Lag1']
cols8=['Lag5','driveB','driveA']
del df_train['DateAS'], df_train['DateOS']
del df_test['DateAS'], df_test['DateOS']
del datedictAS, datedictOS
x=gc.collect()
output = model.predict(X_test)
 output = np.argmax(output, axis = 1)
 output = pd.Series(output, name="Label")
 submission = pd.concat([pd.Series(range(1,28001), name = "ImageId"), output], axis = 1)
 submission.to_csv("bonus_submission.csv", index=False )<import_modules>
cols3 = []
FE = ['Census_OSVersion', 'Census_OSBuildRevision', 'Census_InternalBatteryNumberOfCharges', 'AvSigVersion', 'Lag1']
for col in FE:
cols3 += encode_FE(df_train, col)
 encode_FE(df_test, col)
 FE2 = ['CountryIdentifier', 'Census_InternalBatteryNumberOfCharges']
for col in FE2:
cols3 += encode_FE2(df_train, df_test, col )
print(f'Using Tensorflow Version : {tf.__version__}')
 print(f'Using Keras Version : {keras.__version__}' )<load_from_csv>
CE = ['CountryIdentifier', 'SkuEdition', 'Firewall', 'Census_ProcessorCoreCount', 'Census_OSUILocaleIdentifier', 'Census_FlightRing']
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head()<prepare_x_and_y>
cols = [x for x in df_train.columns if x not in ['HasDetections']+CE+cols3+cols6+cols8]
cols2 = CE; ct = 1
for col in cols.copy() :
rate = df_train[col].value_counts(normalize=True, dropna=False ).values[0]
if rate > 0.98:
del df_train[col]
del df_test[col]
cols.remove(col)
 ct += 1
rmv3=['Census_OSSkuName', 'OsVer', 'Census_OSArchitecture', 'Census_OSInstallLanguageIdentifier']
rmv4=['SMode']
for col in rmv3+rmv4:
del df_train[col]
del df_test[col]
cols.remove(col)
 ct +=1
print('Removed',ct,'variables')
 x=gc.collect()
X = train.drop(['label'], axis = 1)
 y = to_categorical(train['label'].values )<feature_engineering>
print('Factorizing...')
 for col in cols+cols2+cols6:
factor_data(df_train, df_test, col)
 print('Relaxing data...')
 for col in cols+cols2: relax_data(df_train, df_test, col)
 print('Optimizing memory...')
 for col in cols+cols2+cols6:
reduce_memory(df_train, col)
 reduce_memory(df_test, col)
 categorize(df_train, df_test, cols2)
 print('Number of variables is',len(cols+cols2+cols3+cols6+cols8))
display_memory(df_train, df_test )
X = X / 255.0
X_test = test / 255.0<choose_model_class>
pred_val = np.zeros(len(df_test))
folds = StratifiedKFold(n_splits=5, shuffle=True)
 ct = 0
for idxT, idxV in folds.split(df_train[cols+cols2+cols3+cols6], df_train['HasDetections']):
ct += 1; print('
df_trainA = df_train.loc[idxT]
df_trainB = df_train.loc[idxV]
model = lgb.LGBMClassifier(n_estimators=10000, colsample_bytree=0.5, objective='binary', num_leaves=2048,
max_depth=-1, learning_rate=0.04)
 h=model.fit(df_trainA[cols+cols2+cols3+cols6+cols8], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols+cols2+cols3+cols6+cols8], df_trainB['HasDetections'])], verbose=200,
early_stopping_rounds=100)
 del df_trainA, df_trainB; x=gc.collect()
idx = 0; ct2 = 1; chunk = 1000000
print('Predicting test...')
 while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] += model.predict_proba(df_test.iloc[idx][cols+cols2+cols3+cols6+cols8])[:,1]
ct2 += 1; idx = idx2
datagen = ImageDataGenerator(rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )<compute_test_metric>
del df_train; x=gc.collect()
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',
usecols=['MachineIdentifier','AvSigVersion'], nrows=len(pred_val))
datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_test['HasDetections'] = pred_val / 5.0
df_test['X'] = df_test['Date'] - datetime(2018,11,20,4,0)
 df_test['X'] = df_test['X'].map(lambda x: x.total_seconds() /86400)
 df_test['X'].fillna(0,inplace=True)
 s = 5.813888
df_test['F'] = 1.0
df_test['F'] = 1 - df_test['X']/s
df_test.loc[df_test['X']<=0,'F'] = 1.0
df_test.loc[df_test['X']>s,'F'] = 0
df_test['HasDetections'] *= df_test['F']
<choose_model_class><EOS> 
df_test[['MachineIdentifier','HasDetections']].to_csv('submission.csv', index=False )