Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<define_search_model>
!git clone https://github.com/guoday/ctrNet-tool.git
!cp -r ctrNet-tool/*./
!rm -r ctrNet-tool data.git
!ls -all
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<define_variables>
import ctrNet
import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from src import misc_utils as utils
import os
import gc
import random
EPOCHS = 45
BATCH_SIZE = 16 * tpu_strategy.num_replicas_in_sync<split>
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
print('Loading Train and Test Data.
 ')
 train = pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 test['HasDetections']=[0]*len(test )
scores, History = [], []
pred = np.zeros(shape =(len(test), 10))
cv = KFold(n_splits=15, shuffle = True, random_state = 42)
 for fold,(train_index, val_index)in enumerate(cv.split(X)) :
start = default_timer()
K.clear_session()
tf.tpu.experimental.initialize_tpu_system(tpu)
 X_train , y_train = X[train_index], y[train_index]
X_val, y_val = X[val_index], y[val_index]
Train_x, Train_y = None, None
batch = 0
for x_batch, y_batch in datagen.flow(X_train, y_train,
batch_size=BATCH_SIZE):
if batch == 0:
Train_x, Train_y = x_batch, y_batch
elif batch >= X.shape[0] // BATCH_SIZE:
break
else:
Train_x = np.concatenate(( Train_x, x_batch))
Train_y = np.concatenate(( Train_y, y_batch))
batch += 1
with tpu_strategy.scope() :
model = build_model()
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 sv = ModelCheckpoint(f'Model Fold {fold}.h5', monitor='val_accuracy',
save_best_only=True, mode='max')
 history = model.fit(Train_x, Train_y, batch_size = BATCH_SIZE,
epochs = EPOCHS, verbose = 0, callbacks=[annealer, sv],
steps_per_epoch = X_train.shape[0]//BATCH_SIZE,
validation_data =(X_val, y_val))
History.append(history.history)
 model = load_model(f'Model Fold {fold}.h5')
 score = model.evaluate(X_val, y_val, verbose = 0)[1]
scores.append(score)
 pred += model.predict(X_test)
 time = round(default_timer() - start, 4)
 print(f'[INFO] Fold {fold + 1} val_accuracy : {round(score, 4)} - Time : {time} s')
 print()
print(f'[INFO] Mean CV scores : {round(sum(scores)/len(scores), 4)}' )<set_options>
def make_bucket(data,num=10):
data.sort()
bins=[]
for i in range(num):
bins.append(data[int(len(data)*(i+1)//num)-1])
 return bins
float_features=['Census_SystemVolumeTotalCapacity','Census_PrimaryDiskTotalCapacity']
for f in float_features:
train[f]=train[f].fillna(1e10)
 test[f]=test[f].fillna(1e10)
 data=list(train[f])+list(test[f])
 bins=make_bucket(data,num=50)
 train[f]=np.digitize(train[f],bins=bins)
 test[f]=np.digitize(test[f],bins=bins)
 train, dev,_,_ = train_test_split(train,train['HasDetections'],test_size=0.02, random_state=2019)
 features=train.columns.tolist() [1:-1]
%%HTML
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {font-family: Arial;}
img {max-width:100%; height:auto} 
.tab {
overflow: hidden;
border: 1px solid
background-color:
} 
.tab button {
background-color:
float: left;
border: none;
outline: none;
cursor: pointer;
padding: 14px 16px;
transition: 0.3s;
font-size: 17px;
} 
.tab button:hover {
background-color:
} 
.tab button.active {
background-color:
} 
.tabcontent {
display: none;
padding: 6px 12px;
border: 1px solid
border-top: none;
}
</style>
</head>
<body>
<h1>History</h1>
<p>Click on the buttons inside the tabbed menu</p>
<div class="tab">
<button class="tablinks" onclick="openCity(event, 'loss')">Loss</button>
<button class="tablinks" onclick="openCity(event, 'val_loss')">Val Loss</button>
<button class="tablinks" onclick="openCity(event, 'accuracy')">Accuracy</button>
<button class="tablinks" onclick="openCity(event, 'val_accuracy')">Val Accuracy</button>
</div>
<div id="loss" class="tabcontent">
<h3>Loss</h3>
<p>Loss History</p>
<img src="./loss.png" alt="loss.png">
</div>
<div id="val_loss" class="tabcontent">
<h3>Val Loss</h3>
<p>Val Loss History</p>
<img src="./val_loss.png" alt="val_loss.png">
</div>
<div id="accuracy" class="tabcontent">
<h3>Accuracy</h3>
<p>Accuracy History</p>
<img src="./accuracy.png" alt="accuracy.png">
</div>
<div id="val_accuracy" class="tabcontent">
<h3>Val Accuracy</h3>
<p>Val Accuracy History</p>
<img src="./val_accuracy.png" alt="val_accuracy.png">
</div>
<script>
function openCity(evt, cityName){
var i, tabcontent, tablinks;
tabcontent = document.getElementsByClassName("tabcontent");
for(i = 0; i < tabcontent.length; i++){
tabcontent[i].style.display = "none";
}
tablinks = document.getElementsByClassName("tablinks");
for(i = 0; i < tablinks.length; i++){
tablinks[i].className = tablinks[i].className.replace(" active", "");
}
document.getElementById(cityName ).style.display = "block";
evt.currentTarget.className += " active";
}
</script>
</body><load_from_csv>
hparam=tf.contrib.training.HParams(
 model='fm',
norm=True,
batch_norm_decay=0.9,
hidden_size=[128,128],
k=16,
hash_ids=int(2e5),
batch_size=1024,
optimizer="adam",
learning_rate=0.0008,
num_display_steps=1000,
num_eval_steps=1000,
epoch=1,
metric='auc',
init_method='uniform',
init_value=0.1,
feature_nums=len(features),
kfold=10)
 utils.print_hparams(hparam )
sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 sub['Label'] = pred
sub.head()<save_to_csv>
index=set(range(train.shape[0]))
K_fold=[]
for i in range(hparam.kfold):
if i == hparam.kfold-1:
tmp=index
else:
tmp=random.sample(index,int(1.0/hparam.kfold*train.shape[0]))
index=index-set(tmp)
 print("Number:",len(tmp))
K_fold.append(tmp)
 for i in range(hparam.kfold):
print("Fold",i)
 dev_index=K_fold[i]
dev_index=random.sample(dev_index,int(0.1*len(dev_index)))
 train_index=[]
for j in range(hparam.kfold):
if j!=i:
train_index+=K_fold[j]
model=ctrNet.build_model(hparam)
 model.train(train_data=(train.iloc[train_index][features],train.iloc[train_index]['HasDetections']),\
dev_data=(train.iloc[dev_index][features],train.iloc[dev_index]['HasDetections']))
print("Training Done! Inference...")
 if i==0:
preds=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold
else:
preds+=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold
<set_options><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] = preds
print(submission['HasDetections'].head())
 submission.to_csv('nffm_submission.csv', index=False )