Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<create_dataframe>
def load(x):
ignore = ['MachineIdentifier']
if x in ignore: return False
else: return True
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
city_rev = []
for i in full_data['City_Cluster']:
for key, value in mean_dict.items() :
if i == key:
city_rev.append(value)
 df_city_rev = pd.DataFrame({'city_rev':city_rev})
 full_data = pd.concat([full_data,df_city_rev],axis=1)
 full_data.head
nominal_list.extend(['City_Cluster'])
 nominal_list.remove('City')
<categorify>
def factor_data(df_train, df_test, col):
df_comb = pd.concat([df_train[col],df_test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 df_comb += 1
df_comb = np.where(df_comb==0, df_comb.max() +1, df_comb)
 df_train[col] = df_comb[:len(df_train)]
df_test[col] = df_comb[len(df_train):]
del df_comb
def reduce_memory(df,col):
mx = df[col].max()
if mx<256:
df[col] = df[col].astype('uint8')
 elif mx<65536:
df[col] = df[col].astype('uint16')
 else:
df[col] = df[col].astype('uint32')
 def relax_data(df_train, df_test, col):
cv1 = pd.DataFrame(df_train[col].value_counts().reset_index().rename({col:'train'},axis=1))
cv2 = pd.DataFrame(df_test[col].value_counts().reset_index().rename({col:'test'},axis=1))
cv3 = pd.merge(cv1,cv2,on='index',how='outer')
 cv3['train'].fillna(0,inplace=True)
 cv3['test'].fillna(0,inplace=True)
 factor = len(df_test)/len(df_train)
 cv3['remove'] = False
cv3['remove'] = cv3['remove'] |(cv3['train'] < len(df_train)/9000)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] < cv3['test']/4)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] > 4*cv3['test'])
 cv3['new'] = cv3.apply(lambda x: x['index'] if x['remove']==False else 0,axis=1)
 cv3['new'],_ = cv3['new'].factorize(sort=True)
 cv3.set_index('index',inplace=True)
 cc = cv3['new'].to_dict()
df_train[col] = df_train[col].map(cc)
 reduce_memory(df_train,col)
 df_test[col] = df_test[col].map(cc)
 reduce_memory(df_test,col)
 def display_memory(df_train, df_test):
print(len(df_train),'rows of training data use',df_train.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 print(len(df_test),'rows of test data use',df_test.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 def categorize(df_train, df_test, cols):
for col in cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )
le = LabelEncoder()
le_count = 0
for i in range(len(nominal_list)) :
if len(list(full_data[nominal_list[i]].unique())) <= 2:
le.fit(full_data[nominal_list[i]])
 full_data[nominal_list[i]] = le.transform(full_data[nominal_list[i]])
 le_count += 1
print('%d columns were label encoded.' % le_count )<categorify>
del df_trainA, df_trainB, df_trainC, df_trainD
del df_train['DateAS'], df_test['DateAS']; x=gc.collect()
cols = [x for x in df_train.columns if x not in ['HasDetections','AvSigVersion2']]
print('Factorizing...')
 for col in cols: factor_data(df_train, df_test, col)
 print('Reducing memory...')
 for col in cols: reduce_memory(df_train, col)
 for col in cols: reduce_memory(df_test, col)
 categorize(df_train, df_test, cols)
 display_memory(df_train, df_test )
full_data = pd.get_dummies(full_data)
 print('full_dataing Features shape: ', full_data.shape )<randomize_order>
df_trainA = df_train.sample(frac=0.5)
 df_trainB = df_train[ ~df_train.index.isin(df_trainA.index)]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainA[cols], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols], df_trainB['HasDetections'])], verbose=250,
early_stopping_rounds=100 )
def tukey_outliers(x):
q1 = np.percentile(x,25)
 q3 = np.percentile(x,75)
 iqr = q3-q1
min_range = q1 - iqr*1.5
max_range = q3 + iqr*1.5
outliers = x[(x<min_range)|(x>max_range)]
return outliers<define_variables>
del df_trainA, df_trainB; x=gc.collect()
idx = 0; chunk = 2000000
pred_val = np.zeros(len(df_test))
while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] = model.predict_proba(df_test.iloc[idx][cols])[:,1]
idx = idx2
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submit['HasDetections'] = pred_val
submit.to_csv('ModelOne.csv', index=False )
<sort_values>
df_trainC = df_train[ df_train['AvSigVersion2']<275 ]
df_trainD = df_train[ df_train['AvSigVersion2']>=275 ]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainC[cols], df_trainC['HasDetections'], eval_metric='auc',
eval_set=[(df_trainD[cols], df_trainD['HasDetections'])], verbose=250,
early_stopping_rounds=100 )
skewed_data = train[num_list].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewed_data[:10]<train_model>
print('Converting data to Model Two...')
 df_trainC = df_trainC.copy()
df_trainD = df_trainD.copy()
for col in cols: relax_data(df_trainC, df_trainD, col)
 categorize(df_trainC, df_trainD, cols)
 model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainC[cols], df_trainC['HasDetections'], eval_metric='auc',
eval_set=[(df_trainD[cols], df_trainD['HasDetections'])], verbose=250,
early_stopping_rounds=100 )
train = full_data[:num_train]
test = full_data[num_train:]
print("Train :",train.shape)
 print("Test:",test.shape )<train_model>
print('Converting data to Model Two...')
 del df_trainC, df_trainD; x=gc.collect()
for col in cols: relax_data(df_train, df_test, col)
 categorize(df_train, df_test, cols)
 df_trainA = df_train.sample(frac=0.5)
 df_trainB = df_train[ ~df_train.index.isin(df_trainA.index)]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainA[cols], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols], df_trainB['HasDetections'])], verbose=250,
early_stopping_rounds=100 )
( X_train, X_test, y_train, y_test)= train_test_split(train_X, train_y , test_size = 0.3 , random_state = 0)
 print("X_train: "+str(X_train.shape))
print("X_test: "+str(X_test.shape))
print("y_train: "+str(y_train.shape))
print("y_test: "+str(y_test.shape))<import_modules>
del df_trainA, df_trainB, df_train; x=gc.collect()
idx = 0; chunk = 2000000
pred_val = np.zeros(len(df_test))
while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] = model.predict_proba(df_test.iloc[idx][cols])[:,1]
idx = idx2
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submit['HasDetections'] = pred_val
submit.to_csv('ModelTwo.csv', index=False )
from sklearn.model_selection import GridSearchCV, cross_val_score, learning_curve
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import Lasso
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor<choose_model_class>
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load, nrows=10000)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load, nrows=10000)
 cols = [x for x in df_train.columns if x not in ['HasDetections','AvSigVersion2']]
for col in cols: factor_data(df_train, df_test, col)
 for col in cols: reduce_memory(df_train, col)
 for col in cols: reduce_memory(df_test, col)
 categorize(df_train, df_test, cols)
 df_train['HasDetections'] = 0
df_test['HasDetections'] = 1
df_comb = pd.concat([df_train,df_test],axis=0)
 model = DecisionTreeClassifier(max_leaf_nodes=5)
 model.fit(df_comb[cols], df_comb['HasDetections'])
 pred_val = model.predict_proba(df_comb[cols])[:,1]
print('Model One: Adversarial Training AUC = ',round(roc_auc_score(df_comb['HasDetections'],pred_val),4))
print('Adversarial Model has max_leaf_nodes=5')
 tree_graph = tree.export_graphviz(model, out_file=None, max_depth = 10,
impurity = False, feature_names = cols, class_names = ['No', 'Yes'],
rounded = True, filled= True)
 graphviz.Source(tree_graph )
random_state = 2
classifiers = []
classifiers.append(Lasso(random_state=random_state))
classifiers.append(LinearRegression())
 classifiers.append(Ridge(random_state=random_state))
classifiers.append(ElasticNet(random_state=random_state))
classifiers.append(KNeighborsRegressor())
 classifiers.append(SVR())
 classifiers.append(RandomForestRegressor(random_state=random_state))
classifiers.append(GradientBoostingRegressor())
 classifiers.append(AdaBoostRegressor(random_state = random_state))
classifiers.append(DecisionTreeRegressor())
 classifiers.append(XGBRegressor() )<sort_values>
for col in cols: relax_data(df_train, df_test, col)
 categorize(df_train, df_test, cols)
 df_comb = pd.concat([df_train,df_test],axis=0)
 cols2 = cols.copy()
cols2.remove('SMode')
 model = DecisionTreeClassifier(max_leaf_nodes=5)
 model.fit(df_comb[cols2], df_comb['HasDetections'])
 pred_val = model.predict_proba(df_comb[cols2])[:,1]
print('Model Two: Adversarial Training AUC = ',round(roc_auc_score(df_comb['HasDetections'],pred_val),4))
print('Adversarial Model has max_leaf_nodes=5')
 tree_graph = tree.export_graphviz(model, out_file=None, max_depth = 10,
impurity = False, feature_names = cols2, class_names = ['No', 'Yes'],
rounded = True, filled= True)
 graphviz.Source(tree_graph )
<train_model><EOS> 
cols2 = {'AVProductStatesIdentifier':0.01, 'CountryIdentifier':0.4, 'LocaleEnglishNameIdentifier':0.3, 'SmartScreen':0.4,
'Census_OEMNameIdentifier':0.1,'Census_TotalPhysicalRAM':0.5,'Census_InternalPrimaryDiagonalDisplaySizeInInches':0.05,
'Census_OSInstallTypeName':0.75,'Census_OSInstallLanguageIdentifier':0.3,'Census_FirmwareManufacturerIdentifier':0.1,
'EngineVersion':1.0, 'AppVersion':0.7, 'OsBuildLab':0.2, 'Census_OEMModelIdentifier':0.15,
'Census_InternalBatteryNumberOfCharges':0.05}
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
 df_test['DateAS'] = df_test['AvSigVersion'].map(datedictAS)
 df_testA = df_test[ df_test['DateAS']<datetime(2018,10,25)]
df_testB = df_test[ df_test['DateAS']>datetime(2018,10,25)]
for x in df_train.columns[:-2]:
s = 0.5
if x in cols2: s = cols2[x]
comparePlot(df_train,df_testA,x,scale=s, title='Public Test vs.Train', prefix='Public ')
 comparePlot(df_train,df_testB,x,scale=s, title='Private Test vs.Train', prefix='Private ' )