Kaggle competition: Microsoft Malware Prediction
<count_values>
import numpy as np
import pandas as pd
from gensim.models import Word2Vec
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer
from gensim.test.utils import common_texts, get_tmpfile
from keras.preprocessing.text import Tokenizer
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn import metrics
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
from keras.preprocessing.sequence import pad_sequences
from smart_open import smart_open
import datetime
from keras.utils import multi_gpu_model
import os
import time
import gc
import re
import random
l1_l2_and = set(city_train_list)& set(city_test_list)
 print(l1_l2_and)
 print(len(l1_l2_and))<concatenate>
def convertCats2sentence(x,cols):
ref = [[col.lower() + '=' +(str(val)).lower() for val in x[col]] for col in cols]
tmp = pd.DataFrame.from_records(ref)
 tmp = tmp.transpose()
X = tmp.values.tolist()
return X
chunksize = 100000
header = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',nrows=1)
 cols = list(set(header.head(0)) - set(['Census_PrimaryDiskTotalCapacity',
'Census_SystemVolumeTotalCapacity','HasDetections']))
n = 8921484 - 1
s = 500000
skip = sorted(random.sample(range(1,n+1),n-s))
X = []
t0 = time.time()
steps = 1
for chunk in pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', chunksize=chunksize,dtype='category',skiprows=skip):
print('step...'+str(steps))
X.extend(convertCats2sentence(chunk,cols))
steps = steps + 1
t1 = time.time()
print(t1-t0)
 print('train converted...')
 gc.collect()
len(set(city_test_list ).difference(city_train_list)) 
<concatenate>
embed_size = 300
maxlen = len(cols)
 max_features = None
window_size = 40
t0 = time.time()
model = Word2Vec(X,size=embed_size,window=window_size,min_count=1)
 t1 = time.time()
print(t1-t0)
 print('model fit...')
 tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(X)
len(set(city_train_list ).difference(city_test_list))<predict_on_test>
n = 8921484 - 1
s = 1000000
skip = sorted(random.sample(range(1,n+1),n-s))
X = []
t0 = time.time()
steps = 1
gc.collect()
for chunk in pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', chunksize=chunksize,dtype='category',skiprows=skip):
print('step...'+str(steps))
X.extend(convertCats2sentence(chunk,cols))
steps = steps + 1
t1 = time.time()
print(t1-t0)
 print('train converted...')
 X = tokenizer.texts_to_sequences(X)
 X = pad_sequences(X, maxlen=maxlen)
 word_index = tokenizer.word_index
max_features = len(word_index)+1
train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',usecols=['HasDetections'],skiprows=skip)
 Y = train['HasDetections']
del train
gc.collect()
def adjust_cities(full_full_data, train, k):
relevant_pvars = ["P1", "P2", "P11", "P19", "P20", "P23","P30"]
train = train.loc[:, relevant_pvars]
kmeans = cluster.KMeans(n_clusters=k)
 kmeans.fit(train)
 full_data['City_Cluster'] = kmeans.predict(full_data.loc[:, relevant_pvars])
 return full_data<concatenate>
def load_wv(model, word_index):
embedding_matrix = np.random.normal(model.wv[model.wv.vocab].mean() , model.wv[model.wv.vocab].std() , 
(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
try:
embedding_vector = model.wv[word]
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
except:
print(word, ' not found')
 return embedding_matrix
embedding_matrix = load_wv(model, word_index )
num_train = train.shape[0]
num_test = test.shape[0]
print(num_train, num_test)
 full_data = pd.concat([train, test], ignore_index=True )<drop_column>
sub = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=['MachineIdentifier'])
full_data = adjust_cities(full_data, train, 20)
 full_data
full_data = full_data.drop(['City'], axis=1 )<train_model>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3,
kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size,
weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300),
recurrent_initializer=orthogonal(gain=1.0,
seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=5, routings=2, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
def auc(y_true, y_pred):
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1)
 return metrics.auc(fpr, tpr )
<feature_engineering><EOS> 
sub['HasDetections'] = y_test.reshape(( -1, 1))
tmp = pd.read_csv('.. /input/lgbyhat/lgb_submission.csv')
 tmp.columns = ['MachineIdentifier','HD']
sub = pd.merge(sub,tmp)
 sub['mean'] = sub.mean(axis=1)
 sub.drop(['HD','HasDetections'],axis=1,inplace=True)
 sub.columns = ['MachineIdentifier','HasDetections']
sub.to_csv("submission.csv", index=False )