Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<set_options>
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 logging.basicConfig(filename='log.txt',level=logging.DEBUG, format='%(asctime)s %(message)s')
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 print(os.listdir(".. /input/"))
np.random.seed(2)
<load_from_csv>
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
train_orig=pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_orig=pd.read_csv('.. /input/digit-recognizer/test.csv')
 sam_orig=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )<split>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]
train_x , test_x = train_test_split(train_orig , test_size=0.1 , random_state=1)
 train_y , test_y = train_x.pop('label'), test_x.pop('label')
 train_x , test_x = train_x.values , test_x.values
test_orig=test_orig.values
train_x.shape<categorify>
%%time
train = pd.read_csv('.. /input/train.csv', dtype=dtypes )
train_x = train_x.reshape(( train_x.shape[0], 28, 28, 1))
test_x = test_x.reshape(( test_x.shape[0], 28, 28, 1))
test_orig = test_orig.reshape(( test_orig.shape[0], 28 , 28 , 1))
train_x, test_x, test_orig = train_x / 255.0, test_x / 255.0, test_orig/ 255.0
train_y = to_categorical(train_y)
 test_y = to_categorical(test_y)
 train_x.shape, test_x.shape, train_y.shape, test_y.shape 
<choose_model_class>
train = reduce_mem_usage(train )
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(32,(3, 3), activation="relu",
kernel_initializer="he_uniform", input_shape=(28, 28, 1)) ,
tf.keras.layers.Conv2D(32,(3, 3), activation="relu", kernel_initializer="he_uniform"),
tf.keras.layers.BatchNormalization(axis=-1,momentum=0.99),
tf.keras.layers.MaxPooling2D(( 2, 2)) ,
tf.keras.layers.Dropout (.25),
tf.keras.layers.Conv2D(64,(3, 3), activation="relu", kernel_initializer="he_uniform"),
tf.keras.layers.Conv2D(64,(3, 3), activation="relu", kernel_initializer="he_uniform"),
tf.keras.layers.BatchNormalization(axis=-1,momentum=0.99),
tf.keras.layers.MaxPooling2D(( 2, 2)) ,
tf.keras.layers.Dropout (.25),
tf.keras.layers.Flatten() ,
tf.keras.layers.BatchNormalization(axis=-1,momentum=0.99),
tf.keras.layers.Dense(256, activation="relu",kernel_initializer="he_uniform"),
tf.keras.layers.Dense(128, activation="relu",kernel_initializer="he_uniform"),
tf.keras.layers.Dense(84, activation="relu",kernel_initializer="he_uniform"),
tf.keras.layers.Dense(10, activation="softmax")
 ])
 model.compile(optimizer=tf.keras.optimizers.Adam(
 learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,
name='Adam' 
),
loss=tf.keras.losses.BinaryCrossentropy() ,
metrics=["accuracy"])
<init_hyperparams>
stats = []
for col in train.columns:
stats.append(( col, train[col].nunique() , train[col].isnull().sum() * 100 / train.shape[0], train[col].value_counts(normalize=True, dropna=False ).values[0] * 100, train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])
 stats_df.sort_values('Percentage of missing values', ascending=False )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_x )<choose_model_class>
good_cols = list(train.columns)
 for col in train.columns:
rate = train[col].value_counts(normalize=True, dropna=False ).values[0]
if rate > 0.9:
good_cols.remove(col )
reduce_lr = ReduceLROnPlateau(monitor='Test_loss', factor=0.2,
patience=3, min_lr=0.00001)
<train_model>
train = train[good_cols]
history = model.fit(train_x, train_y,batch_size=256, epochs=50, validation_data=(test_x, test_y), verbose=0)
 score1 = model.evaluate(train_x, train_y, verbose=0)
 score = model.evaluate(test_x, test_y, verbose=0)
 print(f"Train_loss: {score1[0]} / Train accuracy: {score1[1]}")
 print(f"Test_loss: {score[0]} / Test accuracy: {score[1]}" )<predict_on_test>
test_dtypes = {k: v for k, v in dtypes.items() if k in good_cols}
test = pd.read_csv('.. /input/test.csv', dtype=test_dtypes, usecols=good_cols[:-1])
 test.loc[6529507, 'OsBuildLab'] = '17134.1.amd64fre.rs4_release.180410-1804'
test['OsBuildLab'] = test['OsBuildLab'].fillna('17134.1.amd64fre.rs4_release.180410-1804')
 test = reduce_mem_usage(test )
results = model.predict(test_orig)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
train['HasDetections'].value_counts()
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<set_options>
train['AVProductStatesIdentifier'] = train['AVProductStatesIdentifier'].astype('category')
 test['AVProductStatesIdentifier'] = test['AVProductStatesIdentifier'].astype('category' )
%matplotlib inline
print("Tensorflow version " + tf.__version__)
 np.random.seed(123 )<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
train['AVProductsInstalled'] = train['AVProductsInstalled'].astype('category')
 test['AVProductsInstalled'] = test['AVProductsInstalled'].astype('category' )
train_data=pd.read_csv(".. /input/digit-recognizer/train.csv")
 test_data=pd.read_csv(".. /input/digit-recognizer/test.csv" )<prepare_x_and_y>
train['CountryIdentifier'] = train['CountryIdentifier'].astype('category')
 test['CountryIdentifier'] = test['CountryIdentifier'].astype('category' )
Y_train=train_data['label']
X_train=train_data.drop('label', axis=1)
<prepare_x_and_y>
train['CityIdentifier'] = train['CityIdentifier'].astype('category')
 test['CityIdentifier'] = test['CityIdentifier'].astype('category' )
( x_train0, y_train0),(x_test0, y_test0)= mnist.load_data()
x_train1 = np.concatenate([x_train0, x_test0], axis=0)
 y_train1 = np.concatenate([y_train0, y_test0], axis=0)
 X_train_keras = x_train1.reshape(-1, 28*28)
 Y_train_keras = y_train1 
<concatenate>
train['OrganizationIdentifier'] = train['OrganizationIdentifier'].astype('category')
 test['OrganizationIdentifier'] = test['OrganizationIdentifier'].astype('category' )
X_train = np.concatenate(( X_train.values, X_train_keras))
Y_train = np.concatenate(( Y_train, Y_train_keras))<count_unique_values>
train['GeoNameIdentifier'] = train['GeoNameIdentifier'].astype('category')
 test['GeoNameIdentifier'] = test['GeoNameIdentifier'].astype('category' )
unique, counts = np.unique(Y_train, return_counts=True)
 dict(zip(unique, counts))<data_type_conversions>
train['LocaleEnglishNameIdentifier'] = train['LocaleEnglishNameIdentifier'].astype('category')
 test['LocaleEnglishNameIdentifier'] = test['LocaleEnglishNameIdentifier'].astype('category' )
X_train = X_train.astype('float32')
 Y_train = Y_train.astype('float32')
 test_data=test_data.astype('float32')
 X_train = X_train / 255.0
test_data = test_data / 255.0<categorify>
train['IeVerIdentifier'] = train['IeVerIdentifier'].astype('category')
 test['IeVerIdentifier'] = test['IeVerIdentifier'].astype('category' )
Y_train=to_categorical(Y_train, num_classes=10)
 print(f"Label size {Y_train.shape}")
<split>
train['Census_OEMNameIdentifier'] = train['Census_OEMNameIdentifier'].astype('category')
 test['Census_OEMNameIdentifier'] = test['Census_OEMNameIdentifier'].astype('category' )
X_train, X_val, Y_train, Y_val=train_test_split(X_train, Y_train, test_size=0.10, random_state=44)
<choose_model_class>
train['Census_OEMModelIdentifier'] = train['Census_OEMModelIdentifier'].astype('category')
 test['Census_OEMModelIdentifier'] = test['Census_OEMModelIdentifier'].astype('category' )
model= Sequential()
model.add(Conv2D(input_shape=(28,28,1), filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.10))
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.10))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation='softmax'))
model.summary()<choose_model_class>
optimizer=RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
train['Census_ProcessorModelIdentifier'] = train['Census_ProcessorModelIdentifier'].astype('category')
 test['Census_ProcessorModelIdentifier'] = test['Census_ProcessorModelIdentifier'].astype('category' )
model.compile(optimizer= optimizer, loss='categorical_crossentropy', metrics=['accuracy'] )<choose_model_class>
learning_rate_redcuing=ReduceLROnPlateau(monitor='val_accuracy',
patience=2,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
train['Census_InternalBatteryNumberOfCharges'] = train['Census_InternalBatteryNumberOfCharges'].astype('category')
 test['Census_InternalBatteryNumberOfCharges'] = test['Census_InternalBatteryNumberOfCharges'].astype('category' )
epochs = 50
batch_size = 64<choose_model_class>
train['Census_OSBuildNumber'] = train['Census_OSBuildNumber'].astype('category')
 test['Census_OSBuildNumber'] = test['Census_OSBuildNumber'].astype('category' )
imagegen=ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 imagegen.fit(X_train )<train_model>
train['Census_OSBuildRevision'] = train['Census_OSBuildRevision'].astype('category')
 test['Census_OSBuildRevision'] = test['Census_OSBuildRevision'].astype('category' )
history=model.fit_generator(imagegen.flow(X_train, Y_train,batch_size=batch_size),
epochs=epochs,
validation_data=(X_val, Y_val),
verbose=2,
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[learning_rate_redcuing])
<load_pretrained>
train['Census_FirmwareManufacturerIdentifier'] = train['Census_FirmwareManufacturerIdentifier'].astype('category')
 test['Census_FirmwareManufacturerIdentifier'] = test['Census_FirmwareManufacturerIdentifier'].astype('category' )
model.save("MNIST_CNN_Model.h5")
 model.save_weights("MNIST_CNN_Model_weights.h5" )<predict_on_test>
train['Census_FirmwareVersionIdentifier'] = train['Census_FirmwareVersionIdentifier'].astype('category')
 test['Census_FirmwareVersionIdentifier'] = test['Census_FirmwareVersionIdentifier'].astype('category' )
results = model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
train['OsBuild'] = train['OsBuild'].astype('category')
 test['OsBuild'] = test['OsBuild'].astype('category' )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("sample_submission.csv",index=False)
 submission.head()<import_modules>
def group_battery(x):
x = x.lower()
if 'li' in x:
return 1
else:
return 0
train['Census_InternalBatteryType'] = train['Census_InternalBatteryType'].apply(group_battery)
 test['Census_InternalBatteryType'] = test['Census_InternalBatteryType'].apply(group_battery )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
train['Census_OSEdition'] = train['Census_OSEdition'].astype(str)
 test['Census_OSEdition'] = test['Census_OSEdition'].astype(str)
 train['Census_OSEdition'] = train['Census_OSEdition'].apply(rename_edition)
 test['Census_OSEdition'] = test['Census_OSEdition'].apply(rename_edition)
 train['Census_OSEdition'] = train['Census_OSEdition'].astype('category')
 test['Census_OSEdition'] = test['Census_OSEdition'].astype('category' )
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<prepare_x_and_y>
train['Census_OSSkuName'] = train['Census_OSSkuName'].astype(str)
 test['Census_OSSkuName'] = test['Census_OSSkuName'].astype(str)
 train['Census_OSSkuName'] = train['Census_OSSkuName'].apply(rename_edition)
 test['Census_OSSkuName'] = test['Census_OSSkuName'].apply(rename_edition)
 train['Census_OSSkuName'] = train['Census_OSSkuName'].astype('category')
 test['Census_OSSkuName'] = test['Census_OSSkuName'].astype('category' )
img_rows, img_cols = 28, 28
num_classes = 10
def data_prep(raw):
out_y = keras.utils.np_utils.to_categorical(raw.label, num_classes)
 num_images = raw.shape[0]
x_as_array = raw.values[:,1:]
x_shaped_array = x_as_array.reshape(num_images, img_rows, img_cols, 1)
 out_x = x_shaped_array / 255
return out_x, out_y
x, y = data_prep(mnist_train)
<split>
train['Census_OSInstallLanguageIdentifier'] = train['Census_OSInstallLanguageIdentifier'].astype('category')
 test['Census_OSInstallLanguageIdentifier'] = test['Census_OSInstallLanguageIdentifier'].astype('category' )
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.10, random_state=42 )<init_hyperparams>
train['Census_OSUILocaleIdentifier'] = train['Census_OSUILocaleIdentifier'].astype('category')
 test['Census_OSUILocaleIdentifier'] = test['Census_OSUILocaleIdentifier'].astype('category' )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
train['OsSuite'] = train['OsSuite'].astype('category')
 test['OsSuite'] = test['OsSuite'].astype('category' )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size=(3, 3),padding = 'Same',
activation='relu',
input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 32, kernel_size=(3, 3),padding = 'Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.4))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(BatchNormalization(axis=1))
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization(axis=1))
model.add(Dropout(0.4))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=128),
epochs=22,callbacks=[learning_rate_reduction],
validation_data =(X_val,y_val))<train_model>
train['Wdft_RegionIdentifier'] = train['Wdft_RegionIdentifier'].astype('category')
 test['Wdft_RegionIdentifier'] = test['Wdft_RegionIdentifier'].astype('category' )
x_test_shaped_array = mnist_test.values.reshape(28000, img_rows, img_cols, 1)
 x_test = x_test_shaped_array / 255<save_to_csv>
train['SkuEdition'].value_counts(dropna=False, normalize=True )
results = model.predict(x_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist.csv",index=False )<set_options>
def warn(*args, **kwargs):
pass
warnings.warn = warn<load_from_csv>
train['SmartScreen'].value_counts(dropna=False, normalize=True ).cumsum()
np.set_printoptions(suppress=True)
 author_data = pd.read_csv('.. /input/spooky-author-identification/train.zip')
 author_data.head()<count_values>
print(author_data['author'].value_counts())
 author_data.info()<feature_engineering>
train['Census_MDC2FormFactor'].value_counts(dropna=False, normalize=True ).cumsum()
author = author_data.copy()
author['text_length'] = author['text'].apply(lambda text: len(text))
author.head()<groupby>
author.groupby('author' ).mean()<prepare_x_and_y>
train['Census_PrimaryDiskTypeName'].value_counts(dropna=False, normalize=True ).cumsum()
X = author_data.drop(['author'], axis=1)
 y = author_data.copy() ['author']<split>
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y ,test_size=0.2, random_state=42 )<string_transform>
train['Census_ProcessorManufacturerIdentifier'].value_counts(dropna=False, normalize=True ).cumsum()
nltk.word_tokenize('We don't do that here' )<choose_model_class>
stemmer = nltk.PorterStemmer()
for word in('computing','computed','compulsive'):
print(word,'=>',stemmer.stem(word))<import_modules>
train['Census_PowerPlatformRoleName'].value_counts(dropna=False, normalize=True ).cumsum()
stop_words.ENGLISH_STOP_WORDS<string_transform>
text_list = 'i myself believe it was beautiful'.split()
less_than_zero = list(filter(lambda x: not x in stop_words.ENGLISH_STOP_WORDS, text_list))
print(less_than_zero )<string_transform>
sentence = 'I love real madrid for real but I dont like real betis'
sentence = re.sub(r'\W+', ' ', sentence, flags=re.M)
 c = Counter(sentence.split())
 print(c.most_common() )<feature_engineering>
c['love'] += 3
print(c.most_common() )<feature_engineering>
def derive_new_features(data):
data = data[['text']].copy()
data['unique_words_count'] = data.text.apply(lambda x: len(set(str(x ).split())))
 data['punctuation_count'] = data.text.apply(lambda x: len([x for x in x.lower().split() if x in string.punctuation]))
data['uppercase_words_count'] = data.text.apply(lambda x: sum([x.isupper() for x in x.split() ]))
data['title_words_count'] = data.text.apply(lambda x: sum([x.istitle() for x in x.split() ]))
return data.drop(['text'], axis=1 )<categorify>
train['OsBuildLab'] = train['OsBuildLab'].cat.add_categories(['0.0.0.0.0-0'])
 train['OsBuildLab'] = train['OsBuildLab'].fillna('0.0.0.0.0-0')
 test['OsBuildLab'] = test['OsBuildLab'].cat.add_categories(['0.0.0.0.0-0'])
 test['OsBuildLab'] = test['OsBuildLab'].fillna('0.0.0.0.0-0' )
class TextToWordCounterTransformer(BaseEstimator, TransformerMixin):
def __init__(self, remove_punctuation=True, lower_case=True, stemming=True, replace_numbers=True, remove_stopwords=True):
self.remove_punctuation = remove_punctuation
self.lower_case = lower_case
self.stemming = stemming
self.replace_numbers = replace_numbers
self.remove_stopwords = remove_stopwords
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
word_counters = []
for text in X['text']:
if self.lower_case:
text = text.lower()
if self.replace_numbers:
text = re.sub(r'\d+(?:\.\d*(?:[eE]\d+)) ?', 'NUMBER', text)
 if self.remove_punctuation:
text = re.sub(r'\W+', ' ', text, flags=re.M)
 if self.remove_stopwords:
words = [word for word in text.split() if not word in stop_words.ENGLISH_STOP_WORDS]
text = ' '.join(words)
 word_list = nltk.word_tokenize(text)
 word_count = Counter(word_list)
 if self.stemming:
stemmed_word_count = Counter()
for word in word_list:
stemmed_word = stemmer.stem(word)
 stemmed_word_count[stemmed_word] += 1
word_count = stemmed_word_count
word_counters.append(word_count)
 return np.array(word_counters )<feature_engineering>
def fe(df):
df['EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2] ).astype('category')
 df['EngineVersion_3'] = df['EngineVersion'].apply(lambda x: x.split('.')[3] ).astype('category')
 df['AppVersion_1'] = df['AppVersion'].apply(lambda x: x.split('.')[1] ).astype('category')
 df['AppVersion_2'] = df['AppVersion'].apply(lambda x: x.split('.')[2] ).astype('category')
 df['AppVersion_3'] = df['AppVersion'].apply(lambda x: x.split('.')[3] ).astype('category')
 df['AvSigVersion_0'] = df['AvSigVersion'].apply(lambda x: x.split('.')[0] ).astype('category')
 df['AvSigVersion_1'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1] ).astype('category')
 df['AvSigVersion_2'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2] ).astype('category')
 df['OsBuildLab_0'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0] ).astype('category')
 df['OsBuildLab_1'] = df['OsBuildLab'].apply(lambda x: x.split('.')[1] ).astype('category')
 df['OsBuildLab_2'] = df['OsBuildLab'].apply(lambda x: x.split('.')[2] ).astype('category')
 df['OsBuildLab_3'] = df['OsBuildLab'].apply(lambda x: x.split('.')[3] ).astype('category')
 df['Census_OSVersion_0'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[0] ).astype('category')
 df['Census_OSVersion_1'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[1] ).astype('category')
 df['Census_OSVersion_2'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[2] ).astype('category')
 df['Census_OSVersion_3'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[3] ).astype('category')
 df['primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
df['non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']
df['aspect_ratio'] = df['Census_InternalPrimaryDisplayResolutionHorizontal']/ df['Census_InternalPrimaryDisplayResolutionVertical']
df['monitor_dims'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str)+ '*' + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')
 df['monitor_dims'] = df['monitor_dims'].astype('category')
 df['dpi'] =(( df['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + df['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(df['Census_InternalPrimaryDiagonalDisplaySizeInInches'])
 df['dpi_square'] = df['dpi'] ** 2
df['MegaPixels'] =(df['Census_InternalPrimaryDisplayResolutionHorizontal'] * df['Census_InternalPrimaryDisplayResolutionVertical'])/1e6
df['Screen_Area'] =(df['aspect_ratio']*(df['Census_InternalPrimaryDiagonalDisplaySizeInInches']**2)) /(df['aspect_ratio']**2 + 1)
 df['ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']
df['new_num_0'] = df['Census_InternalPrimaryDiagonalDisplaySizeInInches'] / df['Census_ProcessorCoreCount']
df['new_num_1'] = df['Census_ProcessorCoreCount'] * df['Census_InternalPrimaryDiagonalDisplaySizeInInches']
df['Census_IsFlightingInternal'] = df['Census_IsFlightingInternal'].fillna(1)
 df['Census_ThresholdOptIn'] = df['Census_ThresholdOptIn'].fillna(1)
 df['Census_IsWIMBootEnabled'] = df['Census_IsWIMBootEnabled'].fillna(1)
 df['Wdft_IsGamer'] = df['Wdft_IsGamer'].fillna(0)
 return df
X_word_count = TextToWordCounterTransformer().fit_transform(X_train)
 X_word_count<prepare_x_and_y>
train = fe(train)
 test = fe(test )
class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):
def __init__(self, vocabulary_size=1000):
self.vocabulary_size = vocabulary_size
def fit(self, X, y=None):
total_count = Counter()
for word_count in X:
for word, count in word_count.items() :
total_count[word] += min(count, 10)
 most_common = total_count.most_common() [:self.vocabulary_size]
self.most_common_ = most_common
self.vocabulary_ = {word: index + 1 for index,(word, count)in enumerate(most_common)}
return self
def transform(self, X, y=None):
data = []
rows = []
cols = []
for row, word_count in enumerate(X):
for word, count in word_count.items() :
data.append(count)
 rows.append(row)
 cols.append(self.vocabulary_.get(word, 0))
return csr_matrix(( data,(rows, cols)) , shape=(len(X), self.vocabulary_size + 1))<feature_engineering>
cat_cols = [col for col in train.columns if col not in ['MachineIdentifier', 'Census_SystemVolumeTotalCapacity', 'HasDetections'] and str(train[col].dtype)== 'category']
len(cat_cols )
vectorizer = WordCounterToVectorTransformer(vocabulary_size=1000)
 vectorizer.fit_transform(X_word_count ).toarray()<categorify>
more_cat_cols = []
add_cat_feats = [
'Census_OSBuildRevision',
'OsBuildLab',
'SmartScreen',
'AVProductsInstalled']
for col1 in add_cat_feats:
for col2 in add_cat_feats:
if col1 != col2:
train[col1 + '__' + col2] = train[col1].astype(str)+ train[col2].astype(str)
 train[col1 + '__' + col2] = train[col1 + '__' + col2].astype('category')
 test[col1 + '__' + col2] = test[col1].astype(str)+ test[col2].astype(str)
 test[col1 + '__' + col2] = test[col1 + '__' + col2].astype('category')
 more_cat_cols.append(col1 + '__' + col2)
 cat_cols = cat_cols + more_cat_cols
preprocess_pipeline = Pipeline([ 
('text_to_word_count', TextToWordCounterTransformer()), 
('word_count_to_vector', WordCounterToVectorTransformer(vocabulary_size=14000)) ,
] )<normalization>
to_encode = []
for col in cat_cols:
if train[col].nunique() > 1000:
print(col, train[col].nunique())
 to_encode.append(col )
class NewFeaturesAdderTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return np.array(derive_new_features(X))<feature_engineering>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test)
 gc.collect()
NewFeaturesAdderTransformer().fit_transform(X_train )<feature_engineering>
def frequency_encoding(variable):
t = pd.concat([train[variable], test[variable]] ).value_counts().reset_index()
t = t.reset_index()
t.loc[t[variable] == 1, 'level_0'] = np.nan
t.set_index('index', inplace=True)
 max_label = t['level_0'].max() + 1
t.fillna(max_label, inplace=True)
 return t.to_dict() ['level_0']
full_pipeline = ColumnTransformer([ 
('feature_adder', NewFeaturesAdderTransformer() , ['text']), 
('text_pipeline', preprocess_pipeline, ['text']),
] )<categorify>
for col in tqdm_notebook(to_encode):
freq_enc_dict = frequency_encoding(col)
 train[col] = train[col].map(lambda x: freq_enc_dict.get(x, np.nan))
test[col] = test[col].map(lambda x: freq_enc_dict.get(x, np.nan))
cat_cols.remove(col )
label_encoder = LabelEncoder()<categorify>
%%time
indexer = {}
for col in cat_cols:
_, indexer[col] = pd.factorize(train[col].astype(str), sort=True)
 for col in tqdm_notebook(cat_cols):
train[col] = indexer[col].get_indexer(train[col].astype(str))
test[col] = indexer[col].get_indexer(test[col].astype(str))
train = reduce_mem_usage(train, verbose=False)
 test = reduce_mem_usage(test, verbose=False )
X_train_transformed = full_pipeline.fit_transform(X_train)
 y_train_transformed = label_encoder.fit_transform(y_train.values )<categorify>
del indexer
label_encoder.classes_<choose_model_class>
y = train['HasDetections']
train = train.drop(['HasDetections', 'MachineIdentifier'], axis=1)
 test = test.drop(['MachineIdentifier'], axis=1)
 gc.collect()
train.sort_values('AvSigVersion')
 train1 = train[:4000000]
train = train[4000000:8000000]
y1 = y[:4000000]
y = y[4000000:8000000]
sgd_clf = SGDClassifier(random_state=42, loss='hinge' )<compute_train_metric>
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=15)
cv_score = cross_val_score(
 sgd_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )<compute_train_metric>
@jit
def fast_auc(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(preds, dtrain):
labels = dtrain.get_label()
return 'auc', fast_auc(labels, preds), True
def predict_chunk(model, test):
initial_idx = 0
chunk_size = 1000000
current_pred = np.zeros(len(test))
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = model.predict(test.iloc[idx], num_iteration=model.best_iteration)
 initial_idx = final_idx
return current_pred
def train_model(X=train, X_test=test, y=y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, averaging='usual', make_oof=False):
result_dict = {}
if make_oof:
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
gc.collect()
print('Fold', fold_n + 1, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train, categorical_feature = cat_cols)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature = cat_cols)
 model = lgb.train(params,
train_data,
num_boost_round=2000,
valid_sets = [train_data, valid_data],
verbose_eval=500,
early_stopping_rounds = 200,
feval=eval_auc)
 del train_data, valid_data
y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 del X_valid
gc.collect()
y_pred = predict_chunk(model, X_test)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = predict_chunk(model, xgb.DMatrix(X_test))
if model_type == 'lcv':
model = LogisticRegressionCV(scoring='roc_auc', cv=3)
 model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = predict_chunk(model, X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = predict_chunk(model, X_test)
 if make_oof:
oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(fast_auc(y_valid, y_pred_valid))
print('Fold roc_auc:', roc_auc_score(y_valid, y_pred_valid))
print('')
 if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_fold
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
logging.info('Top features')
 for f in best_features.sort_values(by="importance", ascending=False)['feature'].values:
logging.info(f)
 plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['prediction'] = prediction
if make_oof:
result_dict['oof'] = oof
return result_dict
log_clf = LogisticRegression(random_state=42)
 cv_score = cross_val_score(
 log_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )<compute_train_metric>
params = {'num_leaves': 256,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 5,
'learning_rate': 0.05,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 5,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"lambda_l1": 0.15,
"lambda_l2": 0.15,
"random_state": 42,
"verbosity": -1}
mnb_clf = MultinomialNB()
cv_score = cross_val_score(
 mnb_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3, 
)
 print('mean score :', cv_score.mean() )<compute_test_metric>
del stats_df, freq_enc_dict
def multiclass_logloss(actual, predicted, eps=1e-15):

 if len(actual.shape)== 1:
actual2 = np.zeros(( actual.shape[0], predicted.shape[1]))
for i, val in enumerate(actual):
actual2[i, val] = 1
actual = actual2
clip = np.clip(predicted, eps, 1 - eps)
 rows = actual.shape[0]
vsota = np.sum(actual * np.log(clip))
return -1.0 / rows * vsota<predict_on_test>
result_dict1 = train_model(X=train1, X_test=test, y=y1, params=params, model_type='lgb', plot_feature_importance=True, averaging='rank' )
y_scores = cross_val_predict(
 log_clf,
X_train_transformed, y_train_transformed,
cv=5,
verbose=3,
method='predict_proba',
)<categorify>
del train1, y1
<compute_test_metric><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] = result_dict1['prediction']
submission.to_csv('lgb_rank.csv', index=False )