Kaggle competition: Microsoft Malware Prediction
X_train, X_test = prcs(df_joined.copy() , fe=['id'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )<compute_test_metric>
warnings.simplefilter(action='ignore', category=FutureWarning)
 init_notebook_mode(connected=True)
 pd.set_option('display.max_columns', 500 )
models, scores = train_cv(X_train, y_train)
 print(scores )<save_to_csv>
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(predict(models, X_test))))
df_preds.to_csv('submission2.csv', index=True, index_label='Id')
 df_preds.head()<prepare_x_and_y>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )<compute_test_metric>
nrows = 4000000
retained_columns = numerical_columns + categorical_columns
train = pd.read_csv('.. /input/train.csv',
nrows = nrows,
usecols = retained_columns,
dtype = dtypes)
 retained_columns += ['MachineIdentifier']
retained_columns.remove('HasDetections')
 test = pd.read_csv('.. /input/test.csv',
usecols = retained_columns,
dtype = dtypes )
models, scores = train_cv(X_train, y_train)
 print(scores )<save_to_csv>
true_numerical_columns = [
'Census_ProcessorCoreCount',
'Census_PrimaryDiskTotalCapacity',
'Census_SystemVolumeTotalCapacity',
'Census_TotalPhysicalRAM',
'Census_InternalPrimaryDiagonalDisplaySizeInInches',
'Census_InternalPrimaryDisplayResolutionHorizontal',
'Census_InternalPrimaryDisplayResolutionVertical',
'Census_InternalBatteryNumberOfCharges'
]
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(predict(models, X_test))))
df_preds.to_csv('submission3.csv', index=True, index_label='Id')
 df_preds.head()<prepare_x_and_y>
binary_variables = [c for c in train.columns if train[c].nunique() == 2]
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies', 'city'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )<compute_test_metric>
categorical_columns = [c for c in train.columns
if(c not in true_numerical_columns)&(c not in binary_variables)]
models, scores = train_cv(X_train, y_train)
 print(scores )<save_to_csv>
def frequency_encoding(variable):
t = pd.concat([train[variable], test[variable]] ).value_counts().reset_index()
t = t.reset_index()
t.loc[t[variable] == 1, 'level_0'] = np.nan
t.set_index('index', inplace=True)
 max_label = t['level_0'].max() + 1
t.fillna(max_label, inplace=True)
 return t.to_dict() ['level_0']
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(predict(models, X_test))))
df_preds.to_csv('submission4.csv', index=True, index_label='Id')
 df_preds.head()<prepare_x_and_y>
frequency_encoded_variables = [
'Census_OEMModelIdentifier',
'CityIdentifier',
'Census_FirmwareVersionIdentifier',
'AvSigVersion',
'Census_ProcessorModelIdentifier',
'Census_OEMNameIdentifier',
'DefaultBrowsersIdentifier'
]
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies', 'city', 'city_group'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )<compute_test_metric>
for variable in tqdm(frequency_encoded_variables):
freq_enc_dict = frequency_encoding(variable)
 train[variable] = train[variable].map(lambda x: freq_enc_dict.get(x, np.nan))
test[variable] = test[variable].map(lambda x: freq_enc_dict.get(x, np.nan))
categorical_columns.remove(variable )
models, scores = train_cv(X_train, y_train)
 print(scores )<save_to_csv>
indexer = {}
for col in tqdm(categorical_columns):
if col == 'MachineIdentifier': continue
_, indexer[col] = pd.factorize(train[col])
 for col in tqdm(categorical_columns):
if col == 'MachineIdentifier': continue
train[col] = indexer[col].get_indexer(train[col])
 test[col] = indexer[col].get_indexer(test[col] )
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(predict(models, X_test))))
df_preds.to_csv('submission5.csv', index=True, index_label='Id')
 df_preds.head()<set_options>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
np.random.seed(0)
 %matplotlib inline<load_from_csv>
target = train['HasDetections']
del train['HasDetections']
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 X_train = train.drop(labels = ["label"], axis = 1)
 y_train = train['label']
X_test = test
print(X_train.shape, X_test.shape )<define_variables>
param = {'num_leaves': 60,
'min_data_in_leaf': 60,
'objective':'binary',
'max_depth': -1,
'learning_rate': 0.1,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8 ,
"bagging_seed": 11,
"metric": 'auc',
"lambda_l1": 0.1,
"random_state": 133,
"verbosity": -1}
img_rows, img_cols = 28, 28
num_pixels = X_train.shape[1]
input_shape =(img_rows, img_cols )<categorify>
max_iter = 5
X_train /= 255
X_test /= 255
y_train = to_categorical(y_train)
 num_classes = y_train.shape[1]
print(X_train.shape, X_test.shape )<split>
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 2 )<choose_model_class>
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train))
categorical_columns = [c for c in categorical_columns if c not in ['MachineIdentifier']]
features = [c for c in train.columns if c not in ['MachineIdentifier']]
predictions = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features],
label=target.iloc[trn_idx],
categorical_feature = categorical_columns 
)
 val_data = lgb.Dataset(train.iloc[val_idx][features],
label=target.iloc[val_idx],
categorical_feature = categorical_columns 
)
 num_round = 10000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=100,
early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 1000000
current_pred = np.zeros(len(test))
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(target.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(target, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter))
def get_mlp() :
return Sequential([
Dense(512, input_dim = num_pixels, activation='relu'),
Dense(num_classes, activation='softmax')
 ] )<choose_model_class>
sub_df = pd.DataFrame({"MachineIdentifier": test["MachineIdentifier"].values})
 sub_df["HasDetections"] = predictions
sub_df[:10]
<train_model><EOS> 
sub_df.to_csv("submit.csv", index=False )