Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<find_best_params>
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 logging.basicConfig(filename='log.txt',level=logging.DEBUG, format='%(asctime)s %(message)s')
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 print(os.listdir(".. /input"))
study = optuna.create_study()
study.optimize(objective, n_trials=100)
 print(f'best score: {study.best_value:.4f}, best params: {study.best_params}' )<find_best_model_class>
@jit
def fast_auc(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(preds, dtrain):
labels = dtrain.get_label()
return 'auc', fast_auc(labels, preds), True
params = {'alpha': 1.9510706324753746, 'fit_intercept': True, 'normalize': True}
reg = Ridge(**params)
 reg.fit(X_train, y_train)
 prediction_log = reg.predict(test_X)
 prediction =np.exp(prediction_log)
 print(prediction )<save_to_csv>
dtypes = {
'MachineIdentifier': 'object',
'ProductName': 'object',
'EngineVersion': 'object',
'AppVersion': 'object',
'AvSigVersion': 'object',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float32',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int16',
'Platform': 'object',
'Processor': 'object',
'OsVer': 'object',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'object',
'OsBuildLab': 'object',
'SkuEdition': 'object',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'object',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'object',
'Firewall': 'float16',
'UacLuaenable': 'float64',
'Census_MDC2FormFactor': 'object',
'Census_DeviceFamily': 'object',
'Census_OEMNameIdentifier': 'float32',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float32',
'Census_ProcessorClass': 'object',
'Census_PrimaryDiskTotalCapacity': 'float64',
'Census_PrimaryDiskTypeName': 'object',
'Census_SystemVolumeTotalCapacity': 'float64',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'object',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float32',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float32',
'Census_InternalPrimaryDisplayResolutionVertical': 'float32',
'Census_PowerPlatformRoleName': 'object',
'Census_InternalBatteryType': 'object',
'Census_InternalBatteryNumberOfCharges': 'float64',
'Census_OSVersion': 'object',
'Census_OSArchitecture': 'object',
'Census_OSBranch': 'object',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'object',
'Census_OSSkuName': 'object',
'Census_OSInstallTypeName': 'object',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'object',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'object',
'Census_ActivationChannel': 'object',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'object',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'float32',
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
submission = pd.DataFrame({"Id":test_Id, "Prediction":prediction})
 submission.to_csv("submission.csv", index=False )<train_model>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]
<init_hyperparams>
%%time
train = pd.read_csv('.. /input/train.csv', dtype=dtypes )
<import_modules>
random_sample_percent = 5/8
random_state = 15
number_of_folds = 5
stop_after_fold_number = 1
shuffle = True
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.neighbors import KNeighborsRegressor
import lightgbm as lgbm<load_from_csv>
if random_sample_percent is not None:
train = train.sample(frac=random_sample_percent, random_state=random_state )
train = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/train.csv.zip")
 test = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/test.csv.zip")
 ref = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv")
 test_id = pd.DataFrame(test["Id"])
 display(ref.head() )<count_missing_values>
train_y = train['HasDetections']
a = train.isnull().sum()
a[a>0]<groupby>
train = reduce_mem_usage(train )
print(train["City"].groupby(train["City"] ).count().count())
 print(train["City Group"].groupby(train["City Group"] ).count().count())
 print(train["Type"].groupby(train["Type"] ).count().count())
 print(test["City"].groupby(test["City"] ).count().count())
 print(test["City Group"].groupby(test["City Group"] ).count().count())
 print(test["Type"].groupby(test["Type"] ).count().count() )<data_type_conversions>
test = pd.read_csv('.. /input/test.csv', dtype=dtypes)
 test = reduce_mem_usage(test )
train["kOpen Year"] = train["Open Date"].str[6:12]
test["kOpen Year"] = test["Open Date"].str[6:12]
train["Open Month"] = train["Open Date"].str[0:2].astype(int)
 test["Open Month"] = test["Open Date"].str[0:2].astype(int)
 train["gOpen Year's"] = train["Open Date"].str[8:9] + "0's"
test["gOpen Year's"] = test["Open Date"].str[8:9] + "0's"
train['Open Month'] = pd.cut(train['Open Month'],4,labels=['1','2','3','4'])
 test['Open Month'] = pd.cut(test['Open Month'],4,labels=['1','2','3','4'])
 train, test = train.drop(["Id","Open Date","City"], axis=1), test.drop(["Id","Open Date","City"], axis=1 )<groupby>
train["gOpen Year's"].groupby(train["gOpen Year's"] ).count()<create_dataframe>
def encode_categorical_columns(x_train, x_test, columns, sort=True):
train_length = x_train.shape[0]
for col in tqdm(columns):
if col == 'MachineIdentifier' or col == 'HasDetections':
continue
combined_data = pd.concat([x_train[col], x_test[col]])
 combined_data, _ = pd.factorize(combined_data, sort=sort)
 combined_data = pd.Series(combined_data ).astype('int32')
 x_train[col] = combined_data.iloc[:train_length].values
x_test[col] = combined_data.iloc[train_length:].values
x_train[col] = x_train[col].fillna(0)
 x_test[col] = x_test[col].fillna(0)
 del combined_data
return x_train, x_test
corr_abs = np.absolute(train.loc[:,"P1":"revenue"].corr() ["revenue"].drop("revenue")).sort_values(ascending=False)
 ex_var = pd.DataFrame(corr_abs[corr_abs<0.1] ).index
train1 = train.drop(ex_var, axis=1)
 test1 = test.drop(ex_var, axis=1 )<categorify>
train, test = encode_categorical_columns(train, test, categorical_columns)
 train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
train_dm = pd.get_dummies(train ).drop("City Group_Other", axis=1)
 test_dm = pd.get_dummies(test ).drop("City Group_Other", axis=1)
 print(train_dm.shape, test_dm.shape)
 print(train_dm.columns,"
", test_dm.columns )<create_dataframe>
def predict_chunk(model, test):
initial_idx = 0
chunk_size = 1000000
current_pred = np.zeros(len(test))
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = model.predict(test.iloc[idx], num_iteration=model.best_iteration)
 initial_idx = final_idx
return current_pred
train_fix = pd.DataFrame(train_dm, columns=a)
 test_fix = pd.DataFrame(test_dm, columns=a )<data_type_conversions>
def train_model(x, y, lgb_params,
number_of_folds=5,
evaluation_metric='auc',
save_feature_importances=False,
early_stopping_rounds=50,
num_round = 50,
identifier_columns=['MachineIdentifier'],
stop_after_fold_number=None):
cross_validator = StratifiedKFold(n_splits=number_of_folds,
random_state=random_state,
shuffle=shuffle)
 validation_scores = []
classifier_models = []
feature_importance_df = pd.DataFrame()
for fold_index,(train_index, validation_index)in enumerate(cross_validator.split(x, y)) :
x_train, x_validation = x.iloc[train_index], x.iloc[validation_index]
y_train, y_validation = y.iloc[train_index], y.iloc[validation_index]
x_train.drop(identifier_columns, axis=1, inplace=True)
 validation_identifier_data = x_validation[identifier_columns]
x_validation.drop(identifier_columns, axis=1, inplace=True)
 x_train_columns = x_train.columns
trn_data = lgb.Dataset(x_train,
label=y_train, 
)
 del x_train
del y_train
val_data = lgb.Dataset(x_validation,
label=y_validation, 
)
 classifier_model = lgb.train(lgb_params,
trn_data,
num_round,
valid_sets=[trn_data, val_data],
verbose_eval=100,
early_stopping_rounds=early_stopping_rounds,
feval=eval_auc 
)
 classifier_models.append(classifier_model)
 predictions = classifier_model.predict(x_validation, num_iteration=classifier_model.best_iteration)
 false_positive_rate, recall, thresholds = metrics.roc_curve(y_validation, predictions)
 score = metrics.auc(false_positive_rate, recall)
 validation_scores.append(score)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = x_train_columns
fold_importance_df["importance"] = classifier_model.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_index + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 if stop_after_fold_number == fold_index + 1:
break
if save_feature_importances:
cols =(feature_importance_df[["feature", "importance"]] 
.groupby("feature")
.mean() 
.sort_values(by="importance", ascending=False)[:1000].index)
 best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(14, 25))
sns.barplot(x="importance",
y="feature",
data=best_features.sort_values(by="importance",
ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances.png')
 best_features.sort_values(by="importance", ascending=False)\ 
.groupby("feature")\ 
.mean() \ 
.sort_values(by="importance", ascending=False)\ 
.to_csv('feature_importances_new.csv', index=True)
 score = sum(validation_scores)/ len(validation_scores)
 return classifier_models, score
train_fix = train_fix.fillna(0)
 test_fix = test_fix.fillna(0 )<prepare_x_and_y>
models, validation_score = train_model(train.drop('HasDetections', axis=1),
train_y, base_params,
num_round=5120,
early_stopping_rounds=200,
stop_after_fold_number=stop_after_fold_number,
save_feature_importances=True)
 del train
X, y = train_fix.drop(["revenue"], axis=1), train_dm["revenue"]
test_fix = test_fix.drop("revenue", axis=1 )<split>
submission_data = test[['MachineIdentifier']]
predictions = np.zeros(len(test))
test = test.drop('MachineIdentifier', axis=1)
 chunk_size = 1000000
for classifier_model in tqdm(models):
current_pred = np.zeros(len(test))
initial_idx = 0
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = classifier_model.predict(test.iloc[idx],
num_iteration=classifier_model.best_iteration)
 initial_idx = final_idx
predictions += current_pred / len(models)
 del test
kX = X.drop(X.loc[:,X.columns.str.startswith('gOpen')], axis=1)
 gX = X.drop(X.loc[:,X.columns.str.startswith('kOpen')], axis=1)
 ktest_fix = test_fix.drop(test_fix.loc[:,test_fix.columns.str.startswith('gOpen')], axis=1)
 gtest_fix = test_fix.drop(test_fix.loc[:,test_fix.columns.str.startswith('kOpen')], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, shuffle=True, random_state=42)
<choose_model_class>
submission_data['HasDetections'] = predictions
filename = 'submission_{:.6f}_{}_folds_{}_data.csv'.format(validation_score,
dt.now().strftime('%Y-%m-%d-%H-%M'),
len(models))
submission_data.to_csv('single_{}'.format(filename), index=False )