Kaggle competition: Data Champions Android App Malware Prediction
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_city_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', "shop_city"], how='left')
 matrix.date_city_avg_item_cnt = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_city_avg_item_cnt'])
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df = pd.read_csv('.. /input/TWHM-WebAttacksTrain.csv', index_col='IDNum' )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 matrix.date_item_city_avg_item_cnt = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_item_city_avg_item_cnt'])
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
def to_int(obj):
return int(re.sub("[^\d]", '', obj))
df['Source IP'] = df['Source IP'].apply(to_int)
 df['Destination IP'] = df['Destination IP'].apply(to_int)
 df['Timestamp'] = df['Timestamp'].apply(to_int)
 df = df.drop(['Usage'], axis=1)
 df = df.drop(['Flow ID'], axis=1)
 df = df.replace([np.nan, 'Infinity'], 2**31-1 )
ts = time.time()
group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["item_id"], how = "left")
 matrix["item_avg_item_price"] = matrix.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["date_block_num","item_id"], how = "left")
 matrix["date_item_avg_item_price"] = matrix.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
matrix = lag_feature(matrix, lags, ["date_item_avg_item_price"])
 for i in lags:
matrix["delta_price_lag_" + str(i)] =(matrix["date_item_avg_item_price_lag_" + str(i)]- matrix["item_avg_item_price"])/ matrix["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
matrix["delta_price_lag"] = matrix.apply(select_trends, axis = 1)
 matrix["delta_price_lag"] = matrix.delta_price_lag.astype(np.float16)
 matrix["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
matrix.drop(features_to_drop, axis = 1, inplace = True)
 time.time() - ts 
<merge>
X = df.iloc[:, :-1]
y = df.iloc[:, -1]
ts = time.time()
group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["shop_id"], how = "left")
 matrix["shop_avg_revenue"] = matrix.shop_avg_revenue.astype(np.float32)
 matrix["delta_revenue"] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix["delta_revenue"] = matrix["delta_revenue"].astype(np.float32)
 matrix = lag_feature(matrix, [1], ["delta_revenue"])
 matrix["delta_revenue_lag_1"] = matrix["delta_revenue_lag_1"].astype(np.float32)
 matrix.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 time.time() - ts 
<data_type_conversions>
enc = LabelEncoder()
y = enc.fit_transform(y)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)
 ohe = OneHotEncoder()
y_train_ohe = ohe.fit_transform(y_train.reshape(-1, 1))
y_test_ohe = ohe.fit_transform(y_test.reshape(-1, 1))
y_ohe = ohe.fit_transform(y.reshape(-1, 1))
scaler = StandardScaler()
X_scale = scaler.fit_transform(X)
 X_train_scale = scaler.fit_transform(X_train)
 X_test_scale = scaler.transform(X_test )
matrix["days"] = matrix["month"].map(days ).astype(np.int8 )<feature_engineering>
model = Sequential()
model.add(Dense(units=64, activation='relu', input_dim=83))
model.add(Dense(units=16, activation='relu'))
model.add(Dense(units=4, activation='softmax'))
ts = time.time()
matrix["item_shop_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 matrix["item_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id"])["date_block_num"].transform('min')
 time.time() - ts<feature_engineering>
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'] )
ts = time.time()
matrix = matrix[matrix["date_block_num"] > 3]
time.time() - ts<set_options>
history = model.fit(X_train_scale,
y_train_ohe,
epochs=80,
batch_size=512,
validation_data=(X_test_scale, y_test_ohe))
rcParams['figure.figsize'] = 12, 4<drop_column>
enc.inverse_transform(model.predict_classes(X_scale))
data = matrix.copy()
del matrix
gc.collect()<prepare_x_and_y>
dftrain = pd.DataFrame(enc.inverse_transform(model.predict_classes(X)) , columns=['Label'], index=X.index ).reset_index()
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
dftrain[dftrain['Label'] != 'BENIGN']
del data
gc.collect() ;<train_model>
datatest = pd.read_csv('.. /input/TWHM-WebAttacksTestPrivat.csv', index_col='IDNum' )
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts<save_to_csv>
datatest['Source IP'] = datatest['Source IP'].apply(to_int)
 datatest['Destination IP'] = datatest['Destination IP'].apply(to_int)
 datatest['Timestamp'] = datatest['Timestamp'].apply(to_int)
 datatest = datatest.drop(['Usage'], axis=1)
 datatest = datatest.drop(['Flow ID'], axis=1)
 datatest = datatest.replace([np.nan, 'Infinity'], 2**31-1)
 datatest_scale = scaler.transform(datatest )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<load_from_csv>
dftest = pd.DataFrame(enc.inverse_transform(model.predict_classes(datatest_scale)) , columns=['Label'], index=datatest.index ).reset_index()
ssub=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/sample_submission.csv" )<load_from_csv>
pd.read_csv('.. /input/TWHM-WebAttacksKeySample.csv' )
xgboost_lagged_features = pd.read_csv(".. /input/xgboost/xgboost_lagged_features.csv" )<load_from_csv>
an = pd.concat([dftest, dftrain] )
linear_regression_lagged_features_6 = pd.read_csv(".. /input/linearregression/linear_regression_lagged_features_6.csv" )<load_from_csv>
an.sort_values('IDNum' )
random_forest_lagged_features = pd.read_csv(".. /input/randfor/random_forest_lagged_features.csv" )<feature_engineering>
an.to_csv('submission.csv', index=False )