Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<set_options>
import numpy as np
import pandas as pd
import os
%%javascript
IPython.OutputArea.auto_scroll_threshold = 9999;<load_from_csv>
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape
random.seed(35)
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' )<data_type_conversions>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )
def downcast_df(df):
before = df.memory_usage()
cols_to_skip = df.isna().sum() [df.isna().sum() > 0].index
numcols = [
col for col in df.select_dtypes("number" ).columns if col not in cols_to_skip
]
ints = df.select_dtypes(int ).columns
floats = df.select_dtypes(float ).columns
for col in ints:
df[col] = pd.to_numeric(df[col].values, downcast="integer")
 for col in floats:
df[col] = pd.to_numeric(df[col].values, downcast="float")
 after = df.memory_usage()
reduction =(before - after)/ before * 100
comparison = pd.concat([before, after, reduction], axis=1)
 comparison.columns = [
"before",
"after",
"reduction(%)",
]
print(f"Downcasting:
{comparison}
")
 return df
df = downcast_df(train)
 df = downcast_df(test)
<feature_engineering>
def make_filename(idx):
return glob.glob('.. /input/moa-1082020/' + str(idx)+ '.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx))
return temp
predictions = [read_predictions(idx)for idx in range(1,18)]
len(predictions )
test = test.drop('ID', axis=1)
 test['date'] = '01.11.2015'
test['date_block_num'] = 34
test['item_cnt_day'] = np.nan
test['item_price'] = np.nan<filter>
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
<filter><EOS> 
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T