Kaggle competition: Mechanisms of Action (MoA) Prediction
tmp = pd.DataFrame()
for result in res:
uid = result[0]
ret = result[1].rename(uid)
 tmp = pd.concat([tmp, ret], axis=1)
 fbp_pd = tmp.T
fbp_pd.index.name = "id"
fbp_pd.head(10 )<load_pretrained>
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape
fbp_pd.to_pickle("model_fit.pkl.gz", compression="gzip" )<merge>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )
yhat_df = pd.merge(yhat_df, df_train.reset_index() [['id', 'level', 'state_id', 'store_id', 'cat_id', 'dept_id', 'item_id']], on=["id"], how="left")
 print(yhat_df.shape)
 yhat_df.head()<define_variables>
kernels = pd.read_csv('/kaggle/input/best-kernels/kernels.csv', index_col='id' )
level_coef_dict = {
12: ["id"],
10: ["item_id"],
5: ["dept_id"],
4: ["cat_id"],
3: ["store_id"],
2: ["state_id"],
1: ["all"],
11: ["state_id", "item_id"],
7: ["state_id", "dept_id"],
9: ["store_id","dept_id"],
6: ["state_id", "cat_id"],
8: ["store_id","cat_id"],
}<prepare_output>
def make_filename(idx):
return glob.glob('/kaggle/input/best-kernels/' + str(idx)+ '__submission.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx))
return temp
predictions = [read_predictions(idx)for idx in kernels.index]
len(predictions )
def predict_topdown(df, df_level12, items):
df_level_forecast = pd.DataFrame()
for idx, row in df.iterrows() :
item1 = items[0]
if item1 is not "all":
item1_id = row[item1]
if len(items)== 2:
item2 = items[1]
item2_id = row[items[1]]
if item1 is not "all":
if len(items)== 2:
df_item = df_level12.loc[(df_level12[item1] == item1_id)&(df_level12[item2] == item2_id)][['id']]
else:
df_item = df_level12.loc[df_level12[item1] == item1_id][['id']]
else:
df_item = df_level12[['id']]
if item1 is not "all":
if len(items)== 2:
df_item['val'] = df_level12[(df_level12[item1] == item1_id)&(df_level12[item2] == item2_id)].iloc[:, np.r_[0,-28:0]].sum(axis = 1)
 else:
df_item['val'] = df_level12[df_level12[item1] == item1_id].iloc[:, np.r_[0,-28:0]].sum(axis = 1)
 else:
df_item['val'] = df_level12.iloc[:, np.r_[0,-28:0]].sum(axis = 1)
 for i in range(1,29):
col = "d_%d" %(1913 + i)
 p_col = "F%d" % i
df_item[p_col] =(df_item['val'] * float(row[col])/ df_item['val'].sum())
 df_level_forecast = pd.concat([df_level_forecast, df_item])
 return df_level_forecast.drop(columns=["val"] )<load_pretrained>
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
for key, value in level_coef_dict.items() :
if(key <= MAX_LEVEL)and(key >= 1):
predictions = yhat_df[yhat_df["level"] == key]
df_levelx_forecast = predict_topdown(predictions, df_sale, value)
 print("Top-Down prediction for level %s, %s, items=%s into %s" %(key, value, predictions.shape[0], df_levelx_forecast.shape[0]))
df_levelx_forecast.to_pickle(MODELS_DIR + "/level_%d.pkl.gz" % key, compression="gzip")
 print(df_levelx_forecast.shape)
 df_levelx_forecast.head()<load_from_csv>
scores = 1.- kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]
SAMPLE_SUBMISSION = TRAIN_DATA_HOME + "sample_submission.csv"
sub_ids = pd.read_csv(SAMPLE_SUBMISSION)[['id']]
sub_ids = sub_ids[sub_ids.id.str.endswith("validation")]<categorify>
weighted_y_pred = pd.DataFrame()
weighted_y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column] * weights[i])
 weighted_y_pred[column] = np.sum(column_data, axis=0)
 weighted_y_pred.shape
for key, value in level_coef_dict.items() :
if(key <= MAX_LEVEL)and(key >= 1):
predictions = yhat_df[yhat_df["level"] == key]
df_levelx_forecast = predict_topdown(predictions, df_sale, value)
 print("Top-Down prediction for level %s, %s, items=%s into %s" %(key, value, predictions.shape[0], df_levelx_forecast.shape[0]))
df_levelx_forecast["id"] = df_levelx_forecast["id"].astype(str)+ "_validation"
part1 = pd.merge(sub_ids, df_levelx_forecast, on="id", how="left")
 part2 = part1.copy()
part2["id"] = part2["id"].str.replace("validation$", "evaluation")
 sub = pd.concat([part1, part2])
 MODE_LEVEL_PATH = MODELS_DIR + "/level_%d" % key
if not os.path.exists(MODE_LEVEL_PATH):
os.makedirs(MODE_LEVEL_PATH)
 sub.to_csv(MODE_LEVEL_PATH + "/submission.csv.gz", compression="gzip", index=False )<set_options>
y_pred = weighted_y_pred
<load_from_csv><EOS> 
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T