Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
class simple_mpl(object):
def __init__(self, path, file_head):
print("Loading", path, "+", file_head)
 file_list = os.listdir(path)
 num = len(file_head + "_")
 self.models = []
for file in file_list:
if file[:num] == file_head + "_":
self.models.append(tf.keras.models.load_model(os.path.join(path, file)))
 cfg = pd.read_pickle(os.path.join(path, 'config.pkl'))
self.mean = cfg['mean']
self.std =cfg['std']
self.output_num = cfg["output_num"].max()
def predict(self, data):
pred = pd.DataFrame(np.zeros([data.index.size, self.output_num]), index=data.index)
 for mdl in self.models:
pred += mdl.predict(( data.astype("float32")- self.mean)/ self.std, batch_size=4096)
 pred /= len(self.models)
 return pred
start = datetime.now()
chi2_target = pd.DataFrame(columns=['Cochran_criterion', 'Chi2', 'p_value'])
 for col in df.columns.drop('target'):
feature = df[col]
chi2_target.loc[col + '_target'] = chi2_test(feature, df.target)
 chi2_target['rejected'] = chi2_target['Cochran_criterion'] &(chi2_target['p_value']<0.05)
 chi2_target.sort_values('p_value', ascending=True, inplace=True)
 chi2_target.to_csv('chi2_target_tests.csv', index=False)
 print(chi2_target, '
')
 runtime = datetime.now() - start
print('Runtime ', runtime )<count_values>
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_target = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_size = train.shape[0]
traintest = pd.concat([train, test])
 traintest.drop(['cp_type'], axis=1, inplace=True)
 traintest = pd.concat([traintest, pd.get_dummies(traintest['cp_time'], prefix='cp_time')], axis=1)
 traintest = pd.concat([traintest, pd.get_dummies(traintest['cp_dose'], prefix='cp_dose')], axis=1)
 traintest = traintest.drop(['cp_time', 'cp_dose'], axis=1)
 train = traintest[:train_size]
test = traintest[train_size:]
del traintest
x_train = train.drop('sig_id', axis=1)
 y_train = train_target.drop('sig_id', axis=1 ).astype(np.float32)
 x_test = test.drop('sig_id', axis=1 )
for col in ['nom_'+str(i)for i in range(5,10)]:
rare_categories = df[col].value_counts().iloc[np.where(df[col].value_counts() <100)].index
print("The predictor " + col + " has ", rare_categories.size, " rare categories among its ", df[col].unique().size,
" categories." )<count_unique_values>
mdl = simple_mpl(path=".. /input/mplmodel11/mpl", file_head="")
 remove_cols=['g-39', 'g-85', 'g-75', 'g-734', 'g-484', 'g-217', 'g-545', 'g-10']
x_test = x_test[np.setdiff1d(x_test.columns.values, remove_cols)]
res = mdl.predict(x_test)
 submit = pd.DataFrame(res.values, index = test['sig_id'])
 submit.columns = y_train.columns
submit.to_csv("submission.csv")
 submit