Kaggle competition: Mechanisms of Action (MoA) Prediction
df_train = pd.DataFrame(training_tasks, columns=['id'])
 df_train['id'] = df_train['id'].apply(lambda x: x[:-5])
 df_train.head()<feature_engineering>
from sklearn.metrics import log_loss
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.backend import clear_session
def get_num_train(x):
data = get_data("{}/{}.json".format(training_path, x))
ans = len(data['train'])
 return ans
df_train['num_train'] = -1
df_train['num_train'] = df_train['id'].apply(lambda x: get_num_train(x))
df_train.head()<feature_engineering>
train_x = pd.read_csv('.. /input/lish-moa/train_features.csv')
 test_x = pd.read_csv('.. /input/lish-moa/test_features.csv' )
def get_num_test(x):
data = get_data("{}/{}.json".format(training_path, x))
ans = len(data['test'])
 return ans
df_train['num_test'] = -1
df_train['num_test'] = df_train['id'].apply(lambda x: get_num_test(x))
df_train.head()<categorify>
train_y = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' )
def get_inputshape_is_output_shape(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if input_array.shape!=output_array.shape:
ans = False
return ans
df_train['inputshape_is_output_shape'] = -1
df_train['inputshape_is_output_shape'] = df_train['id'].apply(lambda x: get_inputshape_is_output_shape(x))
df_train.head()<categorify>
data = pd.concat([train_x, test_x], axis=0)
 data = data.reset_index()
def get_inputshape_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
input_shape = np.array(data['train'][0]['input'] ).shape
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if input_array.shape!=input_shape:
ans = False
return ans
df_train['inputshape_is_same'] = -1
df_train['inputshape_is_same'] = df_train['id'].apply(lambda x: get_inputshape_is_same(x))
df_train.head()<categorify>
data['cp_type'].value_counts()
def get_outputshape_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
output_shape = np.array(data['train'][0]['output'] ).shape
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if output_array.shape!=output_shape:
ans = False
return ans
df_train['outputshape_is_same'] = -1
df_train['outputshape_is_same'] = df_train['id'].apply(lambda x: get_outputshape_is_same(x))
df_train.head()<count_unique_values>
c_cols = []
g_cols = []
for colname in data.columns:
if colname.startswith('c-'):
c_cols.append(colname)
 if colname.startswith('g-'):
g_cols.append(colname )
def get_inputunique_is_outputunique(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(input_array ).shape!=np.unique(output_array ).shape:
ans = False
else:
for j in range(len(np.unique(input_array))):
if np.unique(input_array)[j]!=np.unique(output_array)[j]:
ans = False
return ans
df_train['inputunique_is_outputunique'] = -1
df_train['inputunique_is_outputunique'] = df_train['id'].apply(lambda x: get_inputunique_is_outputunique(x))
df_train.head()<count_unique_values>
somthing_rate = 1e-15
P_MIN = somthing_rate
P_MAX = 1 - P_MIN
def loss_fn(yt, yp):
yp = np.clip(yp, P_MIN, P_MAX)
 return log_loss(yt, yp, labels=[0,1] )
def get_inputunique_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
input_unique = np.unique(np.array(data['train'][0]['input']))
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(input_array ).shape!=input_unique.shape:
ans = False
else:
for j in range(len(input_unique)) :
if np.unique(input_array)[j]!=input_unique[j]:
ans = False
return ans
df_train['inputunique_is_same'] = -1
df_train['inputunique_is_same'] = df_train['id'].apply(lambda x: get_inputunique_is_same(x))
df_train.head()<count_unique_values>
train = data.copy()
train = train.drop(['sig_id', 'index'], axis=1 )
def get_outputunique_is_same(x):
data = get_data("{}/{}.json".format(training_path, x))
len_train = len(data['train'])
 ans = True
output_unique = np.unique(np.array(data['train'][0]['output']))
for i in range(len_train):
input_array = np.array(data['train'][i]['input'])
 output_array = np.array(data['train'][i]['output'])
 if np.unique(output_array ).shape!=output_unique.shape:
ans = False
else:
for j in range(len(output_unique)) :
if np.unique(output_array)[j]!=output_unique[j]:
ans = False
return ans
df_train['outputunique_is_same'] = -1
df_train['outputunique_is_same'] = df_train['id'].apply(lambda x: get_outputunique_is_same(x))
df_train.head()<filter>
label_encoder = LabelEncoder()
onehot_encoder = OneHotEncoder()
tmp = df_train.iloc[:,3:].values==df_train.iloc[0,3:].values
tmp = tmp.min(axis=1)
 tmp<split>
cp_dose_num = pd.DataFrame(label_encoder.fit_transform(train['cp_dose']), columns=['cp_dose_num'])
 cp_type_num = pd.DataFrame(label_encoder.fit_transform(train['cp_type']), columns=['cp_type_num'] )
tmp = df_train.iloc[:,3:].values==df_train.iloc[0,3:].values
tmp = tmp.min(axis=1)
 df_tmp = df_train[tmp]
df_tmp<train_model>
cp_type_num.value_counts()
def copy_horizontal(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred], axis=1)
 return tmp
idx = 56
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, crop0)
 data_idx = proc_base(data_idx, copy_horizontal)
 plot_task2(data_idx )<concatenate>
train = pd.concat([cp_type_num, cp_dose_num, train], axis=1)
 train = train.drop(['cp_type', 'cp_dose'], axis=1)
 train.head()
def copy_flip(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[:,::-1]], axis=1)
 return tmp
def copy_flop(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[::-1]], axis=0)
 return tmp
idx = 82
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, copy_flip)
 data_idx = proc_base(data_idx, copy_flop)
 plot_task2(data_idx )<categorify>
cp_time_onehot = pd.DataFrame(onehot_encoder.fit_transform(train['cp_time'].to_numpy().reshape(-1, 1)).toarray())
 cp_time_onehot = cp_time_onehot.add_prefix('cp_time_onehot_' )
def flip(t_in, t_out, t_pred):
tmp = t_pred[:,::-1]
return tmp
def flop(t_in, t_out, t_pred):
tmp = t_pred[::-1]
return tmp
idx = 115
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, flop)
 data_idx = proc_base(data_idx, copy_flop)
 plot_task2(data_idx )<concatenate>
train = pd.concat([cp_time_onehot, train], axis=1)
 train = train.drop(['cp_time'], axis=1)
 train.head()
def copy_rotate90(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred.T[:,::-1]], axis=1)
 return tmp
def rotate90(t_in, t_out, t_pred):
tmp = t_pred.T[:,::-1]
return tmp
def rotate270(t_in, t_out, t_pred):
tmp = t_pred.T[::-1]
return tmp
def copy_rotate180(t_in, t_out, t_pred):
tmp = np.concatenate([t_pred, t_pred[::-1,::-1]], axis=1)
 return tmp
idx = 193
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, copy_rotate90)
 data_idx = proc_base(data_idx, rotate270)
 plot_task2(data_idx )<categorify>
from sklearn.decomposition import PCA
def proc_base(data, func):
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
t_pred = data['train'][i]['predict']
try:
data['train'][i]['predict'] = func(t_in, t_out, t_pred)
 except:
data['train'][i]['predict'] = t_pred
for i in range(len(data['test'])) :
t_in = data['test'][i]['input']
t_out = data['train'][0]['output']
t_pred = data['test'][i]['predict']
try:
data['test'][i]['predict'] = func(t_in, t_out, t_pred)
 except:
data['test'][i]['predict'] = t_pred
return data
def tile_reflect(t_in, t_out, t_pred):
tmp = np.zeros_like(t_out)
 h, w = t_out.shape[0]//t_in.shape[0], t_out.shape[1]//t_in.shape[1]
for i in range(h):
for j in range(w):
tile = t_pred
if i%2==1: tile = tile[::-1]
if j%2==1: tile = tile[:, ::-1]
tmp[i*t_in.shape[0]:(i+1)*t_in.shape[0], j*t_in.shape[1]:(j+1)*t_in.shape[1]] = tile
return tmp
def rotate180(t_in, t_out, t_pred):
tmp = t_pred[::-1, ::-1]
return tmp
idx = 210
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 data_idx = proc_base(data_idx, rotate180)
 data_idx = proc_base(data_idx, tile_reflect)
 plot_task2(data_idx )<define_variables>
pca_g = PCA(n_components=2)
 pca_c = PCA(n_components=100 )
func_list = [
func_resize,
func_tile0,
slice4,
crop0,
copy_horizontal,
copy_flip,
copy_flop,
flip,
flop,
copy_rotate90,
rotate90,
rotate270,
copy_rotate180,
tile_reflect,
rotate180,
]<categorify>
train_pca_g = pca_g.fit_transform(train[g_cols])
 train_pca_c = pca_c.fit_transform(train[c_cols] )
def judge(data):
ans = True
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
t_pred = data['train'][i]['predict']
if t_out.shape!=t_pred.shape: ans = False
elif(t_out==t_pred ).min() ==0: ans = False
return ans
idx = 210
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 print(judge(data_idx))
data_idx = proc_base(data_idx, rotate180)
 data_idx = proc_base(data_idx, tile_reflect)
 print(judge(data_idx))
plot_task2(data_idx )<import_modules>
train = pd.concat([train, pd.DataFrame(train_pca_g ).add_prefix('pca_g_'),
pd.DataFrame(train_pca_c ).add_prefix('pca_c_')], axis=1 )
import copy<categorify>
means = pd.concat([train[g_cols].mean(axis=1), train[c_cols].mean(axis=1)],
keys=['c_mean', 'g_mean'], axis=1)
 train = pd.concat([train, means], axis=1 )
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 for i in range(1000):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, 2)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
plot_task2(data_tmp )<categorify>
X_train, X_val, y_train, y_val = train_test_split(train.iloc[:len(train_x)],
train_y.drop(['sig_id'], axis=1),
test_size=0.2, random_state=224 )
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
plot_task2(data_tmp )<define_variables>
def build_model(hidden_layers, neurons, dropout_rate):
model = tf.keras.Sequential([tf.keras.layers.Input(len(train.columns)) ])
 for i in range(hidden_layers):
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(dropout_rate))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(neurons // 2**i, activation='swish')))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid")))
 model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),
loss=BinaryCrossentropy(label_smoothing=0.001))
return model
idx = np.random.randint(len(df_train))
print(idx)
 data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 break
print(func_tmp)
 plot_task2(data_tmp )<import_modules>
best_model = build_model(2, 734, 0.5012546298076606 )
import copy<categorify>
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-6, patience=4, verbose=1, mode='auto')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 10, verbose = 1)
 checkpoint_path = 'model.weights'
cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only=True, mode = 'auto')
 history = best_model.fit(X_train, y_train,
batch_size = 64,
epochs = 100,
validation_data =(X_val, y_val),
callbacks = [early, reduce_lr_loss, cb_checkpt])
def proc_init(data):
for i in range(len(data['train'])) :
t_in = data['train'][i]['input']
t_out = data['train'][i]['output']
data['train'][i]['input'] = np.array(t_in)
 data['train'][i]['output'] = np.array(t_out)
 data['train'][i]['predict'] = np.array(t_in)
 rate_h = copy.deepcopy(data['train'][i]['output'].shape[0]/data['train'][i]['input'].shape[0])
 rate_w = copy.deepcopy(data['train'][i]['output'].shape[1]/data['train'][i]['input'].shape[1])
 for i in range(len(data['test'])) :
t_in = np.array(data['test'][i]['input'])
 t_out = np.zeros([int(t_in.shape[0]*rate_h),int(t_in.shape[1]*rate_w)], dtype=t_in.dtype)
 data['test'][i]['input'] = t_in
data['test'][i]['output'] = t_out
data['test'][i]['predict'] = np.array(t_in)
 return data
count = 0
for idx in range(len(df_train.iloc[:])) :
print(idx)
 data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),
]
flag = False
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 print(func_tmp)
 plot_task2(data_tmp)
 flag = True
count += 1
break
if flag: break
print(count )<categorify>
best_model.load_weights('model.weights' )
def flattener(pred):
str_pred = str([row for row in pred.tolist() ])
 str_pred = str_pred.replace(', ', '')
 str_pred = str_pred.replace('[[', '|')
 str_pred = str_pred.replace('][', '|')
 str_pred = str_pred.replace(']]', '|')
 return str_pred<categorify>
pred = best_model.predict(train.iloc[len(train_x):] )
idx = 0
data_idx = get_data("{}/{}.json".format(training_path, df_train['id'][idx]))
data_idx = proc_init(data_idx)
 try_list = [(1, len(func_list)) ,(2, len(func_list)**2),(3, len(func_list)**3)]
flag = False
for num_func, num_try in try_list:
for i in range(num_try):
data_tmp = copy.deepcopy(data_idx)
 func_tmp = np.random.choice(func_list, num_func)
 for func in func_tmp:
data_tmp = proc_base(data_tmp, func)
 if judge(data_tmp):
print(i, "Find!")
 flag = True
break
if flag: break
if flag: break
print(data_tmp['test'][0]['predict'])
 print(flattener(data_tmp['test'][0]['predict']))
plot_task2(data_tmp )<load_from_csv>
pred_df = pd.DataFrame(pred, columns=train_y.columns[1:])
 submmission_df = pd.concat([test_x['sig_id'], pred_df], axis=1 )
df_sub = pd.read_csv(".. /input/abstraction-and-reasoning-challenge/sample_submission.csv")
 df_sub.head()<prepare_output>
submmission_df.to_csv('submission.csv', index=False )
df_test = pd.DataFrame(test_tasks, columns=['id'])
 df_test['id'] = df_test['id'].apply(lambda x: x[:-5])
 df_test.head()<categorify>
pd.read_csv('submission.csv' )