Kaggle competition: Abstraction and Reasoning Challenge
<SOS> metric: MeanBestErrorAtK Kaggle data source: abstraction-and-reasoning-challenge<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import colors
import os
import json
import random
import warnings
import datetime
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values<string_transform>
def get_filenames(directory):

 return [os.path.join(dirname, filename)for dirname, _, filenames in os.walk(directory)for filename in filenames]
def import_json_to_df(filename):

 with open(filename, 'r')as f:
task = json.load(f)
 result = []
for key in task:
df = pd.DataFrame(task[key])
 df['task_id'] = os.path.splitext(os.path.basename(filename)) [0]
if key == 'test':
df['submission_id'] = df.apply(lambda x: f"{x['task_id']}_{x.name}", axis=1)
 df['type'] = key
result.append(df)
 result = pd.concat(result, ignore_index=True, sort=False)
 result['input'] = result['input'].apply(np.array)
 result['output'] = result['output'].apply(np.array)
 return result
data_dir = '/kaggle/input/abstraction-and-reasoning-challenge/'
submission = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
training_files = sorted(get_filenames(os.path.join(data_dir, 'training')))
 evaluation_files = sorted(get_filenames(os.path.join(data_dir, 'evaluation')))
 test_files = sorted(get_filenames(os.path.join(data_dir, 'test')))
 train = pd.concat([import_json_to_df(f)for f in training_files], ignore_index=True, sort=False)
 validation = pd.concat([import_json_to_df(f)for f in evaluation_files], ignore_index=True, sort=False)
 test = pd.concat([import_json_to_df(f)for f in test_files], ignore_index=True, sort=False)
 train.shape, validation.shape, test.shape, submission.shape
print(f'Train: {train.shape}')
 print(f'Validation: {validation.shape}')
 print(f'Test: {test.shape}')
 print(f'Submission: {submission.shape}' )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<compute_test_metric>
print('Example with 1 test pair')
 display(train[train['task_id']=='007bbfb7'])
 print('Example with 2 test pairs')
 display(train[train['task_id']=='e9614598'] )
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))<feature_engineering>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_colwidth', -1)
 SEED = 13
random.seed(SEED)
 np.random.seed(SEED)
 MAX_SIZE = 30
PAD_ID = -1
PAD_COLOR = '
ORG_COLOR_MAP = {0:'
5:'
NUM_COLORS = len(ORG_COLOR_MAP)
 COLOR_MAP = {PAD_ID:PAD_COLOR}
COLOR_MAP.update(ORG_COLOR_MAP)
 DEFAULT_PREDICTION = [[0]]
DEFAULT_SUBMISSION = '|0| |0| |0|'
DF_NAME = {id(train): 'train', id(validation): 'validation', id(test): 'test'}
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
def calculate_score(truth, pred1, pred2=None, pred3=None):

 result = []
result.append([np.array_equal(truth[i], pred1[i])for i in range(len(truth)) ])
 if pred2 is not None:
result.append([np.array_equal(truth[i], pred2[i])for i in range(len(truth)) ])
 if pred3 is not None:
result.append([np.array_equal(truth[i], pred3[i])for i in range(len(truth)) ])
 result = np.vstack(result)
 result = np.any(result, axis=0)
 score =(len(result)-np.sum(result)) /len(result)
 print(f'Score: {np.round(score, 4)}(Found {np.sum(result)}/{len(result)})')
 return score
def prediction_to_submission(pred1, pred2=None, pred3=None):

 def list_to_str(pred):
return '|'+'|'.join([''.join(map(str, r)) for r in pred])+'|'
result = list_to_str(pred1)
 if pred2 is not None:
result += ' '+list_to_str(pred2)
 else:
result += ' '+list_to_str(DEFAULT_PREDICTION)
 if pred3 is not None:
result += ' '+list_to_str(pred3)
 else:
result += ' '+list_to_str(DEFAULT_PREDICTION)
 return result
def submission_to_prediction(sub):

 pred = [np.array([list(map(int, list(s)))for s in sub.split() [i].split('|')[1:-1]])for i in range(3)]
return pred[0], pred[1], pred[2]
def image_padding(img, size=(MAX_SIZE, MAX_SIZE), pad_int=-1):

 top = int(np.ceil(( size[0] - img.shape[0])/2))
bottom = int(np.floor(( size[0] - img.shape[0])/2))
left = int(np.ceil(( size[1] - img.shape[1])/2))
right = int(np.floor(( size[1] - img.shape[1])/2))
return np.pad(img,(( top, bottom),(left, right)) , constant_values=pad_int)
 def remove_padding(img, pad_int=-1):

 row_pad = np.all(img == pad_int, axis=1)
 img = img[~row_pad,:]
col_pad = np.all(img == pad_int, axis=0)
 img = img[:,~col_pad]
return img
def plot_task(df, task_id):

 cmap = colors.ListedColormap(COLOR_MAP.values())
 norm = colors.Normalize(vmin=-1, vmax=9)
 df_name = DF_NAME[id(df)]
df = df[df['task_id'] == task_id].sort_values('type', ascending=False ).reset_index()
fig, axs = plt.subplots(2, df.shape[0], figsize=(3*df.shape[0],6))
axs[0][0].text(0,-10, f'Task ID: {task_id}({df_name})', fontsize=16)
 for idx, row in df.iterrows() :
input_img = row['input']
input_img = image_padding(input_img)
 output_img = row['output']
if np.isnan(output_img ).any() :
output_img = np.array([[]])
 output_img = image_padding(output_img)
 task_type = row['type'].upper()
axs[0][idx].imshow(input_img, cmap=cmap, norm=norm)
 axs[0][idx].set_yticklabels([])
 axs[0][idx].set_xticklabels([])
 axs[0][idx].set_title(f'{task_type}
Input', color='green' if task_type == 'TRAIN' else 'blue', fontsize=16)
 axs[1][idx].imshow(output_img, cmap=cmap, norm=norm)
 axs[1][idx].set_yticklabels([])
 axs[1][idx].set_xticklabels([])
 axs[1][idx].set_title('Output', color='green' if task_type == 'TRAIN' else 'blue', fontsize=16)
 plt.tight_layout()
plt.show()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
pred = np.array([[1,0,1], [0,1,0], [1,0,1], [1,1,1]])
 print(f'Original Prediciton:
{pred}')
 pred_sub = prediction_to_submission(pred)
 print(f'
Prediction to Submission:
{pred_sub}')
 sub_pred = submission_to_prediction(pred_sub)[0]
print(f'
Submission to Prediction:
{sub_pred}' )
model.fit(X_t, y, batch_size=32, epochs=2, validation_split=0.1);<save_to_csv>
for df in [train, validation, test]:
print(f'DataFrame: {DF_NAME[id(df)]}')
 score = calculate_score(df[df['type'] == 'test']['output'].values, [[[0]]]*df.shape[0], [[[1]]]*df.shape[0], [[[2]]]*df.shape[0] )
comp1 = 'jigsaw-toxic-comment-classification-challenge/'
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp1}sample_submission.csv')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('submission.csv', index=False )<import_modules>
def get_features(row):
def increased_height() :

 if row['type'] == 'test':
return None
else:
return row['output'].shape[0] / row['input'].shape[0] - 1
def increased_width() :

 if row['type'] == 'test':
return None
else:
return row['output'].shape[1] / row['input'].shape[1] - 1
def color_ratio(frame):

 if np.isnan(frame ).any() :
return None
else:
result = {i:0.0 for i in range(NUM_COLORS)}
unique, counts = np.unique(frame, return_counts=True)
 counts = counts /(frame.shape[0] * frame.shape[1])
 result.update(dict(zip(unique, counts)))
 return result
def get_objects(img):

 if np.isnan(img ).any() :
return None
else:
org = img.copy()
img = image_padding(img,(img.shape[0]+2, img.shape[1]+2), 0)
 objects = []
while True:
start = np.where(img != 0)
 if len(start[0])== 0:
break
start_x = start[1][0]
start_y = start[0][0]
end_x = start_x
end_y = start_y
control = set([(start_y, start_x)])
 while len(control)> 0:
c = control.pop()
act_y = c[0]
act_x = c[1]
if act_y > end_y: end_y = act_y
if start_y > act_y: start_y = act_y
if act_x > end_x: end_x = act_x
if start_x > act_x: start_x = act_x
img[act_y, act_x] = 0
sub = img[act_y-1:act_y+2, act_x-1: act_x+2]
for n in zip(np.where(sub)[0], np.where(sub)[1]):
n =(n[0]-1+act_y, n[1]-1+act_x)
 img[n[0], n[1]] = 0
control = control | set([n])
 img[start_y:end_y+1, start_x:end_x+1] = 0
start_y -= 1; end_y -= 1; start_x -= 1; end_x -= 1
obj = org[start_y:end_y+1, start_x:end_x+1]
objects.append({'start':(start_y, start_x), 'obj':obj})
 return objects
def has_vertical_split(frame_in=row['input'], frame_out=row['output']):

 if row['type'] == 'test':
return False
if(np.isnan(frame_in ).any())|(np.isnan(frame_out ).any()):
return False
else:
if frame_in.shape[1] % 2 == 0:
return False
mid = int(np.floor(frame_in.shape[1] / 2))
mid_color = np.unique(frame_in[:,mid])
 if mid_color.shape[0] > 1:
return False
mid_color = mid_color[0]
if mid_color in np.delete(frame_in, mid, 1):
return False
if frame_out.shape[1] != mid:
return False
return True
def has_horizontal_split(frame_in=row['input'], frame_out=row['output']):

 if row['type'] == 'test':
return False
if(np.isnan(frame_in ).any())|(np.isnan(frame_out ).any()):
return False
else:
if frame_in.shape[0] % 2 == 0:
return False
mid = int(np.floor(frame_in.shape[0] / 2))
mid_color = np.unique(frame_in[mid,:])
 if mid_color.shape[0] > 1:
return False
mid_color = mid_color[0]
if mid_color in np.delete(frame_in, mid, 0):
return False
if frame_out.shape[0] != mid:
return False
return True
def is_subset(frame, subframe):

 if row['type'] == 'test':
return False
if(np.isnan(frame ).any())|(np.isnan(subframe ).any()):
return False
else:
if(subframe.shape[0] > frame.shape[0])|(subframe.shape[1] > frame.shape[1]):
return False
for i in range(frame.shape[0]-subframe.shape[0]+1):
for j in range(frame.shape[1]-subframe.shape[1]+1):
match = np.array_equal(subframe, frame[i:i+subframe.shape[0], j:j+subframe.shape[1]])
 if match:
return match
return False
def color_change() :

 if row['type'] == 'test':
return False
else:
frame1 = np.where(row['input'] > 0, 1, 0)
 frame2 = np.where(row['output'] > 0, 1, 0)
 return np.array_equal(frame1, frame2)
 def adapted_size() :

 if row['type'] == 'test':
return False
else:
adapted_input = np.resize(row['input'], row['output'].shape)
 adapted_input = np.array_equal(adapted_input, row['output'])
 return adapted_input
result = {
'increased_height': increased_height() ,
'increased_width': increased_width() ,
'color_ratio_input': color_ratio(row['input']),
'color_ratio_output': color_ratio(row['output']),
'input_objects': get_objects(row['input']),
'output_objects': get_objects(row['output']),
'color_change': color_change() ,
'has_vertical_split': has_vertical_split() ,
'has_horizontal_split': has_horizontal_split() ,
'input_in_output': is_subset(row['output'], row['input']),
'output_in_input': is_subset(row['input'], row['output']),
}
return result
def get_features2(row):
def check_fill_out() :

 if(row['increased_width'] == 0)&(row['increased_height'] == 0):
output = row['output'].copy()
blanks = np.where(row['input'] == 0, True, False)
 output[blanks == True] = 0
return np.array_equal(row['input'], output)
 else:
return False
def fill_out_mask_color() :

 if(row['increased_width'] == 0)&(row['increased_height'] == 0):
diff = np.unique(row['input'][row['input'] != row['output']])
 if len(diff)== 1:
return diff[0]
else:
return -1
else:
return -1
def is_most_common_color() :

 if(row['increased_width'] == 0)&(row['increased_height'] == 0):
mc_color = np.argmax(list(row['color_ratio_input'].values()))
return np.sum(np.where(row['output'] == mc_color, 1, 0)) == row['output'].shape[0]*row['output'].shape[1]
else:
return False
def unique_output_color() :

 if row['color_ratio_output'] is None:
return -1
else:
max_color = np.argmax(list(row['color_ratio_output'].values())[1:])+1
if len(np.unique(list(row['color_ratio_output'].values())[1:])) > 2:
return 0
else:
return max_color
result = {
'fill_out': check_fill_out() ,
'fill_out_mask_color': fill_out_mask_color() ,
'is_most_common_color': is_most_common_color() ,
'unique_output_color': unique_output_color()
}
return result
def get_features3(row):
def left_diagonal_symmetric(frame):

 if row['type'] == 'test':
return False
else:
return np.array_equal(frame, frame.T)
 def right_diagonal_symmetric(frame):

 if row['type'] == 'test':
return False
else:
return np.array_equal(frame, np.flip(frame ).T)
 def horizontal_symmetric(frame):

 if row['type'] == 'test':
return False
else:
return np.array_equal(frame, np.flipud(frame))
def vertical_symmetric(frame):

 if row['type'] == 'test':
return False
else:
return np.array_equal(frame, np.fliplr(frame))
result = {
'input_left_diagonal_symmetric': left_diagonal_symmetric(row['input']),
'input_right_diagonal_symmetric': right_diagonal_symmetric(row['input']),
'input_horizontal_symmetric': horizontal_symmetric(row['input']),
'input_vertical_symmetric': vertical_symmetric(row['input']),
'output_left_diagonal_symmetric': left_diagonal_symmetric(row['output']),
'output_right_diagonal_symmetric': right_diagonal_symmetric(row['output']),
'output_horizontal_symmetric': horizontal_symmetric(row['output']),
'output_vertical_symmetric': vertical_symmetric(row['output']),
}
return result
def same_array_size(df, task_id, column='input'):

 result = df[(df['type'] != 'test')&(df['task_id'] == task_id)][column].apply(lambda x: np.array(x ).shape ).unique()
return True if len(result)== 1 else False
print(os.listdir(".. /input")) 
<load_from_csv>
for df in [train, validation, test]:
start_time = datetime.datetime.now()
features = pd.DataFrame(df.apply(lambda x: get_features(x), axis=1 ).to_list())
 df[features.columns] = features
features = pd.DataFrame(df.apply(lambda x: get_features2(x), axis=1 ).to_list())
 df[features.columns] = features
features = pd.DataFrame(df.apply(lambda x: get_features3(x), axis=1 ).to_list())
 df[features.columns] = features
df['same_input_size'] = df['task_id'].apply(lambda x: same_array_size(df, x, 'input'))
df['same_output_size'] = df['task_id'].apply(lambda x: same_array_size(df, x, 'output'))
df['count_input_objects'] = df['input_objects'].apply(lambda x: x if x is None else len(x))
df['count_output_objects'] = df['output_objects'].apply(lambda x: x if x is None else len(x))
end_time = datetime.datetime.now()
print(f'{DF_NAME[id(df)]}: {end_time-start_time}' )
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv' )<prepare_x_and_y>
def get_train_average(df, task_id, column):

 return df[(df['task_id'] == task_id)&(df['type'] != 'test')][column].mean()
def check_boolean(df, task_id, column):

 return df[(df['task_id'] == task_id)&(df['type'] != 'test')][column].min()
column_types = train.columns.to_series().groupby(train.dtypes ).groups
for df in [train, validation, test]:
start_time = datetime.datetime.now()
df_tests = df[df['type'] == 'test']
for column in column_types[np.dtype('float64')]:
df.loc[df_tests.index, column] = df_tests['task_id'].apply(lambda x: get_train_average(df, x, column))
for column in column_types[np.dtype('int64')]:
df.loc[df_tests.index, column] = df_tests['task_id'].apply(lambda x: int(get_train_average(df, x, column)) if np.ceil(get_train_average(df, x, column)) == np.floor(get_train_average(df, x, column)) else -1)
 for column in column_types[np.dtype('bool')]:
df.loc[df_tests.index, column] = df_tests['task_id'].apply(lambda x: check_boolean(df, x, column))
end_time = datetime.datetime.now()
print(f'{DF_NAME[id(df)]}: {end_time-start_time}' )
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()<define_variables>
df = train
df = df[df['type'] == 'train']
max_features=100000
maxlen=150
embed_size=300<compute_train_metric>
plt_data = pd.DataFrame(index=column_types[np.dtype('bool')])
 plt_data['True per Task'] = [len(df[df[column]])for column in column_types[np.dtype('bool')]]
plt_data['True group by Tasks'] = [len(df.groupby('task_id')[column].min() [df.groupby('task_id')[column].min() ])for column in column_types[np.dtype('bool')]]
plt_data
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
test_idx = test[test['type']=='test'].index
test.loc[test_idx, 'output'] = test.loc[test_idx, 'submission_id'].apply(lambda x: validation[validation['submission_id'] == x]['output'].values[0] if x in validation['submission_id'].values else None )
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )<categorify>
def get_prediction_column(df, idx):

 next_prediction = 'prediction_3'
for idx, val in df.loc[idx, ['prediction_1', 'prediction_2', 'prediction_3']].iteritems() :
if np.array_equal(val, DEFAULT_PREDICTION)or val is None:
next_prediction = idx
break
return next_prediction
for df in [train, validation, test]:
df['prediction_1'] = [DEFAULT_PREDICTION] * df.shape[0]
df['prediction_2'] = [DEFAULT_PREDICTION] * df.shape[0]
df['prediction_3'] = [DEFAULT_PREDICTION] * df.shape[0]
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs 
<feature_engineering>
def check_startpoint(frame_in, frame_out, startpoint):

 if startpoint == 'top_left':
return np.array_equal(frame_in, frame_out[:frame_in.shape[0],:frame_in.shape[1]])
 elif startpoint == 'top_right':
return np.array_equal(frame_in, frame_out[:frame_in.shape[0],-frame_in.shape[1]:])
 elif startpoint == 'bottom_left':
return np.array_equal(frame_in, frame_out[-frame_in.shape[0]:,:frame_in.shape[1]])
 elif startpoint == 'bottom_right':
return np.array_equal(frame_in, frame_out[-frame_in.shape[0]:,-frame_in.shape[1]:])
 else:
return False
def rotations(frame, horizontal, vertical, start, hmethod='repeat', vmethod='repeat'):

 result = frame
tmp = frame
for f in range(int(horizontal)) :
if hmethod == 'repeat':
tmp = tmp
elif hmethod == 'flip':
tmp = np.flip(tmp)
 elif hmethod == 'fliplr':
tmp = np.fliplr(tmp)
 elif hmethod == 'flipud':
tmp = np.flipud(tmp)
 elif hmethod == 'rot90_3' and frame.shape[0] == frame.shape[1]:
tmp = np.rot90(tmp, 3)
 elif hmethod == 'rot90_1' and frame.shape[0] == frame.shape[1]:
tmp = np.rot90(tmp, 1)
 else:
tmp = tmp
if 'left' in start:
result = np.append(result, tmp, axis=1)
 elif 'right' in start:
result = np.append(tmp, result, axis=1)
 tmp = result
for f in range(int(vertical)) :
if vmethod == 'repeat':
tmp = tmp
elif vmethod == 'flip':
tmp = np.flip(tmp)
 elif vmethod == 'flipud':
tmp = np.flipud(tmp)
 elif vmethod == 'fliplr':
tmp = np.fliplr(tmp)
 elif vmethod == 'rot90_2':
tmp = np.rot90(tmp, 2)
 elif vmethod == 'test' and frame.shape[0] == frame.shape[1]:
tmp = np.rot90(frame, 2)
 tmp = np.append(tmp, np.rot90(frame, 3), axis=1)
 else:
tmp = tmp
if 'top' in start:
result = np.append(result, tmp, axis=0)
 elif 'bottom' in start:
result = np.append(tmp, result, axis=0)
 return result
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<choose_model_class>
if True:
for df in [train, validation, test]:
tasks = df[(df['type'] == 'test')&(df['input_in_output'])&(np.floor(df['increased_height'])== np.ceil(df['increased_height'])) &(np.floor(df['increased_width'])== np.ceil(df['increased_width'])) ]['task_id'].unique()
df_test = df[(df['type'] == 'test')&(df['task_id'].isin(tasks)) ].copy()
for task in tasks:
df_temp = df[(df['type'] != 'test')&(df['task_id'] == task)].copy()
df_temp['top_left'] = df_temp.apply(lambda x: check_startpoint(x['input'], x['output'], 'top_left'), axis=1)
 df_temp['top_right'] = df_temp.apply(lambda x: check_startpoint(x['input'], x['output'], 'top_right'), axis=1)
 df_temp['bottom_left'] = df_temp.apply(lambda x: check_startpoint(x['input'], x['output'], 'bottom_left'), axis=1)
 df_temp['bottom_right'] = df_temp.apply(lambda x: check_startpoint(x['input'], x['output'], 'bottom_right'), axis=1)
 starts = df_temp[['top_left', 'top_right', 'bottom_left', 'bottom_right']].min()
starts = list(starts[starts].index)
 if len(starts)== 0:
df_test = df_test[df_test['task_id'] != task]
continue
found = None
for start in starts:
for hmethod in ['repeat', 'flip', 'fliplr', 'flipud', 'rot90_3', 'rot90_1']:
for vmethod in ['repeat', 'flip', 'fliplr', 'flipud', 'rot90_2', 'test']:
df_temp[f'pred_{start}'] = df_temp.apply(lambda x: rotations(x['input'], x['increased_width'], x['increased_height'], start, hmethod, vmethod), axis=1)
 df_temp[f'pred_{start}'] = df_temp.apply(lambda x: np.array_equal(x[f'pred_{start}'], x['output']), axis=1)
 if df_temp[f'pred_{start}'].min() :
found = True
break
if found:
break
if found:
for idx, row in df_test[df_test['task_id'] == task].iterrows() :
prediction = rotations(row['input'], row['increased_width'], row['increased_height'], start, hmethod, vmethod)
 df_test.loc[idx, get_prediction_column(df_test, idx)] = [prediction]
break
print(f'
DataFrame: {DF_NAME[id(df)]}')
 score = calculate_score(df_test['output'].values, df_test['prediction_1'].values, df_test['prediction_2'].values, df_test['prediction_3'].values)
 df.loc[df_test.index, ['prediction_1', 'prediction_2', 'prediction_3']] = df_test[['prediction_1', 'prediction_2', 'prediction_3']]
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )<split>
def split_frame(frame, split='vertical'):

 if split=='vertical':
mid = int(np.floor(frame.shape[1]/2))
frame1 = frame[:,:mid]
frame2 = frame[:,mid+1:]
else:
mid = int(np.floor(frame.shape[0]/2))
frame1 = frame[:mid,:]
frame2 = frame[mid+1:,:]
return frame1, frame2
def splitting(frame, split='vertical', aggmethod='min', flip_frame1=None, flip_frame2=None, color=None):

 frame1, frame2 = split_frame(frame, split)
 if flip_frame1 == 'flip':
frame1 = np.flip(frame1)
 elif flip_frame1 == 'fliplr':
frame1 = np.fliplr(frame1)
 elif flip_frame1 == 'flipud':
frame1 = np.flipud(frame1)
 if flip_frame2 == 'flip':
frame2 = np.flip(frame2)
 elif flip_frame2 == 'fliplr':
frame2 = np.fliplr(frame2)
 elif flip_frame2 == 'flipud':
frame2 = np.flipud(frame2)
 if aggmethod == 'min':
result = np.min(np.dstack(( frame1, frame2)) , axis=2)
 elif aggmethod == 'max':
result = np.max(np.dstack(( frame1, frame2)) , axis=2)
 elif aggmethod == 'none_match':
result = frame1 + frame2
result = np.where(result > np.max(np.append(frame1, frame2)) , 0, result)
 elif aggmethod == 'merge_2_in_1':
result = frame1 + frame2
if np.max(result)> np.max(np.append(frame1, frame2)) :
result = frame1
elif aggmethod == 'merge_1_in_2':
result = frame1 + frame2
if np.max(result)> np.max(np.append(frame1, frame2)) :
result = frame2
elif aggmethod == 'blanks':
result = np.max(np.dstack(( frame1, frame2)) , axis=2)
 result = np.where(result == 0, 1, 0)
 if color:
result[result > 0] = color
return result
batch_size = 128
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )<train_model>
def fill_out(frame, method, mask_color=0):

 mask = np.where(frame == mask_color)
 if method == 'output_left_diagonal_symmetric':
transpose = frame.T
elif method == 'output_right_diagonal_symmetric':
transpose = np.flip(frame ).T
elif method == 'output_horizontal_symmetric':
transpose = np.flipud(frame)
 elif method == 'output_vertical_symmetric':
transpose = np.fliplr(frame)
 for i in range(len(mask[0])) :
frame[mask[0][i], mask[1][i]] = transpose[mask[0][i], mask[1][i]]
return frame
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )<save_to_csv>
if True:
for df in [train, validation, test]:
tasks = df[df['type'] == 'test'][df['fill_out_mask_color'] >= 0][(df['output_left_diagonal_symmetric'])|(df['output_right_diagonal_symmetric'])|(df['output_horizontal_symmetric'])|(df['output_vertical_symmetric'])]['task_id'].unique()
df_test = df[(df['type'] == 'test')&(df['task_id'].isin(tasks)) ].copy()
for task in tasks:
found = None
df_temp = df[(df['type'] != 'test')&(df['task_id'] == task)].copy()
df_temp['pred_output'] = df_temp['input']
methods = df_temp[['output_left_diagonal_symmetric', 'output_right_diagonal_symmetric', 'output_horizontal_symmetric', 'output_vertical_symmetric']].min()
methods = list(methods[methods].index)
 for method in methods*2:
df_temp['pred_output'] = df_temp.apply(lambda x: fill_out(x['pred_output'], method, x['fill_out_mask_color']), axis=1)
 df_temp['pred'] = df_temp.apply(lambda x: np.array_equal(x['pred_output'], x['output']), axis=1)
 if df_temp[f'pred'].min() :
found = True
break
if True:
for idx, row in df_test[df_test['task_id'] == task].iterrows() :
prediction = row['input'].copy()
for method in methods*2:
prediction = fill_out(prediction, method, row['fill_out_mask_color'])
 df_test.loc[idx, get_prediction_column(df_test, idx)] = [prediction]
print(f'
DataFrame: {DF_NAME[id(df)]}')
 score = calculate_score(df_test['output'].values, df_test['prediction_1'].values, df_test['prediction_2'].values, df_test['prediction_3'].values)
 df.loc[df_test.index, ['prediction_1', 'prediction_2', 'prediction_3']] = df_test[['prediction_1', 'prediction_2', 'prediction_3']]
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )<load_from_csv>
def get_color_mapping(frame_in, frame_out, color_mapping=None):

 if color_mapping == False:
return False
if color_mapping is None:
color_mapping = dict(zip(range(NUM_COLORS), range(NUM_COLORS)))
 for key in color_mapping.keys() :
val = color_mapping[key]
new_color = frame_out[np.where(frame_in == key)]
new_color = np.unique(new_color)
 if len(new_color)> 1:
return False
elif len(new_color)== 1:
new_color = new_color[0]
if key == val:
color_mapping[key] = new_color
elif val != new_color:
return False
return color_mapping
def change_colors(frame, color_mapping):

 result = frame.copy()
for key in color_mapping.keys() :
val = color_mapping[key]
result[np.where(frame == key)] = val
return result
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv")
 embedding_path = ".. /input/glove840b300dtxt/glove.840B.300d.txt"
embed_size = 300
max_features = 100000
max_len = 150 
<define_variables>
if True:
for df in [train, validation, test]:
tasks = df[(df['type'] == 'test')&(df['color_change'])]['task_id'].unique()
df_test = df[(df['type'] == 'test')&(df['task_id'].isin(tasks)) ].copy()
for task in tasks:
color_mapping = dict(zip(range(NUM_COLORS), range(NUM_COLORS)))
 df_temp = df[(df['type'] != 'test')&(df['task_id'] == task)].copy()
df_temp['pred'] = df_temp.apply(lambda x: get_color_mapping(x['input'], x['output']), axis=1)
 for pred_color_mapping in df_temp['pred'].to_list() :
if pred_color_mapping == False:
color_mapping = False
break
for key in color_mapping.keys() :
val = color_mapping[key]
new_color = pred_color_mapping[key]
if key == val and key != new_color:
color_mapping[key] = new_color
elif val != new_color and key != new_color:
color_mapping = False
break
if color_mapping == False:
break
if color_mapping == False:
continue
df_temp['pred'] = df_temp['input'].apply(lambda x: change_colors(x, color_mapping))
df_temp['pred'] = df_temp.apply(lambda x: np.array_equal(x['pred'], x['output']), axis=1)
 if df_temp[f'pred'].min() :
for idx, row in df_test[df_test['task_id'] == task].iterrows() :
prediction = change_colors(row['input'], color_mapping)
 df_test.loc[idx, get_prediction_column(df_test, idx)] = [prediction]
print(f'
DataFrame: {DF_NAME[id(df)]}')
 score = calculate_score(df_test['output'].values, df_test['prediction_1'].values, df_test['prediction_2'].values, df_test['prediction_3'].values)
 df.loc[df_test.index, ['prediction_1', 'prediction_2', 'prediction_3']] = df_test[['prediction_1', 'prediction_2', 'prediction_3']]
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True )<define_variables>
def object_detection(objects, method):

 objects = [v['obj'] for v in objects]
if method == 'most_common_obj':
str_objects = [str(obj.reshape(-1,)) for obj in objects]
uobj, ucount = np.unique(str_objects, return_counts=True)
 str_obj = uobj[np.argmax(ucount)]
result = objects[str_objects.index(str_obj)]
elif method == 'rarest_obj':
str_objects = [str(obj.reshape(-1,)) for obj in objects]
uobj, ucount = np.unique(str_objects, return_counts=True)
 str_obj = uobj[np.argmin(ucount)]
result = objects[str_objects.index(str_obj)]
elif method == 'rarest_color':
colors = [x for o in objects for x in np.unique(o)if x != 0]
ucol, ucount = np.unique(colors, return_counts=True)
 if len(colors)== len(objects)and len(np.where(ucount == 1)) == 1 and len(np.where(ucount == 1)[0])!= 0:
result = objects[colors.index(ucol[np.where(ucount == 1)][0])]
else:
result = objects[0]
elif method == 'smallest':
sizes = [x.shape[0]*x.shape[1] for x in objects]
result = objects[np.argmin(sizes)]
elif method == 'biggest':
sizes = [x.shape[0]*x.shape[1] for x in objects]
result = objects[np.argmax(sizes)]
return result
repl = {
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " frown ",
":(": " frown ",
":s": " frown ",
":-s": " frown ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
keys = [i for i in repl.keys() ]
new_train_data = []
new_test_data = []
ltr = train["comment_text"].tolist()
lte = test["comment_text"].tolist()
for i in ltr:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_train_data.append(xx)
 for i in lte:
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_test_data.append(xx)
 train["new_comment_text"] = new_train_data
test["new_comment_text"] = new_test_data
trate = train["new_comment_text"].tolist()
tete = test["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 for i, c in enumerate(tete):
tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])
 train["comment_text"] = trate
test["comment_text"] = tete
del trate, tete
train.drop(["new_comment_text"], axis=1, inplace=True)
 test.drop(["new_comment_text"], axis=1, inplace=True)
 train_text = train['comment_text']
test_text = test['comment_text']<concatenate>
if True:
for df in [train, validation, test]:
tasks = df[(df['type'] == 'test')&(df['output_in_input'])]['task_id'].unique()
df_test = df[(df['type'] == 'test')&(df['task_id'].isin(tasks)) ].copy()
for task in tasks:
df_temp = df[(df['type'] != 'test')&(df['task_id'] == task)].copy()
found = None
for method in ['most_common_obj', 'rarest_obj', 'rarest_color', 'smallest', 'biggest']:
df_temp['pred'] = df_temp.apply(lambda x: object_detection(x['input_objects'], method), axis=1)
 df_temp['pred'] = df_temp.apply(lambda x: np.array_equal(x['pred'], x['output']), axis=1)
 if df_temp[f'pred'].min() :
found = True
break
if found:
for idx, row in df_test[df_test['task_id'] == task].iterrows() :
prediction = object_detection(row['input_objects'], method)
 df_test.loc[idx, get_prediction_column(df_test, idx)] = [prediction]
print(f'
DataFrame: {DF_NAME[id(df)]}')
 score = calculate_score(df_test['output'].values, df_test['prediction_1'].values, df_test['prediction_2'].values, df_test['prediction_3'].values)
 df.loc[df_test.index, ['prediction_1', 'prediction_2', 'prediction_3']] = df_test[['prediction_1', 'prediction_2', 'prediction_3']]
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<string_transform>
for df in [train, validation, test]:
print(f'DataFrame: {DF_NAME[id(df)]}')
 df['match'] = df.apply(lambda x: np.array_equal(x['output'], x['prediction_1'])| np.array_equal(x['output'], x['prediction_2'])| np.array_equal(x['output'], x['prediction_3']), axis=1)
 df = df[df['type'] == 'test']
score = calculate_score(df['output'].values, df['prediction_1'].values, df['prediction_2'].values, df['prediction_3'].values )
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()<define_variables>
df = test[test['type'] == 'test'].copy()
df['submission'] = df.apply(lambda x: prediction_to_submission(x['prediction_1'], x['prediction_2'], x['prediction_3']), axis=1)
 df = df.set_index('submission_id')['submission']
id_check = len(set(submission['output_id'])& set(df.index))
print(f'{df.shape[0]} Test ids
{submission.shape[0]} Submssion ids
{id_check} Intersection')
 submission['output'] = submission['output_id'].apply(lambda x: df[x] if x in df.index else DEFAULT_SUBMISSION)
 submission.to_csv('submission.csv', index=False)
 submission[submission['output'] != DEFAULT_SUBMISSION]