Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<import_modules>
tqdm.pandas(desc='Progress')
from sklearn.model_selection import train_test_split<import_modules>
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 1029
from sklearn.model_selection import train_test_split<split>
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
train_x, test_x, train_y, test_y = train_test_split(trainX, trainY, test_size=0.2, random_state=42 )<train_model>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def step(train_x, true_y):
loss = []
with tf.GradientTape() as tape:
pred_y = model(train_x)
 pred_y = tf.cast(pred_y,tf.float32)
 model_loss = custom_loss(true_y, pred_y)
 loss.append(model_loss.numpy())
 model_gradients = tape.gradient(model_loss, model.trainable_variables)
 opt.apply_gradients(zip(model_gradients, model.trainable_variables))
return np.mean(loss )<define_variables>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )
epochs = 20
batch_size = 32
bat_per_epoch = math.floor(len(train_x)/ batch_size)
 epoch_loss = []
for epoch in tqdm(range(epochs)) :
step_loss = []
for i in tqdm(range(bat_per_epoch)) :
n = i*batch_size
step_loss.append(step(train_x[n:n+batch_size], train_y[n:n+batch_size]))
epoch_loss.append(np.mean(step_loss))
print(f'Epoch: {epoch}, Loss: {epoch_loss[epoch]}')
<compute_test_metric>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )
model.compile(optimizer=opt, loss=custom_loss, metrics=['accuracy', tf.keras.metrics.MeanIoU(num_classes=2)])
 print(model.evaluate(test_x, test_y, verbose=0))<predict_on_test>
sin = len(df_train[df_train["target"]==0])
 insin = len(df_train[df_train["target"]==1])
 persin =(sin/(sin+insin)) *100
perinsin =(insin/(sin+insin)) *100
print("
print("
preds = model.predict(test_x)
 preds.shape<categorify>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )
preds_thresh = tf.where(preds>0.5,1,0)
 pred_fwd = tf.map_fn(life_step_1_tensor, preds_thresh, fn_output_signature=tf.uint32)
 actual_fwd = tf.map_fn(life_step_1_tensor, test_y, fn_output_signature=tf.uint32 )<define_variables>
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
train = add_features(train_df)
 test = add_features(test_df)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index
start_features = [f for f in train_df.columns if "start" in f]
stop_features = [f for f in train_df.columns if "stop" in f]<prepare_x_and_y>
x_train, x_test, y_train, features, test_features, word_index = load_and_prec()
sub = pd.DataFrame()
m = tf.keras.metrics.Accuracy()
accuracies = []
t1 = tqdm(range(1,6), desc=f'Delta: ')
 for delta in t1:
m.reset_states()
t1.set_description(f'Delta: {delta}')
 t1.refresh()
test_data_iter = train_df.loc[train_df['delta'] == delta]
tmp_sub = test_data_iter[["id"]].copy()
testY = test_data_iter.loc[:, test_data_iter.columns.str.startswith('start')].values.reshape(-1, 25, 25)
 testX = test_data_iter.loc[:, test_data_iter.columns.str.startswith('stop')].values.reshape(-1, 25, 25)
 testX = np.array([preprocess(xi)for xi in testX] ).astype(np.float32)
 testX = np.expand_dims(testX, axis=-1 ).astype(np.float32)
 testY = np.array([preprocess(xi)for xi in testY] ).astype(np.float32)
 testY = np.expand_dims(testY, axis=-1 ).astype(np.float32)
 t2 = tqdm(range(delta))
for i in t2:
if i == 0:
preds = model.predict(testX)
 else:
preds = tf.where(preds>0.5,1,0)
 preds = model.predict(preds)
 preds = tf.cast(tf.where(preds>0.5,1,0), tf.uint32)
 m.update_state(preds, testY)
 acc = m.result().numpy()
print(f'Accuracy: {acc}')
 accuracies.append(acc)
 preds = preds[:, 4:-4, 4:-4,:].numpy()
tmp = pd.DataFrame(preds.reshape(-1, 625 ).astype(np.uint8), columns=start_features, index=tmp_sub['id'])
 tmp_sub = tmp_sub.join(tmp)
 sub = sub.append(tmp_sub)
 sub.sort_index(inplace = True )<train_model>
np.save("x_train",x_train)
 np.save("x_test",x_test)
 np.save("y_train",y_train)
 np.save("features",features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index )
print(f'Mean accuracy: {np.array(accuracies ).mean() }')
 print(f'LB score estimate from training: {1 - np.array(accuracies ).mean() }' )<create_dataframe>
x_train = np.load("x_train.npy")
 x_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 features = np.load("features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()
sub = pd.DataFrame()
m = tf.keras.metrics.Accuracy()
accuracies = []
t1 = tqdm(range(1,6), desc=f'Delta: ')
 for delta in t1:
m.reset_states()
t1.set_description(f'Delta: {delta}')
 t1.refresh()
test_data_iter = test_df.loc[test_df['delta'] == delta]
tmp_sub = test_data_iter[["id"]].copy()
tmp_sub.set_index(tmp_sub['id'].values)
 testX = test_data_iter.loc[:, test_data_iter.columns.str.startswith('stop')].values.reshape(-1, 25, 25)
 testX = np.array([preprocess(xi)for xi in testX] ).astype(np.float32)
 testX = np.expand_dims(testX, axis=-1 ).astype(np.float32)
 t2 = tqdm(range(delta))
for i in t2:
if i == 0:
preds = model.predict(testX)
 else:
preds = tf.where(preds>0.5,1,0)
 preds = model.predict(preds)
 preds = tf.cast(tf.where(preds>0.5,1,0), tf.uint32)
 preds = preds[:, 4:-4, 4:-4,:].numpy()
tmp = pd.DataFrame(preds.reshape(-1, 625 ).astype(np.uint8), columns=start_features, index=tmp_sub['id'].values)
 tmp.insert(loc = 0, column='id', value=tmp_sub['id'].values)
 sub = sub.append(tmp)
 sub.sort_index(inplace = True)
 sub.reset_index(drop = True, inplace = True )<save_to_csv>
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings, fasttext_embeddings
gc.collect()
np.shape(embedding_matrix )
sub.to_csv('submission.csv', index=False )<save_to_csv>
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
splits[:3]
sub.to_csv('submission.csv', index=False )<set_options>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
notebook_start = time.perf_counter()
display(HTML())
 %load_ext autoreload
%autoreload 2<install_modules>
embedding_dim = 300
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=300,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale
class Capsule_Main(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None):
super(Capsule_Main, self ).__init__()
self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)
 self.gru_layer = GRU_Layer()
self.gru_layer.init_weights()
self.caps_layer = Caps_Layer()
self.dense_layer = Dense_Layer()
def forward(self, content):
content1 = self.embed_layer(content)
 content2, _ = self.gru_layer(
 content1)
 content3 = self.caps_layer(content2)
 output = self.dense_layer(content3)
 return output
! python3 -m pip install -q z3-solver
! apt-get install -qq tree moreutils<install_modules>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
fc_layer = 16
fc_layer1 = 16
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.bn = nn.BatchNorm1d(16, momentum=0.5)
 self.linear = nn.Linear(hidden_size*8+3, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 content3 = self.caps_layer(h_gru)
 content3 = self.dropout(content3)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.relu(self.lincaps(content3))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out
!rm -rf /ai-games/
!git clone https://github.com/JamesMcGuigan/ai-games/ /ai-games/
!cp -rf /ai-games/puzzles/game_of_life/*./
!rm -rf /kaggle/working/neural_networks/
!cd /ai-games/; git log -n1<import_modules>
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset )
from utils.util import *
from utils.plot import *
from utils.game import *
from utils.datasets import *
from utils.tuplize import *
from hashmaps.crop import *
from hashmaps.hash_functions import *
from hashmaps.translation_solver import *
from hashmaps.repeating_patterns import *
from constraint_satisfaction.fix_submission import *<prepare_x_and_y>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []
@njit()
def get_concentric_prime_mask(shape: Tuple[int,int]=(25,25)) -> np.ndarray:
pattern = 'diamond'
assert shape[0] == shape[1]
assert pattern in [ 'diamond', 'oval' ]
x =(shape[0])//2
y =(shape[1])//2
max_r = max(shape)+ 1 if max(shape)% 2 == 0 else max(shape)
 mask = np.zeros(shape, dtype=np.int64)
 for r in range(max_r):
primes = hashable_primes[:r+1]
for dr in range(r+1):
if pattern == 'diamond': prime = primes[r]
elif pattern == 'oval': prime = primes[r] + primes[dr]
coords = { 
(x+(r-dr),y+(dr)) , 
(x-(r-dr),y+(dr)) , 
(x+(r-dr),y-(dr)) , 
(x-(r-dr),y-(dr)) ,
}
for coord in coords:
if min(coord)>= 0 and max(coord)< min(shape):
mask[coord] = prime
return mask
@njit()
def hash_geometric_concentric(board: np.ndarray)-> int:

 assert board.shape[0] == board.shape[1]
mask = get_concentric_prime_mask(shape=board.shape)
 hashed = 0
for x in range(board.shape[0]):
for y in range(board.shape[1]):
for dx in range(mask.shape[0]):
for dy in range(mask.shape[1]):
coords =(( x+dx)%board.shape[0],(y+dy)%board.shape[1])
 hashed += board[coords] * mask[dx,dy]
return hashed
hash_geometric = hash_geometric_concentric<categorify>
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
def label_board(board):

 tessellation = tessellate_board(board)
 tessellation = scipy.ndimage.convolve(tessellation, [[0,1,0],[1,1,1],[0,1,0]] ).astype(np.bool ).astype(np.int8)
 labeled = skimage.measure.label(tessellation, background=0, connectivity=2)
 labeled = detessellate_board(labeled)
 return labeled
def extract_clusters(board: np.ndarray)-> List[np.ndarray]:
labeled = label_board(board)
 return extract_clusters_from_labels(board, labeled)
 def extract_clusters_from_labels(board: np.ndarray, labeled: np.ndarray)-> List[np.ndarray]:
labels = np.unique(labeled)
 clusters = []
for label in labels:
cluster = board *(labeled == label)
 clusters.append(cluster)
 return clusters<define_variables>
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )
<load_pretrained><EOS> 
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )