Kaggle competition: Gendered Pronoun Resolution
df_train = pd.read_csv('.. /input/juneextra/juneExtra.csv', usecols=train_cols )<load_from_csv>
DATA_ROOT = '.. /input/'
GAP_DATA_FOLDER = os.path.join(DATA_ROOT, 'gap-coreference')
 SUB_DATA_FOLDER = os.path.join(DATA_ROOT, 'gendered-pronoun-resolution')
 FAST_TEXT_DATA_FOLDER = os.path.join(DATA_ROOT, 'fasttext-crawl-300d-2m' )
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )<filter>
sub_test_df_path = os.path.join(SUB_DATA_FOLDER, 'test_stage_2.tsv')
 test_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-test.tsv')
 train_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-development.tsv')
 dev_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-validation.tsv')
 train_df = pd.read_csv(train_df_path, sep='\t')
 test_df = pd.read_csv(test_df_path, sep='\t')
 dev_df = pd.read_csv(dev_df_path, sep='\t')
 sub_test_df = pd.read_csv(sub_test_df_path, sep='\t')
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']<drop_column>
spacy_model = "en_core_web_lg"
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )<data_type_conversions>
from spacy.lang.en import English
from spacy.pipeline import DependencyParser
import spacy
from nltk import Tree
from keras.preprocessing import sequence
from keras.preprocessing import text as ktext
df_train["renta"] = pd.to_numeric(df_train["renta"], errors="coerce")
 df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce" )<feature_engineering>
EMBEDDING_FILE = os.path.join(FAST_TEXT_DATA_FOLDER, 'crawl-300d-2M.vec')
 def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
for cod in unique_prov:
df_test.loc[df_test['cod_prov']==cod,['renta']] = df_test.loc[df_test['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df_train.loc[df_train['cod_prov']==cod,['renta']] = df_train.loc[df_train['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values<correct_missing_values>
nlp = spacy.load(spacy_model)
 def bs(list_, target_):
lo, hi = 0, len(list_)-1
while lo < hi:
mid = lo + int(( hi - lo)/ 2)
 if target_ < list_[mid]:
hi = mid
elif target_ > list_[mid]:
lo = mid + 1
else:
return mid + 1
return lo
def bs_(list_, target_):
lo, hi = 0, len(list_)-1
while lo < hi:
mid = lo + int(( hi - lo)/ 2)
 if target_ < list_[mid]:
hi = mid
elif target_ > list_[mid]:
lo = mid + 1
else:
return mid
return lo
def ohe_dist(dist, buckets):
idx = bs_(buckets, dist)
 oh = np.zeros(shape=(len(buckets),), dtype=np.float32)
 oh[idx] = 1
return oh
df_test.renta.fillna(df_test["renta"].median() , inplace=True )<categorify>
num_pos_features = 45
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )<define_variables>
def extrac_positional_features(text, char_offset1, char_offset2):
doc = nlp(text)
 max_len = 64
lens = [token.idx for token in doc]
mention_offset1 = bs(lens, char_offset1)- 1
mention_offset2 = bs(lens, char_offset2)- 1
lens = [len(sent)for sent in doc.sents]
acc_lens = [len_ for len_ in lens]
pre_len = 0
for i in range(0, len(acc_lens)) :
pre_len += acc_lens[i]
acc_lens[i] = pre_len
sent_index1 = bs(acc_lens, mention_offset1)
 sent_index2 = bs(acc_lens, mention_offset2)
 sent1 = list(doc.sents)[sent_index1]
sent2 = list(doc.sents)[sent_index2]
bucket_dist = [1, 2, 3, 4, 5, 8, 16, 32, 64]
dist = mention_offset2 - mention_offset1
dist_oh = ohe_dist(dist, bucket_dist)
 bucket_pos = [0, 1, 2, 3, 4, 5, 8, 16, 32]
sent_pos1 = mention_offset1 + 1
if sent_index1 > 0:
sent_pos1 = mention_offset1 - acc_lens[sent_index1-1]
sent_pos_oh1 = ohe_dist(sent_pos1, bucket_pos)
 sent_pos_inv1 = len(sent1)- sent_pos1
assert sent_pos_inv1 >= 0
sent_pos_inv_oh1 = ohe_dist(sent_pos_inv1, bucket_pos)
 sent_pos2 = mention_offset2 + 1
if sent_index2 > 0:
sent_pos2 = mention_offset2 - acc_lens[sent_index2-1]
sent_pos_oh2 = ohe_dist(sent_pos2, bucket_pos)
 sent_pos_inv2 = len(sent2)- sent_pos2
if sent_pos_inv2 < 0:
print(sent_pos_inv2)
 print(len(sent2))
print(sent_pos2)
 raise ValueError
sent_pos_inv_oh2 = ohe_dist(sent_pos_inv2, bucket_pos)
 sent_pos_ratio1 = sent_pos1 / len(sent1)
 sent_pos_ratio2 = sent_pos2 / len(sent2)
 return dist_oh, sent_pos_oh1, sent_pos_oh2, sent_pos_inv_oh1, sent_pos_inv_oh2
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']<count_values>
def create_dist_features(df, text_column, pronoun_offset_column, name_offset_column):
text_offset_list = df[[text_column, pronoun_offset_column, name_offset_column]].values.tolist()
num_features = num_pos_features
pos_feature_matrix = np.zeros(shape=(len(text_offset_list), num_features))
for text_offset_index in range(len(text_offset_list)) :
text_offset = text_offset_list[text_offset_index]
dist_oh, sent_pos_oh1, sent_pos_oh2, sent_pos_inv_oh1, sent_pos_inv_oh2 = extrac_positional_features(text_offset[0], text_offset[1],text_offset[2])
 feature_index = 0
pos_feature_matrix[text_offset_index, feature_index:feature_index+len(dist_oh)] = np.asarray(dist_oh)
 feature_index += len(dist_oh)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_oh1)] = np.asarray(sent_pos_oh1)
 feature_index += len(sent_pos_oh1)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_oh2)] = np.asarray(sent_pos_oh2)
 feature_index += len(sent_pos_oh2)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_inv_oh1)] = np.asarray(sent_pos_inv_oh1)
 feature_index += len(sent_pos_inv_oh1)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_inv_oh2)] = np.asarray(sent_pos_inv_oh2)
 feature_index += len(sent_pos_inv_oh2)
 return pos_feature_matrix
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
df_train.loc[row_index,col] = np.nan
for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan<categorify>
max_len = 50
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )<categorify>
seq_list = list()
def extract_sents(text, char_offset_p, char_offset_a, char_offset_b, id):
global max_len
global seq_list
seq_list.append(list())
 doc = nlp(text)
 token_lens = [token.idx for token in doc]
char_offsets = [char_offset_p, char_offset_a, char_offset_b]
sent_list = list()
for char_offset in char_offsets:
mention_offset = bs(token_lens, char_offset)- 1
mention = doc[mention_offset]
lens = [len(sent)for sent in doc.sents]
acc_lens = [len_ for len_ in lens]
pre_len = 0
for i in range(0, len(acc_lens)) :
pre_len += acc_lens[i]
acc_lens[i] = pre_len
sent_index = bs(acc_lens, mention_offset)
 sent = list(doc.sents)[sent_index]
sent_pos = mention_offset + 1
if sent_index > 0:
sent_pos = mention_offset - acc_lens[sent_index-1]
if len(sent)> max_len:
if sent_pos < max_len-1:
sent_list.append(sent[0:max_len].text)
 sent_list.append(sent_pos)
 seq_list[-1].append(sent[0:max_len])
 else:
sent_list.append(sent[sent_pos-max_len+2 : min(sent_pos+2, len(sent)) ].text)
 sent_list.append(max_len-2)
 seq_list[-1].append(sent[sent_pos-max_len+2 : min(sent_pos+2, len(sent)) ])
 else:
sent_list.append(sent.text)
 sent_list.append(sent_pos)
 seq_list[-1].append(sent)
 return pd.Series([id] + sent_list, index=['ID', 'Pronoun-Sent', 'Pronoun-Sent-Offset', 'A-Sent', 'A-Sent-Offset', 'B-Sent', 'B-Sent-Offset'])
 def add_sent_columns(df, text_column, pronoun_offset_column, a_offset_column, b_offset_column):
global seq_list
seq_list = list()
sent_df = df.apply(lambda row: extract_sents(row.loc[text_column], row[pronoun_offset_column], row[a_offset_column], row[b_offset_column], row['ID']), axis=1)
 df = df.join(sent_df.set_index('ID'), on='ID')
 return df, seq_list
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )<train_model>
seq_list = list()
train_df, train_tokenized = add_sent_columns(train_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
test_df, test_tokenized = add_sent_columns(test_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
dev_df, dev_tokenized = add_sent_columns(dev_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
sub_test_df, sub_test_tokenized = add_sent_columns(sub_test_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 train_tokenized = train_tokenized[1:]
test_tokenized = test_tokenized[1:]
dev_tokenized = dev_tokenized[1:]
sub_test_tokenized = sub_test_tokenized[1:]
models = {}
model_preds = {}
id_preds = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = LogisticRegression()
clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds[id].append(p)
<load_from_csv>
embed_size = 300
max_features = 80000
word_index = dict()
idx = 1
for text_ in train_tokenized+test_tokenized+dev_tokenized+sub_test_tokenized:
for sent_ in text_:
for word_ in sent_:
if word_.text not in word_index and word_.text in embeddings_index:
word_index[word_.text] = idx
idx += 1
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = None
if word in embeddings_index:
embedding_vector = embeddings_index[word]
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
print(embedding_matrix.shape)
 pos_index = dict()
idx = 1
for text_ in train_tokenized+test_tokenized+dev_tokenized+sub_test_tokenized:
for sent_ in text_:
for word_ in sent_:
if word_.pos not in pos_index:
pos_index[word_.pos] = idx
idx += 1
def sentences_to_sequences(tokenized_):
return list(map(
 lambda sent_tokenized: list(map(
 lambda token_: word_index[token_.text] if token_.text in word_index else 0,
sent_tokenized 
)) ,
tokenized_ 
))
def poses_to_sequences(tokenized_):
return list(map(
 lambda sent_tokenized: list(map(
 lambda token_: pos_index[token_.pos] if token_.pos in pos_index else 0,
sent_tokenized 
)) ,
tokenized_ 
))
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
 df_recent = df_recent.drop_duplicates(['ncodpers'], keep='last' )<load_from_csv>
train_p_tokenized = sentences_to_sequences([row[0] for row in train_tokenized])
 train_a_tokenized = sentences_to_sequences([row[1] for row in train_tokenized])
 train_b_tokenized = sentences_to_sequences([row[2] for row in train_tokenized])
 test_p_tokenized = sentences_to_sequences([row[0] for row in test_tokenized])
 test_a_tokenized = sentences_to_sequences([row[1] for row in test_tokenized])
 test_b_tokenized = sentences_to_sequences([row[2] for row in test_tokenized])
 sub_test_p_tokenized = sentences_to_sequences([row[0] for row in sub_test_tokenized])
 sub_test_a_tokenized = sentences_to_sequences([row[1] for row in sub_test_tokenized])
 sub_test_b_tokenized = sentences_to_sequences([row[2] for row in sub_test_tokenized])
 dev_p_tokenized = sentences_to_sequences([row[0] for row in dev_tokenized])
 dev_a_tokenized = sentences_to_sequences([row[1] for row in dev_tokenized])
 dev_b_tokenized = sentences_to_sequences([row[2] for row in dev_tokenized])
 seq_p_train = sequence.pad_sequences(train_p_tokenized, maxlen = max_len, padding='post')
 seq_a_train = sequence.pad_sequences(train_a_tokenized, maxlen = max_len, padding='post')
 seq_b_train = sequence.pad_sequences(train_b_tokenized, maxlen = max_len, padding='post')
 seq_p_test = sequence.pad_sequences(test_p_tokenized, maxlen = max_len, padding='post')
 seq_a_test = sequence.pad_sequences(test_a_tokenized, maxlen = max_len, padding='post')
 seq_b_test = sequence.pad_sequences(test_b_tokenized, maxlen = max_len, padding='post')
 seq_p_sub_test = sequence.pad_sequences(sub_test_p_tokenized, maxlen = max_len, padding='post')
 seq_a_sub_test = sequence.pad_sequences(sub_test_a_tokenized, maxlen = max_len, padding='post')
 seq_b_sub_test = sequence.pad_sequences(sub_test_b_tokenized, maxlen = max_len, padding='post')
 seq_p_dev = sequence.pad_sequences(dev_p_tokenized, maxlen = max_len, padding='post')
 seq_a_dev = sequence.pad_sequences(dev_a_tokenized, maxlen = max_len, padding='post')
 seq_b_dev = sequence.pad_sequences(dev_b_tokenized, maxlen = max_len, padding='post')
 train_p_pos = poses_to_sequences([row[0] for row in train_tokenized])
 train_a_pos = poses_to_sequences([row[1] for row in train_tokenized])
 train_b_pos = poses_to_sequences([row[2] for row in train_tokenized])
 test_p_pos = poses_to_sequences([row[0] for row in test_tokenized])
 test_a_pos = poses_to_sequences([row[1] for row in test_tokenized])
 test_b_pos = poses_to_sequences([row[2] for row in test_tokenized])
 sub_test_p_pos = poses_to_sequences([row[0] for row in sub_test_tokenized])
 sub_test_a_pos = poses_to_sequences([row[1] for row in sub_test_tokenized])
 sub_test_b_pos = poses_to_sequences([row[2] for row in sub_test_tokenized])
 dev_p_pos = poses_to_sequences([row[0] for row in dev_tokenized])
 dev_a_pos = poses_to_sequences([row[1] for row in dev_tokenized])
 dev_b_pos = poses_to_sequences([row[2] for row in dev_tokenized])
 pos_p_train = sequence.pad_sequences(train_p_pos, maxlen = max_len, padding='post')
 pos_a_train = sequence.pad_sequences(train_a_pos, maxlen = max_len, padding='post')
 pos_b_train = sequence.pad_sequences(train_b_pos, maxlen = max_len, padding='post')
 pos_p_test = sequence.pad_sequences(test_p_pos, maxlen = max_len, padding='post')
 pos_a_test = sequence.pad_sequences(test_a_pos, maxlen = max_len, padding='post')
 pos_b_test = sequence.pad_sequences(test_b_pos, maxlen = max_len, padding='post')
 pos_p_sub_test = sequence.pad_sequences(sub_test_p_pos, maxlen = max_len, padding='post')
 pos_a_sub_test = sequence.pad_sequences(sub_test_a_pos, maxlen = max_len, padding='post')
 pos_b_sub_test = sequence.pad_sequences(sub_test_b_pos, maxlen = max_len, padding='post')
 pos_p_dev = sequence.pad_sequences(dev_p_pos, maxlen = max_len, padding='post')
 pos_a_dev = sequence.pad_sequences(dev_a_pos, maxlen = max_len, padding='post')
 pos_b_dev = sequence.pad_sequences(dev_b_pos, maxlen = max_len, padding='post')
 index_p_train = train_df['Pronoun-Sent-Offset'].values
index_a_train = train_df['A-Sent-Offset'].values
index_b_train = train_df['B-Sent-Offset'].values
index_p_test = test_df['Pronoun-Sent-Offset'].values
index_a_test = test_df['A-Sent-Offset'].values
index_b_test = test_df['B-Sent-Offset'].values
index_p_sub_test = sub_test_df['Pronoun-Sent-Offset'].values
index_a_sub_test = sub_test_df['A-Sent-Offset'].values
index_b_sub_test = sub_test_df['B-Sent-Offset'].values
index_p_dev = dev_df['Pronoun-Sent-Offset'].values
index_a_dev = dev_df['A-Sent-Offset'].values
index_b_dev = dev_df['B-Sent-Offset'].values
pa_pos_tra = create_dist_features(train_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_dev = create_dist_features(dev_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_test = create_dist_features(test_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_sub_test = create_dist_features(sub_test_df, 'Text', 'Pronoun-offset', 'A-offset')
 pb_pos_tra = create_dist_features(train_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_dev = create_dist_features(dev_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_test = create_dist_features(test_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_sub_test = create_dist_features(sub_test_df, 'Text', 'Pronoun-offset', 'B-offset' )
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
<load_from_csv>
X_train = [seq_p_train, seq_a_train, seq_b_train, pos_p_train, pos_a_train, pos_b_train, index_p_train, index_a_train, index_b_train, pa_pos_tra, pb_pos_tra]
X_dev = [seq_p_dev, seq_a_dev, seq_b_dev, pos_p_dev, pos_a_dev, pos_b_dev, index_p_dev, index_a_dev, index_b_dev, pa_pos_dev, pb_pos_dev]
X_test = [seq_p_test, seq_a_test, seq_b_test, pos_p_test, pos_a_test, pos_b_test, index_p_test, index_a_test, index_b_test, pa_pos_test, pb_pos_test]
X_sub_test = [seq_p_sub_test, seq_a_sub_test, seq_b_sub_test, pos_p_sub_test, pos_a_sub_test, pos_b_sub_test, index_p_sub_test, index_a_sub_test, index_b_sub_test, pa_pos_sub_test, pb_pos_sub_test]
sample = pd.read_csv('.. /input/santander-product-recommendation/sample_submission.csv' )<define_variables>
def _row_to_y(row):
if row.loc['A-coref']:
return 0
if row.loc['B-coref']:
return 1
return 2
y_tra = train_df.apply(_row_to_y, axis=1)
 y_dev = dev_df.apply(_row_to_y, axis=1)
 y_test = test_df.apply(_row_to_y, axis=1 )
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p)) 
<prepare_output>
import numpy as np
from keras import backend
from keras import layers
from keras import models
from keras import initializers, regularizers, constraints, activations
from keras.engine import Layer
import keras.backend as K
from keras.layers import merge
import tensorflow as tf
sample['added_products'] = test_preds 
<save_to_csv>
def _dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class AttentionWeight(Layer):

 def __init__(self,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(AttentionWeight, self ).__init__(**kwargs)
 def build(self, input_shape):
shape1 = input_shape[0]
shape2 = input_shape[1]
self.W = self.add_weight(( shape2[-1], shape1[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( shape2[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, inputs, mask=None):
x = inputs[0]
u = inputs[1]
uit = _dot_product(x, self.W)
 if self.bias:
uit += self.b
uit = K.tanh(uit)
 ait = K.batch_dot(uit, u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 return a
def compute_output_shape(self, input_shape):
if not isinstance(input_shape, list)or len(input_shape)!= 2:
raise ValueError('A `Dot` layer should be called '
'on a list of 2 inputs.')
 shape1 = list(input_shape[0])
 shape2 = list(input_shape[1])
 return shape1[0], shape1[1]
def get_config(self):
config = {
'W_regularizer': regularizers.serialize(self.W_regularizer),
'b_regularizer': regularizers.serialize(self.b_regularizer),
'W_constraint': constraints.serialize(self.W_constraint),
'b_constraint': constraints.serialize(self.b_constraint),
'bias': self.bias
}
base_config = super(AttentionWeight, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
class FeatureSelection1D(Layer):

 def __init__(self, num_selects, **kwargs):
self.num_selects = num_selects
self.supports_masking = True
super(FeatureSelection1D, self ).__init__(**kwargs)
 def build(self, input_shape):
super(FeatureSelection1D, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, inputs, mask=None):
if not isinstance(inputs, list)or len(inputs)!= 2:
raise ValueError('FeatureSelection1D layer should be called '
'on a list of 2 inputs.')
 if mask is not None:
a = K.cast(mask, K.floatx())* inputs[0]
else:
a = inputs[0]
b = inputs[1]
a = tf.batch_gather(a, b)
 return a
def compute_output_shape(self, input_shape):
if not isinstance(input_shape, list)or len(input_shape)!= 2:
raise ValueError('A `FeatureSelection1D` layer should be called '
'on a list of 2 inputs.')
 shape1 = list(input_shape[0])
 shape2 = list(input_shape[1])
 if shape2[0] != shape1[0]:
raise ValueError("batch size must be same")
 if shape2[1] != self.num_selects:
raise ValueError("must conform to the num_select")
 return(shape1[0], self.num_selects, shape1[2])
 def get_config(self):
config = {
'num_selects': self.num_selects
}
base_config = super(FeatureSelection1D, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
sample.to_csv('LR_sub.csv', index=False)
 print(datetime.datetime.now() -start_time)
<import_modules>
histories = list()
file_paths = list()
cos = list()
import numpy as np
import pandas as pd
import os<set_options>
def build_e2e_birnn_attention_model(
 voca_dim, time_steps, pos_tag_size, pos_tag_dim, extra_feature_dims, output_dim, rnn_dim, model_dim, mlp_dim,
item_embedding=None, rnn_depth=1, mlp_depth=1,
drop_out=0.5, rnn_drop_out=0., rnn_state_drop_out=0.,
trainable_embedding=False, gpu=False, return_customized_layers=False):

 if item_embedding is not None:
inputp = models.Input(shape=(time_steps,), dtype='int32', name='inputp')
 inputa = models.Input(shape=(time_steps,), dtype='int32', name='inputa')
 inputb = models.Input(shape=(time_steps,), dtype='int32', name='inputb')
 inputs = [inputp, inputa, inputb]
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
embed_dim = item_embedding.shape[1]
emb_layer = layers.Embedding(
 voca_dim, item_embedding.shape[1], input_length=time_steps,
weights=[item_embedding, ], trainable=trainable_embedding,
mask_zero=False, name='embedding_layer0' 
)
 elif utils.is_integer(item_embedding):
embed_dim = item_embedding
emb_layer = layers.Embedding(
 voca_dim, item_embedding, input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False, name='embedding_layer0' 
)
 else:
raise ValueError("item_embedding must be either integer or numpy matrix")
 xs = list(map(
 lambda input_: emb_layer(input_),
inputs 
))
else:
inputp = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputp')
 inputa = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputa')
 inputb = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputb')
 embed_dim = voca_dim
xs = [inputp, inputa, inputb]
inputposp = models.Input(shape=(time_steps,), dtype='int32', name='inputposp')
 inputposa = models.Input(shape=(time_steps,), dtype='int32', name='inputposa')
 inputposb = models.Input(shape=(time_steps,), dtype='int32', name='inputposb')
 inputpos = [inputposp, inputposa, inputposb]
pos_emb_layer = layers.Embedding(
 pos_tag_size, pos_tag_dim, input_length=time_steps,
trainable=True, mask_zero=False, name='pos_embedding_layer0' 
)
 xpos = list(map(
 lambda input_: pos_emb_layer(input_),
inputpos 
))
embed_concate_layer = layers.Concatenate(axis=2, name="embed_concate_layer")
 for i in range(len(xs)) :
xs[i] = embed_concate_layer([xs[i], xpos[i]])
 inputpi = models.Input(shape=(1,), dtype='int32', name='inputpi')
 inputai = models.Input(shape=(1,), dtype='int32', name='inputai')
 inputbi = models.Input(shape=(1,), dtype='int32', name='inputbi')
 xis = [inputpi, inputai, inputbi]
birnns = list()
rnn_batchnorms = list()
rnn_dropouts = list()
if gpu:
for i in range(rnn_depth):
rnn_dropout = layers.SpatialDropout1D(rnn_drop_out)
 birnn = layers.Bidirectional(
 layers.CuDNNGRU(rnn_dim, return_sequences=True),
name='bi_lstm_layer' + str(i))
rnn_batchnorm = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))
birnns.append(birnn)
 rnn_dropouts.append(rnn_dropout)
 rnn_batchnorms.append(rnn_batchnorm)
 xs_ = list()
for x_ in xs:
for i in range(len(birnns)) :
x_ = rnn_dropouts[i](x_)
 x_ = birnns[i](x_)
 x_ = rnn_batchnorms[i](x_)
 xs_.append(x_)
 xs = xs_
else:
for i in range(rnn_depth):
birnn = layers.Bidirectional(
 layers.GRU(rnn_dim, return_sequences=True, dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))
rnn_batchnorm = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))
birnns.append(birnn)
 rnn_batchnorms.append(rnn_batchnorm)
 xs_ = list()
for x_ in xs:
for i in range(len(birnns)) :
x_ = birnns[i](x_)
 x_ = rnn_batchnorms[i](x_)
 xs_.append(x_)
 xs = xs_
select_layer = FeatureSelection1D(1, name='boundary_selection_layer')
 flatten_layer1 = layers.Flatten('channels_first', name="flatten_layer1")
 permute_layer = layers.Permute(( 2, 1), name='permuted_attention_x')
 attent_weight = AttentionWeight(name="attention_weight")
 focus_layer = layers.Dot([2, 1], name='focus' + '_layer')
 reshape_layer = layers.Reshape(( 1, rnn_dim*2), name="reshape_layer")
 concate_layer = layers.Concatenate(axis=1, name="attention_concate_layer")
 atten_dropout_layer = layers.Dropout(drop_out, name='attention_dropout_layer')
 map_layer1 = layers.Dense(model_dim, activation="relu", name="map_layer1")
 map_layer2 = map_layer1
flatten_layer = layers.Flatten('channels_first', name="flatten_layer")
 for i in range(len(xs)) :
if i == 0:
map_layer = map_layer1
else:
map_layer = map_layer2
select_ = select_layer([xs[i], xis[i]])
 flatten_select_ = flatten_layer1(select_)
 att = attent_weight([xs[i], flatten_select_])
 focus = focus_layer([permute_layer(xs[i]), att])
 xs[i] = concate_layer([select_, reshape_layer(focus)])
 xs[i] = flatten_layer(xs[i])
 xs[i] = atten_dropout_layer(xs[i])
 xs[i] = map_layer(xs[i])
 x = layers.Concatenate(axis=1, name="concat_feature_layer" )(xs)
 x = layers.Dropout(drop_out, name='dropout_layer' )(x)
 for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)
 x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
 outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0" )(x)
 model = models.Model([inputp, inputa, inputb] + inputpos + xis , outputs)
 if return_customized_layers:
return model, {'FeatureSelection1D': FeatureSelection1D, 'AttentionWeight': AttentionWeight}
return model
torch.cuda.set_device(0)
 torch.cuda.set_device(0 )<load_from_csv>
voca_dim = embedding_matrix.shape[0]
pos_tag_size = len(pos_index)
 time_steps = max_len
embed_dim = embedding_matrix.shape[1]
pos_tag_dim = 5
extra_feature_dims = num_pos_features
output_dim = 3
rnn_dim = 50
model_dim = 10
mlp_dim = 10
rnn_depth = 1
mlp_depth=1
drop_out=0.2
rnn_drop_out=0.5
gpu = True
return_customized_layers=True
model, co = build_e2e_birnn_attention_model(
 voca_dim, time_steps, pos_tag_size, pos_tag_dim, extra_feature_dims, output_dim, rnn_dim, model_dim, mlp_dim,
item_embedding=embedding_matrix, rnn_depth=rnn_depth, mlp_depth=mlp_depth,
drop_out=drop_out, rnn_drop_out=rnn_drop_out, rnn_state_drop_out=rnn_drop_out,
trainable_embedding=False, gpu=gpu, return_customized_layers=return_customized_layers)
 cos.append(co )
MASKS = 'train.csv'
SUB = 'sample_submission.csv'
TRAIN = Path('train/')
 TEST = Path('test/')
 PATH = Path('/kaggle/input/')
 TMP = Path('/kaggle/working/tmp/')
 MODEL = Path('/kaggle/working/model/')
 seg = pd.read_csv(PATH/MASKS ).set_index('Id')
 sample_sub = pd.read_csv(PATH/SUB ).set_index('Id')
 sample= 31072
seg.head()<define_variables>
img = Image.open('model.png')
 display(img )
train_names_png = [TRAIN/f for f in os.listdir(PATH/TRAIN)]
train_names = list(seg.index.values)
 train_names_sample = list(seg.index.values)[0:sample]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]
test_names = list(sample_sub.index.values)
 test_names_sample = list(sample_sub.index.values)[0:sample]
len(train_names_sample), len(test_names )<data_type_conversions>
adam = ko.Nadam(clipnorm=1.0)
 model.compile(adam, loss="sparse_categorical_crossentropy", metrics=["sparse_categorical_accuracy"])
 file_path = "best_e2e_rnn_atten_model.hdf5"
check_point = kc.ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1, save_best_only = True, mode = "min")
 history = model.fit(X_train, y_tra, batch_size=30, epochs=40, validation_data=(X_dev, y_dev), callbacks = [check_point])
 file_paths.append(file_path)
 histories.append(np.min(np.asarray(history.history['val_loss'])))
 del model, history
gc.collect()
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return img<find_best_params>
print("load best model: " + str(file_paths[np.argmin(histories)]))
model = models.load_model(
 file_paths[np.argmin(histories)], cos[np.argmin(histories)] )
seg2 = seg.iloc[0:sample]
val_idxs = get_cv_idxs(sample )<prepare_x_and_y>
loss, score = model.evaluate(x=X_test, y=y_test)
 print('model evalutation loss = {} and score = {}'.format(loss, score))
indexes = seg2.Target.apply(str.split)
 y = np.zeros(( sample, 28))
for i in range(sample):
y[i,np.array(indexes[i], dtype=int)]=1<split>
y_sub_preds = model.predict(X_sub_test, batch_size = 1024, verbose = 1)
 sub_df_path = os.path.join(SUB_DATA_FOLDER, 'sample_submission_stage_2.csv')
 sub_df = pd.read_csv(sub_df_path)
 sub_df.loc[:, 'A'] = pd.Series(y_sub_preds[:, 0])
 sub_df.loc[:, 'B'] = pd.Series(y_sub_preds[:, 1])
 sub_df.loc[:, 'NEITHER'] = pd.Series(y_sub_preds[:, 2])
 sub_df.head()
<create_dataframe><EOS> 
sub_df.to_csv("submission.csv", index=False )