Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<string_transform>
np.random.seed(369 )
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<compute_test_metric>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<compute_train_metric>
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch, score)) 
<compute_train_metric>
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=True )(inp)
 x = Bidirectional(LSTM(50, return_sequences=True,dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = BatchNormalization()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 def loss(y_true, y_pred):
return K.binary_crossentropy(y_true, y_pred)
 model.compile(loss=loss, optimizer='nadam', metrics=['accuracy'] )<train_model>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
def schedule(ind):
a = [0.002,0.003, 0.000]
return a[ind]
lr = callbacks.LearningRateScheduler(schedule)
 [X_train, X_val, y_train, y_val] = train_test_split(X_t, y, train_size=0.95)
 ra_val = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)
 model.fit(X_train, y_train, batch_size=64, epochs=3, validation_data=(X_val, y_val), callbacks=[lr, ra_val])
<save_to_csv>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('LSTM-submission.csv', index=False )<import_modules>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = ['pic_'+str(i)for i in range(train_feats.shape[1])]
import numpy as np
import pandas as pd
from contextlib import contextmanager
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.sparse import hstack
import time
import re
import string
from scipy.sparse import csr_matrix
from sklearn.preprocessing import MinMaxScaler
import lightgbm as lgb
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import gc
from collections import defaultdict
import os
import psutil<load_from_csv>
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = ['pic_'+str(i)for i in range(test_feats.shape[1])]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<define_variables>
test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats.head()
cont_patterns = [ 
(b'(W|w)on't', b'will not'), 
(b'(C|c)an't', b'can not'), 
(b'(I|i)'m', b'i am'), 
(b'(A|a)in't', b'is not'), 
(b'(\w+)'ll', b'\g<1> will'), 
(b'(\w+)n't', b'\g<1> not'), 
(b'(\w+)'ve', b'\g<1> have'), 
(b'(\w+)'s', b'\g<1> is'), 
(b'(\w+)'re', b'\g<1> are'), 
(b'(\w+)'d', b'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]<categorify>
print('Train')
 train = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/petfinder-adoption-prediction/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/petfinder-adoption-prediction/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/petfinder-adoption-prediction/state_labels.csv")
 print(states.shape)
 target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train['second_color'] = train['Color2']*10 + train['Color3']
train['len_desc'] = train['Description'].str.len()
train['len_name'] = train['Name'].str.len()
train['Breed1_Cat'] = train['Breed1'][train['Type']==1].fillna(-1)
 train['Breed1_Dog'] = train['Breed1'][train['Type']==2].fillna(-1)
 train['Breed2_Cat'] = train['Breed2'][train['Type']==1].fillna(-1)
 train['Breed2_Dog'] = train['Breed2'][train['Type']==2].fillna(-1)
 train['age_bins'] = pd.cut(train['Age'],5,labels=[1,2,3,4,5] ).astype(int)
 train['gender+age_bins'] = train['Gender']*10+train['age_bins']
test['second_color'] = test['Color2']*10 + test['Color3']
test['len_desc'] = test['Description'].str.len()
test['len_name'] = test['Name'].str.len()
test['Breed1_Cat'] = test['Breed1'][test['Type']==1].fillna(-1)
 test['Breed1_Dog'] = test['Breed1'][test['Type']==2].fillna(-1)
 test['Breed2_Cat'] = test['Breed2'][test['Type']==1].fillna(-1)
 test['Breed2_Dog'] = test['Breed2'][test['Type']==2].fillna(-1)
 test['age_bins'] = pd.cut(test['Age'],5,labels=[1,2,3,4,5] ).astype(int)
 test['gender+age_bins'] = test['Gender']*10+test['age_bins']
test['AdoptionSpeed'] = np.nan
X = pd.concat([train, test], ignore_index=True, sort=False)
 rescuer_count = X.groupby(['RescuerID'])['PetID'].count()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
train['RescuerID_COUNT'] = train['RescuerID'].map(rescuer_count.to_dict())
 test['RescuerID_COUNT'] = test['RescuerID'].map(rescuer_count.to_dict())
 train = pd.merge(train, train_feats, how='left', on='PetID')
 test = pd.merge(test, test_feats, how='left', on='PetID')
 del X; gc.collect()
def prepare_for_char_n_gram(text):

 in r""
clean = bytes(text.lower() , encoding="utf-8")
 and \t
clean = clean.replace(b"
", b" ")
 clean = clean.replace(b"\t", b" ")
 clean = clean.replace(b"\b", b" ")
 clean = clean.replace(b"\r", b" ")
 for(pattern, repl)in patterns:
clean = re.sub(pattern, repl, clean)
 exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
 clean = b" ".join([exclude.sub(b'', token)for token in clean.split() ])
 clean = re.sub(b"\d+", b" ", clean)
 clean = re.sub(b'\s+', b' ', clean)
 clean = re.sub(b'\s+$', b'', clean)
 clean = re.sub(b" ", b"
clean = b"
return str(clean, 'utf-8' )<count_values>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score
def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))<feature_engineering>
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=200)
 svd.fit(X)
 svd_col = svd.transform(X)
 svd_col = pd.DataFrame(svd_col, columns=['svd_{}'.format(i)for i in range(200)])
 train = pd.concat(( train, svd_col), axis=1)
 svd_col_test = svd.transform(X_test)
 svd_col_test = pd.DataFrame(svd_col_test, columns=['svd_{}'.format(i)for i in range(200)])
 test = pd.concat(( test, svd_col_test), axis=1)
 nmf = NMF(n_components=200)
 nmf.fit(X)
 nmf_col = nmf.transform(X)
 nmf_col = pd.DataFrame(nmf_col, columns=['nmf_{}'.format(i)for i in range(200)])
 train = pd.concat(( train, nmf_col), axis=1)
 nmf_col_test = svd.transform(X_test)
 nmf_col_test = pd.DataFrame(nmf_col_test, columns=['nmf_{}'.format(i)for i in range(200)])
 test = pd.concat(( test, nmf_col_test), axis=1)
 print("train:", train.shape )
def get_indicators_and_clean_comments(df):

 df["ant_slash_n"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"
", x))
df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
df["nb_ng"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
df["has_http"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"http[s]{0,1}://\S+", x))
df["has_mail"] = df["comment_text"].apply(
 lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', x)
)
 df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r""{4}\S+"{4}", x))
df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
 df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x)))/ df["clean_comment"].apply(
 lambda x: 1 + min(99, len(x)) )<load_from_csv>
gc.collect()
def get_desc_vector(sent):
v = np.zeros(( 300,))
n_words = 0
for w in sent.split() :
if w in embeddings_index:
v += embeddings_index.get(w)
 n_words += 1
v = v / n_words if n_words > 0 else np.zeros(( 300,))
return v
def preprocess(x):
x = str(x ).lower()
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 x = re_tok.sub(r' \1 ', x)
 return x
train.Description = train.Description.apply(preprocess)
 test.Description = test.Description.apply(preprocess)
 EMBEDDING_FILE = '.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split(" ")) for o in open(EMBEDDING_FILE))
pool = Pool(2)
 try: train_x_1 = pool.map(get_desc_vector, train.Description.values.tolist())
 except ValueError as error: print(error)
 pool.terminate()
train_x_1 = np.array(train_x_1)
 pool = Pool(2)
 try: test_x_1 = pool.map(get_desc_vector, test.Description.values.tolist())
 except ValueError as error: print(error)
 pool.terminate()
test_x_1 = np.array(test_x_1)
 del embeddings_index; gc.collect()
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split(" ")) for o in open(EMBEDDING_FILE))
pool = Pool(2)
 try: train_x_2 = pool.map(get_desc_vector, train.Description.values.tolist())
 except ValueError as error: print(error)
 pool.terminate()
train_x_2 = np.array(train_x_2)
 pool = Pool(2)
 try: test_x_2 = pool.map(get_desc_vector, test.Description.values.tolist())
 except ValueError as error: print(error)
 pool.terminate()
test_x_2 = np.array(test_x_2)
 del embeddings_index; gc.collect()
train_x_mean = np.mean([train_x_1, train_x_2], axis=0)
 test_x_mean = np.mean([test_x_1, test_x_2], axis=0)
 embedding_train = pd.DataFrame()
embedding_train['PetID'] = train.PetID.values
for i in range(300):
embedding_train[f'emb_{i}'] = train_x_mean[:,i]
embedding_test = pd.DataFrame()
embedding_test['PetID'] = test.PetID.values
for i in range(300):
embedding_test[f'emb_{i}'] = test_x_mean[:,i]
train = pd.merge(train, embedding_train, how='left', on='PetID')
 test = pd.merge(test, embedding_test, how='left', on='PetID')
 print(train.shape)
 del train_x_mean, embedding_train, embedding_test;gc.collect()
def char_analyzer(text):

 tokens = text.split()
return [token[i: i + 3] for token in tokens for i in range(len(token)- 2)]
if __name__ == '__main__':
gc.enable()
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
with timer("Reading input files"):
train = pd.read_csv('.. /input/train.csv' ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 with timer("Performing basic NLP"):
get_indicators_and_clean_comments(train)
 get_indicators_and_clean_comments(test)
 with timer("Creating numerical features"):
num_features = [f_ for f_ in train.columns
if f_ not in ["comment_text", "clean_comment", "id", "remaining_chars",
'has_ip_address'] + class_names]
skl = MinMaxScaler()
train_num_features = csr_matrix(skl.fit_transform(train[num_features]))
test_num_features = csr_matrix(skl.fit_transform(test[num_features]))
train_text = train['clean_comment']
test_text = test['clean_comment']
all_text = pd.concat([train_text, test_text])
 with timer("Tfidf on word"):
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 2),
max_features=20000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text)
 del word_vectorizer
gc.collect()
with timer("Tfidf on char n_gram"):
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
tokenizer=char_analyzer,
analyzer='word',
ngram_range=(1, 1),
max_features=50000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text)
 del char_vectorizer
gc.collect()
print(( train_char_features > 0 ).sum(axis=1 ).max())
 del train_text
del test_text
gc.collect()
with timer("Staking matrices"):
csr_trn = hstack(
 [
train_char_features,
train_word_features,
train_num_features
]
).tocsr()
del train_num_features
del train_char_features
gc.collect()
csr_sub = hstack(
 [
test_char_features,
test_word_features,
test_num_features
]
).tocsr()
del test_num_features
del test_char_features
gc.collect()
submission = pd.DataFrame.from_dict({'id': test['id']})
 del test
gc.collect()
drop_f = [f_ for f_ in train if f_ not in ["id"] + class_names]
train.drop(drop_f, axis=1, inplace=True)
 gc.collect()
params = {
"objective": "binary",
'metric': {'auc'},
"boosting_type": "gbdt",
"verbosity": -1,
"num_threads": 4,
"bagging_fraction": 0.8,
"feature_fraction": 0.8,
"learning_rate": 0.1,
"num_leaves": 31,
"verbose": -1,
"min_split_gain":.1,
"reg_alpha":.1
}
with timer("Scoring Light GBM"):
scores = []
folds = KFold(n_splits=4, shuffle=True, random_state=1)
 lgb_round_dict = defaultdict(int)
 trn_lgbset = lgb.Dataset(csr_trn, free_raw_data=False)
 del csr_trn
gc.collect()
for class_name in class_names:
print("Class %s scores : " % class_name)
 class_pred = np.zeros(len(train))
train_target = train[class_name]
trn_lgbset.set_label(train_target.values)
 lgb_rounds = 500
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train, train_target)) :
watchlist = [
trn_lgbset.subset(trn_idx),
trn_lgbset.subset(val_idx)
 ]
model = lgb.train(
 params=params,
train_set=watchlist[0],
num_boost_round=lgb_rounds,
valid_sets=watchlist,
early_stopping_rounds=50,
verbose_eval=0 
)
 class_pred[val_idx] = model.predict(trn_lgbset.data[val_idx], num_iteration=model.best_iteration)
 score = roc_auc_score(train_target.values[val_idx], class_pred[val_idx])
 lgb_round_dict[class_name] += model.best_iteration
print("\t Fold %d : %.6f in %3d rounds" %(n_fold + 1, score, model.best_iteration))
print("full score : %.6f" % roc_auc_score(train_target, class_pred))
scores.append(roc_auc_score(train_target, class_pred))
train[class_name + "_oof"] = class_pred
train[["id"] + class_names + [f + "_oof" for f in class_names]].to_csv("lvl0_lgbm_clean_oof.csv",
index=False,
float_format="%.8f")
 print('Total CV score is {}'.format(np.mean(scores)))
 with timer("Predicting probabilities"):
for class_name in class_names:
with timer("Predicting probabilities for %s" % class_name):
train_target = train[class_name]
trn_lgbset.set_label(train_target.values)
 model = lgb.train(
 params=params,
train_set=trn_lgbset,
num_boost_round=int(lgb_round_dict[class_name] / folds.n_splits)
)
 submission[class_name] = model.predict(csr_sub, num_iteration=model.best_iteration)
 submission.to_csv("lgbm_clean_submission.csv", index=False, float_format="%.8f" )<define_variables>
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores
EXP_NUMBER = 63<import_modules>
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 train.drop(['Breed1', 'Breed2', 'age_bins'], axis=1, inplace=True)
 test.drop(['Breed1', 'Breed2', 'age_bins'], axis=1, inplace=True)
 train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )
import pandas as pd
import numpy as np<import_modules>
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed',
'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac',
'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance',
'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score', 'len_desc', 'len_name',
'RescuerID_COUNT', 'second_color'] +\
[col for col in train.columns if col.startswith('pic')or col.startswith('svd')or col.startswith('nmf')or col.startswith('emb')]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 foo = train.dtypes
cat_feature_names = foo[foo == "category"]
cat_features = [train.columns.get_loc(c)for c in train.columns if c in cat_feature_names]
import pandas as pd
import numpy as np<load_from_csv>
for i in cat_cols:
train.loc[:, i] = pd.factorize(train.loc[:, i])[0]
test.loc[:, i] = pd.factorize(test.loc[:, i])[0]
df_train = pd.read_csv('.. /input/champs-scalar-coupling/train.csv')
 df_test = pd.read_csv('.. /input/champs-scalar-coupling/test.csv')
 df_struct = pd.read_csv('.. /input/champs-scalar-coupling/structures.csv' )<import_modules>
tf.reset_default_graph()
N_SPLITS = 5
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=N_SPLITS, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], N_SPLITS))
all_coefficients = np.zeros(( N_SPLITS, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/' + str(N_SPLITS))
if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i - 1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / float(N_SPLITS)
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
n_splits = 5
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = xgb.DMatrix(data=train_X, label=train_y, feature_names=train_X.columns)
 d_valid = xgb.DMatrix(data=test_X, label=test_y, feature_names=test_X.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
print('Train LGB')
 model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 print('Predict 1/2')
 pred_test_y = model.predict(xgb.DMatrix(test_X, feature_names=test_X.columns), ntree_limit=model.best_ntree_limit)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(xgb.DMatrix(test_X2, feature_names=test_X2.columns), ntree_limit=model.best_ntree_limit)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb')
from sklearn.preprocessing import LabelEncoder<categorify>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )
type_enc = LabelEncoder()
df_train['type'] = type_enc.fit_transform(df_train['type'])
 df_test['type'] = type_enc.transform(df_test['type'] )<categorify>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )
atom_enc = LabelEncoder()
df_struct['atom'] = atom_enc.fit_transform(df_struct['atom'] )<merge>
print("True Distribution:")
 print(pd.value_counts(target, normalize=True ).sort_index())
 print("Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index())
 print("Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index() )
df_train = df_train.merge(df_struct,
left_on=['molecule_name','atom_index_0'],
right_on=['molecule_name','atom_index'] )<merge>
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )
df_train = df_train.merge(df_struct,
left_on=['molecule_name','atom_index_1'],
right_on=['molecule_name','atom_index'],
suffixes=('_0', '_1'))<define_variables>
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()
n_neighbors = 5
neigh_da_cols = [f'da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_d0_cols = [f'd0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_d1_cols = [f'd1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)for typ in range(5)]
neigh_cosa_cols = [f'cosine_da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_cos0_cols = [f'cosine_d0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_cos1_cols = [f'cosine_d1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1)
 for typ in range(5)]
neigh_all_cols = neigh_d0_cols + neigh_d1_cols
neigh_cols = ['id'] + neigh_all_cols<load_from_csv>
submission.to_csv('submission.csv', index=False )
<load_from_csv><EOS> 
submission.to_csv('submission.csv', index=False )