Kaggle competition: Shelter Animal Outcomes
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 test.has_cactus = preds.numpy() [:, 0]<save_to_csv>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )
test.to_csv("submit.csv", index=False )<import_modules>
submisson=pd.read_csv(".. /input/sample_submission.csv")
 submisson.head()
pyLDAvis.enable_notebook()
np.random.seed(2018)
 warnings.filterwarnings('ignore' )<load_from_csv>
train.groupby("Breed" ).AnimalID.count()
train.head()
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )<load_from_csv>
data=pd.concat([train_x,test_x],axis=0)
 data.DateTime=pd.to_datetime(data.DateTime)
 data.head()
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )<load_from_csv>
h = FeatureHasher(n_features=100,input_type="string")
 data["Name"]=data["Name"].fillna("None")
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )<define_variables>
for i in data["Name"].value_counts().index[0:400]:
print(i )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
] 
<feature_engineering>
fre_name=data["Name"].value_counts().index[0:400]
def etl(data):
data["year"]=data.DateTime.dt.year
data["month"]=data.DateTime.dt.month
data["day"]=data.DateTime.dt.day
data["dayofweek"]=data.DateTime.dt.dayofweek
data["hour"]=data.DateTime.dt.hour
data["AgeuponOutcome"]= data.AgeuponOutcome.fillna("-1")
 data["SexuponOutcome"]=data.SexuponOutcome.fillna("Unknown")
 data["color"]=data.Color.apply(lambda e: e.split("/")[0].split(" ")[0])
 data["color1"]=data.Color.apply(lambda e: re.split("/",e)[1].split(" ")[0]
if len(re.split("/",e)) >1 else "None")
 data["breed"]=data.Breed.apply(lambda e : e.split("/")[0])
 data["breed1"]=data.Breed.apply(lambda e: re.split("/",e)[1]
if len(re.split("/",e)) >1 else "None")
 data["is_mix"]=data.Breed.apply(lambda e : '1' if "Mix" in e else '0')
 cols=["year","month","day","dayofweek","hour","AgeuponOutcome","SexuponOutcome","color","color1",
"breed","breed1","is_mix"]
d=pd.get_dummies(data.AnimalType,prefix="AnimalType")
 for col in cols:
d=pd.concat([d,pd.get_dummies(data[col],prefix=col)],axis=1)
 for name in fre_name:
d[name]=data.Name.apply(lambda e :1 if e ==name else 0)
 d.drop(["Name"])
 return d
lengths = train['question_title'].apply(len)
 train['lengths'] = lengths
lengths = train.loc[train['lengths']<4000]['lengths']
sns.distplot(lengths, color='b')
 plt.show()<feature_engineering>
d=etl(data)
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )<count_unique_values>
enc = LabelEncoder()
enc.fit(train.OutcomeType)
 train_y=enc.transform(train.OutcomeType)
 train_y
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value 
<set_options>
clf = linear_model.LogisticRegression(C=0.1,max_iter=2000,random_state=234)
 clf = RandomForestClassifier(n_estimators=1000,max_depth=50, min_samples_split=1, random_state=234)
 clf.fit(train1, train_y)
 y_pred = clf.predict(train1)
 print("Number of mislabeled points out of a total %d points : %d" %(train1.shape[0],(train_y != y_pred ).sum()))
np.set_printoptions(suppress=True )<load_from_csv>
T_train_xgb = xgb.DMatrix(train1, label=train_y)
 watchlist = [(T_train_xgb, 'train'),(T_train_xgb, 'eval')]
params = {"objective": "multi:softprob",
"booster" : "gbtree",
"num_class" : 5,
"eta": 0.3,
"subsample": 0.9,
"colsample_bytree": 0.7,
"seed": 1301,
"max_depth":80,
"eval_metric":"mlogloss"
}
gbm = xgb.train(dtrain=T_train_xgb,params=params, evals=watchlist,num_boost_round = 35,verbose_eval=True)
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
id=test.ID
pred = gbm.predict(xgb.DMatrix(test1))
id.shape,pred.shape
submission=pd.DataFrame({"ID":id})
 submission['Adoption']=pred[:,0]
submission['Died']=pred[:,1]
submission['Euthanasia']=pred[:,2]
submission['Return_to_owner']=pred[:,3]
submission['Transfer']=pred[:,4]
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
id=test.ID
pred = clf.predict_proba(test1)
 id.shape,pred.shape
submission=pd.DataFrame({"ID":id})
 submission['Adoption']=pred[:,0]
submission['Died']=pred[:,1]
submission['Euthanasia']=pred[:,2]
submission['Return_to_owner']=pred[:,3]
submission['Transfer']=pred[:,4]
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<load_pretrained>
submission.to_csv("submission.csv",index=False )
<load_pretrained><EOS> 
submission.to_csv("submission.csv",index=False)