Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<load_from_csv>
K.tensorflow_backend._get_available_gpus()
MAXLEN=128
PATH = '.. /input/tf-roberta/'
tokenizer=tokenizers.ByteLevelBPETokenizer(vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
<feature_engineering>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
pd.set_option('display.max_columns', 1000)
ct=train.shape[0]
input_ids=np.ones(( ct,MAXLEN),dtype='int32')
 attention_mask=np.zeros(( ct,MAXLEN),dtype='int32')
 token_type_ids=np.zeros(( ct,MAXLEN),dtype='int32')
 start_tokens=np.zeros(( ct,MAXLEN),dtype='int32')
 end_tokens=np.zeros(( ct,MAXLEN),dtype='int32')
 for k in tqdm(range(ct)) :
text1=" "+" ".join(train.loc[k,"text"].split())
 text2=" ".join(train.loc[k,"selected_text"].split())
 idx=text1.find(text2)
 chars=np.zeros(len(text1))
chars[idx:idx+len(text2)]=1
if(text1[idx-1]==" "):
chars[idx-1]=1
enc=tokenizer.encode(text1)
 offsets=enc.offsets
toks=[]
for i,(a,b)in enumerate(offsets):
sm=np.sum(chars[a:b])
 if sm > 0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1 
<feature_engineering>
pd.set_option('display.max_columns', 50 )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAXLEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAXLEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAXLEN),dtype='int32')
 for k in tqdm(range(test.shape[0])) :
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1 
<load_pretrained>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
def build_model() :
ids = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<choose_model_class>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
model=build_model()<define_variables>
df_all = pd.concat([train, test], ignore_index=True, sort=False)
 df_all.loc[np.isfinite(df_all.AdoptionSpeed), 'is_train'] = 1
df_all.loc[~np.isfinite(df_all.AdoptionSpeed), 'is_train'] = 0
models=os.listdir(".. /input/tfroberta5fold6epochs")
 model_file_path=".. /input/tfroberta5fold6epochs/"<predict_on_test>
all_columns = []
start_token=np.zeros(( ct,MAXLEN),dtype='float64')
 end_token=np.zeros(( ct,MAXLEN),dtype='float64')
 for model_path in models:
model.load_weights(model_file_path+model_path)
 pred=model.predict([input_ids_t,attention_mask_t,token_type_ids_t])
 start_token+=pred[0]/len(models)
 end_token+=pred[1]/len(models)
<string_transform>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
start_token=np.argmax(start_token,axis=1)
 end_token=np.argmax(end_token,axis=1)
 selected_text=[]
for k in range(ct):
a=start_token[k]
b=end_token[k]
if a<b:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 selected_text.append(tokenizer.decode(enc.ids[a-1:b]))
else:
if test.loc[k,'sentiment']=="neutral":
selected_text.append(test.loc[k,'text'])
 else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 selected_text.append(tokenizer.decode(enc.ids[b-1:a])) 
<save_to_csv>
train_df_ids = train[['PetID']]
train_df_metadata = pd.DataFrame(train_metadata_files)
 train_df_metadata.columns = ['metadata_filename']
train_metadata_pets = train_df_metadata['metadata_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_metadata = train_df_metadata.assign(PetID=train_metadata_pets)
 print(len(train_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(train_metadata_pets.unique() , train_df_ids['PetID'].unique()))
print(f'fraction of pets with metadata: {pets_with_metadatas / train_df_ids.shape[0]:.3f}')
 train_df_ids = train[['PetID']]
train_df_sentiment = pd.DataFrame(train_sentiment_files)
 train_df_sentiment.columns = ['sentiment_filename']
train_sentiment_pets = train_df_sentiment['sentiment_filename'].apply(lambda x: x.split(split_char)[-1].split('.')[0])
 train_df_sentiment = train_df_sentiment.assign(PetID=train_sentiment_pets)
 print(len(train_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(train_sentiment_pets.unique() , train_df_ids['PetID'].unique()))
print(f'fraction of pets with sentiment: {pets_with_sentiments / train_df_ids.shape[0]:.3f}' )
test['selected_text']=selected_text
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.head(5 )<set_options>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
del model
gc.collect()<create_dataframe>
aggregates = ['sum', 'mean', 'max', 'min', 'std']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
cols=['textID','text','sentiment','selected_text']
train_df=train[cols].copy()
del train
test_df=test.copy()
del test
gc.collect()<import_modules>
all_sentiment_gr = pd.concat([train_sentiment_gr, test_sentiment_gr], ignore_index=True, sort=False)
 all_metadata_gr = pd.concat([train_metadata_gr, test_metadata_gr], ignore_index=True, sort=False)
 all_metadata_desc = pd.concat([train_metadata_desc, test_metadata_desc], ignore_index=True, sort=False)
 all_sentiment_desc = pd.concat([train_sentiment_desc, test_sentiment_desc], ignore_index=True, sort=False )
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation
from tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D,Bidirectional
from tensorflow.keras.models import Model
from tensorflow.keras.layers import TimeDistributed
from tensorflow.keras.layers import concatenate
from tensorflow.compat.v1.keras.layers import CuDNNLSTM
import gc
import os<load_from_csv>
df_all = df_all.merge(
 all_sentiment_gr, how='left', on='PetID')
 df_all = df_all.merge(
 all_metadata_gr, how='left', on='PetID')
 df_all = df_all.merge(
 all_metadata_desc, how='left', on='PetID')
 df_all = df_all.merge(
 all_sentiment_desc, how='left', on='PetID')
 all_columns.extend(list(all_sentiment_gr.columns[1:])+list(all_metadata_gr.columns[1:]))
train_df=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 test_df=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv" )<categorify>
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
X_text = df_all[text_columns].copy()
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=128):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<load_pretrained>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1 )
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
 save_path = '/kaggle/working/distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True)
 fast_tokenizer<categorify>
df_all = pd.concat([df_all, text_features.reset_index(drop=True)], axis=1)
 all_columns.extend(text_features.columns )
x_train = fast_encode(train_df.text.astype(str), fast_tokenizer, maxlen=128)
 x_test = fast_encode(test_df.text.astype(str),fast_tokenizer,maxlen=128 )<load_pretrained>
train_image_nums = Counter([i.split('/')[-1].split('-')[0] for i in train_image_files])
 test_image_nums = Counter([i.split('/')[-1].split('-')[0] for i in test_image_files])
 train['image_num'] = train.PetID.apply(lambda x: train_image_nums[x] if x in train_image_nums else 0)
 test['image_num'] = test.PetID.apply(lambda x: test_image_nums[x] if x in test_image_nums else 0 )
transformer_layer = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased' )<feature_engineering>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'max', 'min', 'std'],
'width': ['sum', 'mean', 'max', 'min', 'std'],
'height': ['sum', 'mean', 'max', 'min', 'std'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
def create_targets(df):
df['t_text'] = df['text'].apply(lambda x: tokenizer.tokenize(str(x)))
 df['t_selected_text'] = df['selected_text'].apply(lambda x: tokenizer.tokenize(str(x)))
 def func(row):
x,y = row['t_text'],row['t_selected_text'][:]
for offset in range(len(x)) :
d = dict(zip(x[offset:],y))
check = [k==v for k,v in d.items() ]
if all(check)== True:
break
return [0]*offset + [1]*len(y)+ [0]*(len(x)-offset-len(y))
df['targets'] = df.apply(func,axis=1)
 return df
train_df = create_targets(train_df)
 print('MAX_SEQ_LENGTH_TEXT', max(train_df['t_text'].apply(len)))
 print('MAX_TARGET_LENGTH',max(train_df['targets'].apply(len)))
 MAX_TARGET_LEN=108<feature_engineering>
df_all = pd.merge(df_all,agg_imgs,on='PetID',how='left')
 all_columns.extend(new_columns + ['image_num'] )
train_df['targets'] = train_df['targets'].apply(lambda x :x + [0] *(MAX_TARGET_LEN-len(x)))
 targets=np.asarray(train_df['targets'].values.tolist())
<categorify>
breed_job = df_all[['Breed1','Breed2']]
breed_job['mixed_breed'] =(( breed_job[['Breed1','Breed2']]==307 ).sum(axis=1)>0 ).astype('int')
 breed_job.loc[breed_job['Breed1']==307,'Breed1'] = 0
breed_job.loc[breed_job['Breed2']==307,'Breed2'] = 0
breed_job.loc[breed_job['mixed_breed']==1,'Breed1'] = breed_job.loc[breed_job['mixed_breed']==1,'Breed1']+breed_job.loc[breed_job['mixed_breed']==1,'Breed2']
breed_job.loc[breed_job['mixed_breed']==1,'Breed2'] = 0
breed_job.loc[(breed_job['mixed_breed']==0)&(breed_job['Breed2']!=0),'extra_mixed'] = 1
breed_job.loc[breed_job['extra_mixed']==1,'Breed2'] = 0
breed_job['mixed_breed'] = breed_job['mixed_breed'] + breed_job['extra_mixed'].fillna(0)
 df_all = df_all.drop(['Breed1', 'Breed2',],axis=1)
 df_all = pd.concat(
 [df_all, breed_job[['Breed1','mixed_breed']]], axis=1 )
lb=LabelEncoder()
sent_train=lb.fit_transform(train_df['sentiment'])
 sent_test=lb.fit_transform(test_df['sentiment'] )<choose_model_class>
df_all['Type'] = df_all['Type'].map({1:1,2:0})
 df_all = df_all.rename(columns={'Type':'is_dog'} )
def new_model(transformer_layer):
inp = Input(shape=(128,))
inp2= Input(shape=(1,))
embedding_matrix=transformer_layer.weights[0].numpy()
x = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],weights=[embedding_matrix],trainable=False )(inp)
 x = CuDNNLSTM(150, return_sequences=True,name='lstm_layer', )(x)
 x = CuDNNLSTM(100, return_sequences=False,name='lstm_layer-2', )(x)
 y =Dense(10,activation='relu' )(inp2)
 x= concatenate([x,y])
 x = Dense(MAX_TARGET_LEN,activation='sigmoid' )(x)
 model = Model(inputs=[inp,inp2], outputs=x)
 model.compile(loss='binary_crossentropy',
optimizer='adam')
 return model<train_model>
df_all_breed_names = df_all[['Breed1']].merge(
 labels_breed[['BreedID','BreedName']], how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
model=new_model(transformer_layer)
 history=model.fit([x_train,sent_train],targets,epochs=3 )<predict_on_test>
def path_to_tensor(img_path):
img = image.load_img(img_path, target_size=(299, 299))
x = image.img_to_array(img)
 return np.expand_dims(x, axis=0)
 def paths_to_tensor(img_paths):
list_of_tensors = [path_to_tensor(img_path)for img_path in img_paths]
return np.vstack(list_of_tensors )
predictions=model.predict([x_test,sent_test] )<categorify>
predicted_df = df_all[(df_all.Breed1 == 0)&(df_all.mixed_breed == 1)& df_all.is_dog][['PetID', 'is_train']].copy().reset_index(drop=True)
 predicted_df['img_path'] = predicted_df.apply(lambda x: f".. /input/petfinder-adoption-prediction/{'train' if x['is_train'] else 'test'}_images/{x['PetID']}-1.jpg", axis=1)
 predicted_df = predicted_df[predicted_df.img_path.map(os.path.exists)].reset_index(drop=True)
 predicted_df['pred_breed'] = 0
fns = predicted_df.img_path.values
def convert_output(sub,predictions):
preds=[]
for i,row in enumerate(sub['text']):
text,target=row.lower() ,predictions[i].tolist()
target=np.round(target ).tolist()
try:
start,end=target.index(1),target[::-1].index(1)
 text_list=tokenizer.tokenize(text)
 text_list=text_list+(( 108-len(text_list)) *['pad'])
 start_w,end_w=text_list[start],text_list[-end]
start=text.find(start_w.replace("
end=text.find(end_w.replace("
pred=text[start:end]
except:
pred=text
preds.append(pred)
 return preds 
<compute_test_metric>
ImageFile.LOAD_TRUNCATED_IMAGES = True
def create_generator(batch_size):
while True:
for i in range(0, len(fns), batch_size):
chunk = fns[i:i+batch_size]
x = paths_to_tensor(chunk ).astype('float32')/255
yield x
prediction_text=convert_output(test_df,predictions )<import_modules>
batchsize = 256
pred_generator = create_generator(batchsize )
import numpy as np
import pandas as pd
import json<load_from_csv>
inp_ten = Input(shape=(299, 299, 3))
x = InceptionResNetV2(
 weights='.. /input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5',
include_top=False,
pooling=None
)(inp_ten)
 x = GlobalAveragePooling2D()(x)
 out = Dense(133, activation='softmax' )(x)
 model = Model(inp_ten, out)
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
model.load_weights('.. /input/dog-identification-pretrained/dog_identification_model.hdf5' )
train_data = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 submission_data = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 train = np.array(train_data)
 test = np.array(test_data)
 !mkdir -p data<find_best_params>
breed_dict = {i:j
for i, j in zip(
 labels_breed[labels_breed.Type == 1].BreedName.apply(lambda x: x.lower().replace(' ', '_')).values,
labels_breed[labels_breed.Type == 1].BreedID.values)}
%%time
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1<train_model>
dog_names = ['in/001.Affenpinscher', 'in/002.Afghan_hound', 'in/003.Airedale_terrier', 'in/004.Akita', 'in/005.Alaskan_malamute', 'in/006.American_eskimo_dog', 'in/007.American_foxhound', 'in/008.American_staffordshire_terrier', 'in/009.American_water_spaniel', 'in/010.Anatolian_shepherd_dog', 'in/011.Australian_cattle_dog', 'in/012.Australian_shepherd', 'in/013.Australian_terrier', 'in/014.Basenji', 'in/015.Basset_hound', 'in/016.Beagle', 'in/017.Bearded_collie', 'in/018.Beauceron', 'in/019.Bedlington_terrier', 'in/020.Belgian_malinois', 'in/021.Belgian_sheepdog', 'in/022.Belgian_tervuren', 'in/023.Bernese_mountain_dog', 'in/024.Bichon_frise', 'in/025.Black_and_tan_coonhound', 'in/026.Black_russian_terrier', 'in/027.Bloodhound', 'in/028.Bluetick_coonhound', 'in/029.Border_collie', 'in/030.Border_terrier', 'in/031.Borzoi', 'in/032.Boston_terrier', 'in/033.Bouvier_des_flandres', 'in/034.Boxer', 'in/035.Boykin_spaniel', 'in/036.Briard', 'in/037.Brittany', 'in/038.Brussels_griffon', 'in/039.Bull_terrier', 'in/040.Bulldog', 'in/041.Bullmastiff', 'in/042.Cairn_terrier', 'in/043.Canaan_dog', 'in/044.Cane_corso', 'in/045.Cardigan_welsh_corgi', 'in/046.Cavalier_king_charles_spaniel', 'in/047.Chesapeake_bay_retriever', 'in/048.Chihuahua', 'in/049.Chinese_crested', 'in/050.Chinese_shar-pei', 'in/051.Chow_chow', 'in/052.Clumber_spaniel', 'in/053.Cocker_spaniel', 'in/054.Collie', 'in/055.Curly-coated_retriever', 'in/056.Dachshund', 'in/057.Dalmatian', 'in/058.Dandie_dinmont_terrier', 'in/059.Doberman_pinscher', 'in/060.Dogue_de_bordeaux', 'in/061.English_cocker_spaniel', 'in/062.English_setter', 'in/063.English_springer_spaniel', 'in/064.English_toy_spaniel', 'in/065.Entlebucher_mountain_dog', 'in/066.Field_spaniel', 'in/067.Finnish_spitz', 'in/068.Flat-coated_retriever', 'in/069.French_bulldog', 'in/070.German_pinscher', 'in/071.German_shepherd_dog', 'in/072.German_shorthaired_pointer', 'in/073.German_wirehaired_pointer', 'in/074.Giant_schnauzer', 'in/075.Glen_of_imaal_terrier', 'in/076.Golden_retriever', 'in/077.Gordon_setter', 'in/078.Great_dane', 'in/079.Great_pyrenees', 'in/080.Greater_swiss_mountain_dog', 'in/081.Greyhound', 'in/082.Havanese', 'in/083.Ibizan_hound', 'in/084.Icelandic_sheepdog', 'in/085.Irish_red_and_white_setter', 'in/086.Irish_setter', 'in/087.Irish_terrier', 'in/088.Irish_water_spaniel', 'in/089.Irish_wolfhound', 'in/090.Italian_greyhound', 'in/091.Japanese_chin', 'in/092.Keeshond', 'in/093.Kerry_blue_terrier', 'in/094.Komondor', 'in/095.Kuvasz', 'in/096.Labrador_retriever', 'in/097.Lakeland_terrier', 'in/098.Leonberger', 'in/099.Lhasa_apso', 'in/100.Lowchen', 'in/101.Maltese', 'in/102.Manchester_terrier', 'in/103.Mastiff', 'in/104.Miniature_schnauzer', 'in/105.Neapolitan_mastiff', 'in/106.Newfoundland', 'in/107.Norfolk_terrier', 'in/108.Norwegian_buhund', 'in/109.Norwegian_elkhound', 'in/110.Norwegian_lundehund', 'in/111.Norwich_terrier', 'in/112.Nova_scotia_duck_tolling_retriever', 'in/113.Old_english_sheepdog', 'in/114.Otterhound', 'in/115.Papillon', 'in/116.Parson_russell_terrier', 'in/117.Pekingese', 'in/118.Pembroke_welsh_corgi', 'in/119.Petit_basset_griffon_vendeen', 'in/120.Pharaoh_hound', 'in/121.Plott', 'in/122.Pointer', 'in/123.Pomeranian', 'in/124.Poodle', 'in/125.Portuguese_water_dog', 'in/126.Saint_bernard', 'in/127.Silky_terrier', 'in/128.Smooth_fox_terrier', 'in/129.Tibetan_mastiff', 'in/130.Welsh_springer_spaniel', 'in/131.Wirehaired_pointing_griffon', 'in/132.Xoloitzcuintli', 'in/133.Yorkshire_terrier']
def do_qa_train(train):
output = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 output.append({'context': context.lower() , 'qas': qas})
 return output
qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile )<categorify>
preds = model.predict_generator(pred_generator, steps=len(fns)// batchsize + 1, verbose=1)
 preds = np.argmax(preds, axis=1)
 mapping = {
'american_foxhound': 'foxhound',
'anatolian_shepherd_dog': 'anatolian_shepherd',
'australian_cattle_dog': 'australian_cattle_dog/blue_heeler',
'belgian_malinois': 'belgian_shepherd_malinois',
'belgian_sheepdog': 'belgian_shepherd_dog_sheepdog',
'belgian_tervuren': 'belgian_shepherd_tervuren',
'bouvier_des_flandres': 'bouvier_des_flanders',
'brittany': 'brittany_spaniel',
'bulldog': 'american_bulldog',
'cane_corso': 'cane_corso_mastiff',
'cardigan_welsh_corgi': 'welsh_corgi',
'chinese_crested': 'chinese_crested_dog',
'chinese_shar-pei': 'shar_pei',
'dandie_dinmont_terrier': 'dandi_dinmont_terrier',
'entlebucher_mountain_dog': 'entlebucher',
'icelandic_sheepdog': 'shetland_sheepdog_sheltie',
'irish_red_and_white_setter': 'irish_setter',
'miniature_schnauzer': 'schnauzer',
'newfoundland': 'newfoundland_dog',
'nova_scotia_duck_tolling_retriever': 'nova_scotia_duck-tolling_retriever',
'parson_russell_terrier': 'jack_russell_terrier',
'pembroke_welsh_corgi': 'welsh_corgi',
'plott': 'plott_hound',
'wirehaired_pointing_griffon': 'german_wirehaired_pointer',
'xoloitzcuintli': 'xoloitzcuintle/mexican_hairless',
'yorkshire_terrier': 'yorkshire_terrier_yorkie'
}
def get_breed_id(prediction):
dog_name = dog_names[prediction].split('.')[-1].lower()
dog_name = dog_name if dog_name not in mapping else mapping[dog_name]
return breed_dict[dog_name]
predicted_df['pred_breed'] = [get_breed_id(x)for x in preds]
del model; gc.collect()
K.clear_session()
%%time
def do_qa_test(test):
output = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 output.append({'context': context.lower() , 'qas': qas})
 return output
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile )<install_modules>
predicted_df = predicted_df[['PetID', 'pred_breed']]
df_all = df_all.merge(predicted_df, on='PetID', how='left')
 del predicted_df; gc.collect()
df_all.pred_breed = df_all.pred_breed.fillna(0)
 get_values = lambda i, j: int(j)if j != 0 else i
df_all.pred_breed = [get_values(i, j)for i,j in zip(df_all.Breed1.values, df_all.pred_breed.values)]
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q<define_variables>
for e in ['hair','domestic', 'short', 'medium', 'retriever', 'terrier', 'tabby', 'long']:
df_all_breed_names['Breed_{}'.format(e)] =(df_all_breed_names.BreedName.apply(lambda x: e in str(x ).lower())).astype(int)
 breed_detail_features = ['Breed_{}'.format(e)for e in ['hair','domestic', 'short', 'medium', 'retriever', 'terrier', 'tabby', 'long']]
df_all = pd.concat(
 [df_all, df_all_breed_names[breed_detail_features]], axis=1 )
%%time
MODEL_PATH = '.. /input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model_args = {'reprocess_input_data': True,
'overwrite_output_dir': True,
'learning_rate': 5e-5,
'num_train_epochs': 2,
'max_seq_length': 192,
'doc_stride': 64,
'fp16': False
}
model = QuestionAnsweringModel('distilbert',
MODEL_PATH,
args = model_args,
use_cuda = True)
 model.train_model('data/train.json' )<save_to_csv>
le = LabelEncoder()
df_all['Breed1_le'] = le.fit_transform(df_all['Breed1'] )
%%time
predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_data['selected_text'] = predictions_df['answer']
submission_data.to_csv('submission.csv', index = False)
 print("File submitted successfully." )<count_values>
all_columns = []
all_columns.extend(breed_columns )
print('Number of submissions above: ' + str(len(lbs)) )<import_modules>
df_all['Age_bins'] = pd.cut(np.log1p(df_all.Age),9,labels=['Age_{}'.format(e)for e in range(9)])
 le = LabelEncoder()
df_all['Age_bins'] = le.fit_transform(df_all['Age_bins'])
 age_columns = ['Age_bins']
print('TF version',tf.__version__ )<define_variables>
all_columns.extend(age_columns )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
df_all['Name_isna'] = df_all['Name'].isna().astype(int)
 df_all.loc[~df_all['Name'].isna() ,'Name_len'] = df_all.loc[~df_all['Name'].isna() ,'Name'].apply(lambda x : len(str(x ).split()))
df_all['Name_len'] = df_all['Name_len'].fillna(0)
 name_with_numbers = [e for e in df_all.Name.unique() if len(set([str(t)for t in range(10)] ).intersection(set(str(e ).split())))>0]
df_all['Name_with_numbers'] =(df_all.Name.isin(name_with_numbers)).astype(int )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<feature_engineering>
name_columns = ['Name_len','Name_isna',"Name_with_numbers"]
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
all_columns.extend(name_columns )
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
df_all['Color_range'] =(df_all[['Color1','Color2','Color3',]]>0 ).sum(axis=1 )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<init_hyperparams>
df_all['Fee_bins']= pd.cut(np.log1p(df_all.Fee),5,labels=['Fee_{}'.format(e)for e in range(5)])
 le = LabelEncoder()
df_all['Fee_bins'] = le.fit_transform(df_all['Fee_bins'])
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
fee_columns = ['Fee_bins']
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
all_columns.extend(fee_columns )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
df_all["state_gdp"] = df_all.State.map(state_gdp)
 df_all["state_population"] = df_all.State.map(state_population)
 le = LabelEncoder()
df_all['State_le'] = le.fit_transform(df_all['State'] )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
state_columns = ['state_gdp',
'state_population',
'State_le',]
print('TF version',tf.__version__ )<load_from_csv>
all_columns.extend(state_columns )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
df_all['is_VideoAmt'] =(df_all.VideoAmt>0 ).astype('int')
 df_all['PhotoAmt_bins']= pd.cut(np.log1p(df_all.PhotoAmt),5,labels=['PhotoAmt_{}'.format(e)for e in range(5)])
 le = LabelEncoder()
df_all['PhotoAmt_bins'] = le.fit_transform(df_all['PhotoAmt_bins'] )
def jaccard_improve(str1, str2):
str1=str1.lower()
str2=str2.lower()
index=str1.find(str2)
 text1=str1[:index]
text2=str1[index:].replace(str2,'')
 words1=text1.split()
words2=text2.split()
if len(words1)>len(words2):
words1=words1[-3:]
mod_text=" ".join(words1)+" "+ str2
else:
words2=words2[0:2]
mod_text=str2+" "+" ".join(words2)
 return mod_text<string_transform>
video_photo_columns = ['is_VideoAmt','PhotoAmt_bins']
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<feature_engineering>
all_columns.extend(video_photo_columns )
train_df['selected_text_mod']=train_df['selected_text']
train_df['mod']=0<define_variables>
df_all['Description_isna'] = df_all['Description'].isna().astype(int)
 df_all['len_words_Description'] = df_all.Description.apply(lambda x: len(str(x ).split()))
n = 6
df_all['len_words_Description_bins']= pd.cut(np.log1p(df_all.len_words_Description),n,labels=['Description_{}'.format(e)for e in range(n)])
 le = LabelEncoder()
df_all['len_words_Description_bins'] = le.fit_transform(df_all['len_words_Description_bins'])
 df_all['len_isupper_Description'] = df_all.Description.apply(lambda x: len([c for c in str(x)if c.isupper() ]))
df_all['len_islower_Description'] = df_all.Description.apply(lambda x: len([c for c in str(x)if c.islower() ]))
count = lambda l1, l2: len(list(filter(lambda c: c in l2, l1)))
 df_all['len_punctuation_Description'] = df_all.Description.apply(lambda x: count(str(x), string.punctuation))
df_all['len_letters_Description'] = df_all.Description.apply(lambda x: count(str(x), string.ascii_letters))
df_all['len_characters_Description'] = df_all.Description.apply(lambda x: len(str(x)))
 df_all['len_isupper_Description_to_words'] = df_all['len_isupper_Description'] / df_all['len_words_Description']
df_all['len_islower_Description_to_words'] = df_all['len_islower_Description'] / df_all['len_words_Description']
df_all['len_punctuation_Description_to_words'] = df_all['len_punctuation_Description'] / df_all['len_words_Description']
df_all['len_letters_Description_to_words'] = df_all['len_letters_Description'] / df_all['len_words_Description']
df_all['len_characters_Description_to_words'] = df_all['len_characters_Description'] / df_all['len_words_Description']
df_all['len_isupper_Description_to_characters'] = df_all['len_isupper_Description'] / df_all['len_characters_Description']
df_all['len_islower_Description_to_characters'] = df_all['len_islower_Description'] / df_all['len_characters_Description']
df_all['len_punctuation_Description_to_characters'] = df_all['len_punctuation_Description'] / df_all['len_characters_Description']
df_all['len_letters_Description_to_characters'] = df_all['len_letters_Description'] / df_all['len_characters_Description']
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
description_columns = [
'Description_isna',
'len_words_Description',
'len_words_Description_bins',
'len_isupper_Description',
'len_islower_Description',
'len_punctuation_Description',
'len_letters_Description',
'len_characters_Description',
'len_isupper_Description_to_words',
'len_islower_Description_to_words',
'len_punctuation_Description_to_words',
'len_letters_Description_to_words',
'len_isupper_Description_to_characters',
'len_islower_Description_to_characters',
'len_punctuation_Description_to_characters',
'len_letters_Description_to_characters',
'len_characters_Description_to_words',
]
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32' )<define_variables>
all_columns.extend(description_columns )
<define_variables>
nlp = spacy.load('.. /input/spacyen-vectors-web-lg/spacy-en_vectors_web_lg/en_vectors_web_lg')
 desc_embd_cols = ['Describe_spacy_{}'.format(e)for e in range(300)]
desc_embd = pd.DataFrame(np.vstack(df_all[['Description']].dropna().apply(lambda x: nlp(x['Description'] ).vector,axis=1 ).values),
index=df_all[['Description']].dropna().index)
 desc_embd.columns= desc_embd_cols
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 print(train_df)
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 print(tokenizer.decode([2]))
offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
print(input_ids[k])
 attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
del nlp; gc.collect()
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
Describe_SVD = True
if Describe_SVD:
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1881)
 svd_col = svd_.fit_transform(desc_embd)
 svd_col_df = pd.DataFrame(svd_col,index=df_all[['Description']].dropna().index)
 svd_col_df = svd_col_df.add_prefix('Describe_spacy_SVD')
 df_all = df_all.join(svd_col_df)
 all_columns.extend(svd_col_df.columns)
 else:
df_all = df_all.join(desc_embd)
 all_columns.extend(desc_embd_cols )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model 
<define_variables>
del svd_col_df
del desc_embd; gc.collect()
n_splits = 5<load_pretrained>
images_df = df_all[['PetID', 'is_train', 'is_dog']].copy()
images_df['img_path'] = images_df.apply(lambda x: f".. /input/petfinder-adoption-prediction/{'train' if x['is_train'] else 'test'}_images/{x['PetID']}-1.jpg", axis=1)
 images_df = images_df[images_df.img_path.map(os.path.exists)].reset_index(drop=True )
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model8/v8-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
def load_image(img_path, preprocess_fn, img_size):
image = cv2.imread(img_path)
 image = cv2.resize(image,(img_size, img_size))
image = preprocess_fn(image)
 return image
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 st1=st.strip()
if st1=='****' or st1=='****!' or st1=='****!' or st1=='****!' or st1=='****,' or st1=='****,' or st1=='****.' or st1=='****.':
st=text1
elif st1=='(good':
st='good'
elif st1=='__joy':
st='joy'
all.append(st )<feature_engineering>
images_df = images_df.set_index('PetID' )
<save_to_csv>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False)
<import_modules>
batch_size = 256
pet_ids = images_df.index.values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),256,256,3))
for i,pet_id in enumerate(batch_pets):
image_ins = images_df.loc[pet_id]
batch_images[i] = load_image(image_ins['img_path'], preprocess_input, 256)
 batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
<import_modules>
img_feats = pd.DataFrame.from_dict(features, orient='index')
 img_feats.columns = [f'pic_{i}' for i in range(img_feats.shape[1])]
img_feats = img_feats.reset_index()
img_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
<set_options>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features = img_feats[[f'pic_{i}' for i in range(len(img_feats.columns)- 1)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_Dense_')
 img_feats = pd.concat([all_ids, svd_col], axis=1 )
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 tf.config.optimizer.set_jit(True)
 tf.config.optimizer.set_experimental_options(
 {"auto_mixed_precision": True} )<load_from_csv>
df_all = df_all.merge(img_feats, on='PetID', how='left')
 all_columns.extend(img_feats.columns.values)
 del img_feats
del m; gc.collect()
K.clear_session()
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df.dropna(inplace=True)
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df.loc[:, "selected_text"] = test_df.text.values
submission_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 print("train shape =", train_df.shape)
 print("test shape =", test_df.shape)
 PATH = ".. /input/bert-base-uncased/"
MAX_SEQUENCE_LENGTH = 128
TOKENIZER = BertWordPieceTokenizer(f"{PATH}/vocab.txt", lowercase=True, add_special_tokens=False)
 train_df.head(10 )<categorify>
inp = Input(shape=(224, 224, 3))
backbone = InceptionV3(
 input_tensor=inp,
weights='.. /input/keras-pretrained-models/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
include_top=False,
pooling=None, 
)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
def preprocess(tweet, selected_text, sentiment):

 tweet = tweet.decode('utf-8')
 selected_text = selected_text.decode('utf-8')
 sentiment = sentiment.decode('utf-8')
 tweet = " ".join(str(tweet ).split())
 selected_text = " ".join(str(selected_text ).split())
 idx_start, idx_end = None, None
for index in(i for i, c in enumerate(tweet)if c == selected_text[0]):
if tweet[index:index+len(selected_text)] == selected_text:
idx_start = index
idx_end = index + len(selected_text)
 break
intersection = [0] * len(tweet)
 if idx_start != None and idx_end != None:
for char_idx in range(idx_start, idx_end):
intersection[char_idx] = 1
enc = TOKENIZER.encode(tweet)
 input_ids_orig, offsets = enc.ids, enc.offsets
target_idx = []
for i,(o1, o2)in enumerate(offsets):
if sum(intersection[o1: o2])> 0:
target_idx.append(i)
 target_start = target_idx[0]
target_end = target_idx[-1]
sentiment_map = {
'positive': 3893,
'negative': 4997,
'neutral': 8699,
}
input_ids = [101] + [sentiment_map[sentiment]] + [102] + input_ids_orig + [102]
input_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 attention_mask = [1] *(len(input_ids_orig)+ 4)
 offsets = [(0, 0),(0, 0),(0, 0)] + offsets + [(0, 0)]
target_start += 3
target_end += 3
padding_length = MAX_SEQUENCE_LENGTH - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 attention_mask = attention_mask +([0] * padding_length)
 input_type_ids = input_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 elif padding_length < 0:
input_ids = input_ids[:padding_length-1] + [102]
attention_mask = attention_mask[:padding_length-1] + [1]
input_type_ids = input_type_ids[:padding_length-1] + [1]
offsets = offsets[:padding_length-1] + [(0, 0)]
if target_start >= MAX_SEQUENCE_LENGTH:
target_start = MAX_SEQUENCE_LENGTH - 1
if target_end >= MAX_SEQUENCE_LENGTH:
target_end = MAX_SEQUENCE_LENGTH - 1
return(
 input_ids, attention_mask, input_type_ids, offsets,
target_start, target_end, tweet, selected_text, sentiment, 
)
 class TweetSentimentDataset(tf.data.Dataset):
OUTPUT_TYPES =(
 tf.dtypes.int32, tf.dtypes.int32, tf.dtypes.int32,
tf.dtypes.int32, tf.dtypes.float32, tf.dtypes.float32,
tf.dtypes.string, tf.dtypes.string, tf.dtypes.string, 
)
 OUTPUT_SHAPES =(
(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,),(MAX_SEQUENCE_LENGTH,), 
(MAX_SEQUENCE_LENGTH, 2),() ,() , 
() ,() ,() , 
)
 def _generator(tweet, selected_text, sentiment):
for tw, st, se in zip(tweet, selected_text, sentiment):
yield preprocess(tw, st, se)
 def __new__(cls, tweet, selected_text, sentiment):
return tf.data.Dataset.from_generator(
 cls._generator,
output_types=cls.OUTPUT_TYPES,
output_shapes=cls.OUTPUT_SHAPES,
args=(tweet, selected_text, sentiment)
)
 @staticmethod
def create(dataframe, batch_size, shuffle_buffer_size=-1):
dataset = TweetSentimentDataset(
 dataframe.text.values,
dataframe.selected_text.values,
dataframe.sentiment.values 
)
 dataset = dataset.cache()
if shuffle_buffer_size != -1:
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset 
<choose_model_class>
batch_size = 256
pet_ids = images_df.index.values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),224, 224,3))
for i,pet_id in enumerate(batch_pets):
image_ins = images_df.loc[pet_id]
batch_images[i] = load_image(image_ins['img_path'], preprocess_input, 224)
 batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
class BertQAModel(TFBertPreTrainedModel):
DROPOUT_RATE = 0.1
NUM_HIDDEN_STATES = 2
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.bert = TFBertMainLayer(config, name="bert")
 self.concat = L.Concatenate()
self.dropout = L.Dropout(self.DROPOUT_RATE)
 self.qa_outputs = L.Dense(
 config.num_labels,
kernel_initializer=TruncatedNormal(stddev=config.initializer_range),
dtype='float32',
name="qa_outputs")
 @tf.function
def call(self, inputs, **kwargs):
_, _, hidden_states = self.bert(inputs, **kwargs)
 hidden_states = self.concat([
hidden_states[-i] for i in range(1, self.NUM_HIDDEN_STATES+1)
 ])
 hidden_states = self.dropout(hidden_states, training=kwargs.get("training", False))
logits = self.qa_outputs(hidden_states)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 return start_logits, end_logits
def train(model, dataset, loss_fn, optimizer):
@tf.function
def train_step(model, inputs, y_true, loss_fn, optimizer):
with tf.GradientTape() as tape:
y_pred = model(inputs, training=True)
 loss = loss_fn(y_true[0], y_pred[0])
 loss += loss_fn(y_true[1], y_pred[1])
 scaled_loss = optimizer.get_scaled_loss(loss)
 scaled_gradients = tape.gradient(scaled_loss, model.trainable_variables)
 gradients = optimizer.get_unscaled_gradients(scaled_gradients)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, y_pred
epoch_loss = 0.
 for batch_num, sample in enumerate(dataset):
loss, y_pred = train_step(
 model, sample[:3], sample[4:6], loss_fn, optimizer)
 epoch_loss += loss
print(
 f"training...batch {batch_num+1:03d} : "
f"train loss {epoch_loss/(batch_num+1):.3f} ",
end='\r')
 def predict(model, dataset, loss_fn, optimizer):
@tf.function
def predict_step(model, inputs):
return model(inputs)
 def to_numpy(*args):
out = []
for arg in args:
if arg.dtype == tf.string:
arg = [s.decode('utf-8')for s in arg.numpy() ]
out.append(arg)
 else:
arg = arg.numpy()
out.append(arg)
 return out
offset = tf.zeros([0, MAX_SEQUENCE_LENGTH, 2], dtype=tf.dtypes.int32)
 text = tf.zeros([0,], dtype=tf.dtypes.string)
 selected_text = tf.zeros([0,], dtype=tf.dtypes.string)
 sentiment = tf.zeros([0,], dtype=tf.dtypes.string)
 pred_start = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 pred_end = tf.zeros([0, MAX_SEQUENCE_LENGTH], dtype=tf.dtypes.float32)
 for batch_num, sample in enumerate(dataset):
print(f"predicting...batch {batch_num+1:03d}"+" "*20, end='\r')
 y_pred = predict_step(model, sample[:3])
 pred_start = tf.concat(( pred_start, y_pred[0]), axis=0)
 pred_end = tf.concat(( pred_end, y_pred[1]), axis=0)
 offset = tf.concat(( offset, sample[3]), axis=0)
 text = tf.concat(( text, sample[6]), axis=0)
 selected_text = tf.concat(( selected_text, sample[7]), axis=0)
 sentiment = tf.concat(( sentiment, sample[8]), axis=0)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
to_numpy(pred_start, pred_end, text, selected_text, sentiment, offset)
 return pred_start, pred_end, text, selected_text, sentiment, offset
def decode_prediction(pred_start, pred_end, text, offset, sentiment):
def decode(pred_start, pred_end, text, offset):
decoded_text = ""
for i in range(pred_start, pred_end+1):
decoded_text += text[offset[i][0]:offset[i][1]]
if(i+1)< len(offset)and offset[i][1] < offset[i+1][0]:
decoded_text += " "
return decoded_text
decoded_predictions = []
for i in range(len(text)) :
if sentiment[i] == "neutral" or len(text[i].split())< 2:
decoded_text = text[i]
else:
idx_start = np.argmax(pred_start[i])
 idx_end = np.argmax(pred_end[i])
 if idx_start > idx_end:
idx_end = idx_start
decoded_text = str(decode(idx_start, idx_end, text[i], offset[i]))
if len(decoded_text)== 0:
decoded_text = text[i]
decoded_predictions.append(decoded_text)
 return decoded_predictions
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c)) 
<choose_model_class>
img_feats = pd.DataFrame.from_dict(features, orient='index')
 img_feats.columns = [f'pic_inception{i}' for i in range(img_feats.shape[1])]
img_feats = img_feats.reset_index()
img_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
num_folds = 5
num_epochs = 5
batch_size = 32
learning_rate = 3e-5
optimizer = tf.keras.optimizers.Adam(learning_rate)
 optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, 'dynamic')
 config = BertConfig(output_hidden_states=True, num_labels=2)
 BertQAModel.DROPOUT_RATE = 0.1
BertQAModel.NUM_HIDDEN_STATES = 2
model = BertQAModel.from_pretrained(PATH, config=config)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 kfold = model_selection.StratifiedKFold(
 n_splits=num_folds, shuffle=True, random_state=42)
 test_preds_start = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 test_preds_end = np.zeros(( len(test_df), MAX_SEQUENCE_LENGTH), dtype=np.float32)
 for fold_num,(train_idx, valid_idx)in enumerate(
 kfold.split(X=train_df.text, y=train_df.sentiment.values)) :
print("
fold %02d" %(fold_num+1))
train_dataset = TweetSentimentDataset.create(
 train_df.iloc[train_idx], batch_size, shuffle_buffer_size=2048)
 valid_dataset = TweetSentimentDataset.create(
 train_df.iloc[valid_idx], batch_size, shuffle_buffer_size=-1)
 test_dataset = TweetSentimentDataset.create(
 test_df, batch_size, shuffle_buffer_size=-1)
 best_score = float('-inf')
 for epoch_num in range(num_epochs):
print("
epoch %03d" %(epoch_num+1))
train(model, train_dataset, loss_fn, optimizer)
 pred_start, pred_end, text, selected_text, sentiment, offset = \
predict(model, valid_dataset, loss_fn, optimizer)
 selected_text_pred = decode_prediction(
 pred_start, pred_end, text, offset, sentiment)
 jaccards = []
for i in range(len(selected_text)) :
jaccards.append(
 jaccard(selected_text[i], selected_text_pred[i]))
score = np.mean(jaccards)
 print(f"valid jaccard epoch {epoch_num+1:03d}: {score}"+" "*15)
 if score > best_score:
best_score = score
test_pred_start, test_pred_end, test_text, _, test_sentiment, test_offset = \
predict(model, test_dataset, loss_fn, optimizer)
 test_preds_start += test_pred_start * 0.2
test_preds_end += test_pred_end * 0.2
session = tf.compat.v1.get_default_session()
graph = tf.compat.v1.get_default_graph()
del session, graph, model
model = BertQAModel.from_pretrained(PATH, config=config)
 selected_text_pred = decode_prediction(
 test_preds_start, test_preds_end, test_text, test_offset, test_sentiment)
 submission_df.loc[:, 'selected_text'] = selected_text_pred
submission_df.to_csv("submission.csv", index=False )<load_from_csv>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features = img_feats[[f'pic_inception{i}' for i in range(len(img_feats.columns)- 1)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_Incep_')
 img_feats = pd.concat([all_ids, svd_col], axis=1 )
train_data= pd.read_csv('.. /input/titanic/train.csv')
 train_data.head(10 )<load_from_csv>
df_all = df_all.merge(img_feats, on='PetID', how='left')
 all_columns.extend(img_feats.columns.values)
 del img_feats
del m; gc.collect()
K.clear_session()
test_data=pd.read_csv('.. /input/titanic/test.csv')
 test_data.head()<count_missing_values>
inp = Input(shape=(224, 224, 3))
backbone = Xception(
 input_tensor=inp,
weights='.. /input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
include_top=False,
pooling=None, 
)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
print(train_data.isnull().sum())
 print(test_data.isnull().sum() )<prepare_x_and_y>
batch_size = 256
pet_ids = images_df.index.values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),224, 224,3))
for i,pet_id in enumerate(batch_pets):
image_ins = images_df.loc[pet_id]
batch_images[i] = load_image(image_ins['img_path'], preprocess_input, 224)
 batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
features=['Pclass', 'Sex' ,'Age', 'SibSp', 'Parch' ,'Fare' ,'Embarked']
y=train_data.Survived 
<feature_engineering>
img_feats = pd.DataFrame.from_dict(features, orient='index')
 img_feats.columns = [f'pic_xception{i}' for i in range(img_feats.shape[1])]
img_feats = img_feats.reset_index()
img_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
train_data['Age']=train_data['Age'].fillna(train_data['Age'].mode() [0])
 train_data['Embarked']=train_data['Embarked'].fillna(train_data['Embarked'].mode() [0])
 test_data['Age']=test_data['Age'].fillna(test_data['Age'].mode() [0])
 test_data['Fare']=test_data['Fare'].fillna(test_data['Fare'].median() )<categorify>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features = img_feats[[f'pic_xception{i}' for i in range(len(img_feats.columns)- 1)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_Xcep_')
 img_feats = pd.concat([all_ids, svd_col], axis=1 )
train_data.isnull().sum()
test_data.isnull().sum()
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 X.head()<train_model>
df_all = df_all.merge(img_feats, on='PetID', how='left')
 all_columns.extend(img_feats.columns.values)
 del img_feats
del images_df
del m; gc.collect()
K.clear_session()
x_train,x_test,y_train,y_test = train_test_split(X,y,test_size=0.25,random_state=6)
 sc= StandardScaler()
x_train= sc.fit_transform(x_train)
 x_test =sc.transform(x_test)
 model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(x_train, y_train)
 predictions = model.score(x_test,y_test)
 predictions<normalization>
df_external_1 = pd.read_csv('.. /input/petfindercomexternal/petfinder.com_external.csv')
 df_external_1 = df_external_1.drop('attr_other_names',axis=1)
 df_external_1 = df_external_1.rename(columns={'BreedID':'pred_breed'} )
sc= StandardScaler()
X= sc.fit_transform(X)
 X_test =sc.transform(X_test)
 shuffle(X, y,random_state=6 )<save_to_csv>
df_all = df_all.merge(df_external_1,how='left',on='pred_breed' )
model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
all_columns.extend(df_external_1.columns[:].values )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<create_dataframe>
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f)
 cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
breed_id = {}
for id_,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[name] = id_
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]
Sib = train_data.groupby(['SibSp','Survived'])
 Sibf = pd.DataFrame(Sib.count() ['PassengerId'])
 print(Sibf)
 Par = train_data.groupby(['Parch','Survived'])
 Parf = pd.DataFrame(Par.count() ['PassengerId'])
 print(Parf )<count_values>
df_cat_breeds = pd.DataFrame(cat_ratings ).T
df_dog_breeds = pd.DataFrame(dog_ratings ).T
df_detail_breeds = pd.concat([df_cat_breeds,df_dog_breeds],axis=1)
 df_detail_breeds = df_detail_breeds.reset_index().rename(columns={'index':'Breed1'} )
train_data.Cabin.value_counts()<feature_engineering>
df_detail_breeds = df_detail_breeds.append({'Breed1':'Terrier',
'Affectionate with Family': 4.606060606060606,
'Friendly Toward Strangers': 3.757575757575758,
'General Health': 4.0606060606060606,
'Intelligence': 4.090909090909091,
' Adaptability': 3.212121212121212,
' All Around Friendliness': 4.121212121212121,
' Exercise Needs': 4.454545454545454,
' Health Grooming': 2.8181818181818183,
' Trainability': 3.606060606060606,
'Adapts Well to Apartment Living': 3.9696969696969697,
'Amount Of Shedding': 2.515151515151515,
'Dog Friendly': 3.1818181818181817,
'Drooling Potential': 1.2424242424242424,
'Easy To Groom': 3.1515151515151514,
'Easy To Train': 3.3636363636363638,
'Energy Level': 4.212121212121212,
'Exercise Needs': 4.090909090909091,
'Good For Novice Owners': 3.0606060606060606,
'Incredibly Kid Friendly Dogs': 4.212121212121212,
'Intensity': 4.03030303030303,
'Potential For Mouthiness': 2.8484848484848486,
'Potential For Playfulness': 4.666666666666667,
'Potential For Weight Gain': 3.3333333333333335,
'Prey Drive': 3.5757575757575757,
'Sensitivity Level': 3.5454545454545454,
'Size': 2.0,
'Tendency To Bark Or Howl': 3.212121212121212,
'Tolerates Being Alone': 2.1818181818181817,
'Tolerates Cold Weather': 3.0303030303030303,
'Tolerates Hot Weather': 3.242424242424242,
'Wanderlust Potential': 3.6363636363636362}, ignore_index=True )
train_data['Embarked'] = train_data['Embarked'].fillna('S' )<count_values>
breed_mapper = {"White German Shepherd":"German Shepherd Dog",
"Jack Russell Terrier(Parson Russell Terrier)": "Jack Russell Terrier",
"Belgian Shepherd Dog Sheepdog": "Belgian Sheepdog",
"Shetland Sheepdog Sheltie": "Shetland Sheepdog",
"English Pointer": "Pointer",
"Appenzell Mountain Dog": "Bernese Mountain Dog",
"Yorkshire Terrier Yorkie": "Yorkshire Terrier",
"Thai Ridgeback": "Rhodesian Ridgeback",
"Spitz": "Finnish Spitz",
"Standard Poodle": "Poodle",
"Havana": "Havana Brown",
"Munsterlander": "Small Munsterlander Pointer",
"English Bulldog": "Bulldog",
"Husky": "Siberian Husky",
"Entlebucher": "Entlebucher Mountain Dog",
"Wire-haired Pointing Griffon": "Wirehaired Pointing Griffon",
"Chocolate Labrador Retriever": "Labrador Retriever",
"Belgian Shepherd Malinois": "Belgian Sheepdog",
"German Spitz": "Finnish Spitz",
"German Spitz": "Japanese Spitz",
"Smooth Fox Terrier": "Fox Terrier",
"Belgian Shepherd Tervuren": "Belgian Tervuren",
"Wirehaired Terrier": "German Wirehaired Pointer",
"Galgo Spanish Greyhound": "Greyhound",
"Welsh Corgi": "Cardigan Welsh Corgi",
"Eskimo Dog": "American Eskimo Dog",
"Sheep Dog": "Old English Sheepdog",
"American Hairless Terrier": "American Staffordshire Terrier",
"Retriever": "Labrador Retriever",
"Caucasian Sheepdog(Caucasian Ovtcharka)": "Caucasian Shepherd Dog",
"Oriental Tabby": "Oriental",
"Flat-coated Retriever": "Flat-Coated Retriever",
"Oriental Short Hair": "Oriental",
"Exotic Shorthair": "American Shorthair",
"Spaniel": "Cocker Spaniel",
"Wire Fox Terrier": "Fox Terrier",
"Oriental Long Hair": "Oriental",
"Chinese Crested Dog": "Chinese Crested",
"Applehead Siamese": "Siamese Cat",
"Klee Kai": "Alaskan Klee Kai",
"Dandi Dinmont Terrier": "Dandie Dinmont Terrier",
"Yellow Labrador Retriever": "Labrador Retriever",
"Bobtail": "American Bobtail",
"Anatolian Shepherd": "Anatolian Shepherd Dog",
"Cane Corso Mastiff": "Cane Corso",
"Bengal": "Bengal Cats",
"Pit Bull Terrier": "American Pit Bull Terrier",
"Shepherd": "German Shepherd Dog",
"Scottish Terrier Scottie": "Scottish Terrier",
"Mountain Dog": "Bernese Mountain Dog",
"Jindo": "Korean Jindo Dog",
"Foxhound": "American Foxhound",
"Bouvier des Flanders": "Bouvier des Flandres",
"Schnauzer": "Standard Schnauzer",
"Newfoundland Dog": "Newfoundland",
"Cattle Dog": "Australian Cattle Dog",
"West Highland White Terrier Westie": "West Highland White Terrier",
"Australian Cattle Dog/Blue Heeler": "Australian Cattle Dog",
"Maremma Sheepdog": "Belgian Sheepdog",
"Ragdoll": "Ragdoll Cats",
"Wheaten Terrier": "Soft Coated Wheaten Terrier",
"Setter": "English Setter",
"Siamese": "Siamese Cat",
"Black Labrador Retriever": "Labrador Retriever",
"Norwegian Forest Cat": "Norwegian Forest",
"English Coonhound": "American English Coonhound",
"Coonhound": "American English Coonhound",
"English Shepherd": "Old English Sheepdog",
"Plott Hound": "Plott",
"Brittany Spaniel": "Brittany",
"Corgi": "Cardigan Welsh Corgi",
"Illyrian Sheepdog": "Old English Sheepdog",
"Patterdale Terrier(Fell Terrier)": "Terrier",
"Nova Scotia Duck-Tolling Retriever": "Nova Scotia Duck Tolling Retriever",
"Hound": "Afghan Hound",
"Belgian Shepherd Laekenois": "Belgian Sheepdog",
"Sphynx(hairless cat)": "Sphynx",
"Mountain Cur": "Bernese Mountain Dog",
"Kai Dog": "Alaskan Klee Kai",}
train_data['Title'] = train_data['Name'].map(lambda x:x.split(',')[1].split('.')[0].strip())
 train_data['Title'].value_counts()<count_values>
labels_breed_temp = labels_breed.copy()
choose = labels_breed_temp.BreedName.isin(list(breed_mapper.keys()))
labels_breed_temp['NewBreedName'] = labels_breed_temp['BreedName']
labels_breed_temp.loc[choose,'NewBreedName'] = labels_breed_temp.loc[choose,'BreedName'].map(breed_mapper)
 breed_mapper_all = labels_breed_temp.set_index('BreedID')['NewBreedName'].to_dict()
TitleDict={}
TitleDict['Mr']='Mr'
TitleDict['Mlle']='Miss'
TitleDict['Miss']='Miss'
TitleDict['Master']='Master'
TitleDict['Jonkheer']='Master'
TitleDict['Mme']='Mrs'
TitleDict['Ms']='Mrs'
TitleDict['Mrs']='Mrs'
TitleDict['Don']='Royalty'
TitleDict['Sir']='Royalty'
TitleDict['the Countess']='Royalty'
TitleDict['Dona']='Royalty'
TitleDict['Lady']='Royalty'
TitleDict['Capt']='Officer'
TitleDict['Col']='Officer'
TitleDict['Major']='Officer'
TitleDict['Dr']='Officer'
TitleDict['Rev']='Officer'
train_data['Title']=train_data['Title'].map(TitleDict)
 train_data['Title'].value_counts()<categorify>
df_all_temp = df_all.copy()
df_all_temp.Breed1 = df_all_temp.pred_breed.map(breed_mapper_all)
 df_all_temp = df_all_temp.merge(df_detail_breeds,on='Breed1',how='left')
 df_detail_breeds_cols = df_detail_breeds.columns[1:]
def fare_category(fare):
if fare <= 4:
return 0
elif fare <= 10:
return 1
elif fare <= 30:
return 2
elif fare <= 45:
return 3
else:
return 4
train_data['Fare_Category'] = train_data['Fare'].map(fare_category )<feature_engineering>
df_all = pd.concat(( df_all,df_all_temp[df_detail_breeds_cols]),axis=1)
 all_columns.extend(list(df_detail_breeds.columns.values[1:]))
Ticket_Count = train_data.groupby('Ticket', as_index = False)['PassengerId'].count()
Ticket_Count_0 = Ticket_Count[Ticket_Count.PassengerId == 1]['Ticket']
train_data['GroupTicket'] = np.where(train_data.Ticket.isin(Ticket_Count_0), 0, 1)
<prepare_x_and_y>
df_external_3_cat = pd.read_csv('.. /input/pet-breed-characteristics/cat_breed_characteristics.csv')
 df_external_3_dog = pd.read_csv('.. /input/pet-breed-characteristics/dog_breed_characteristics.csv' )
def file_missing(document):
age_features = ['Age','Fare','Parch','SibSp','Pclass', 'Title']
age_features_dummies = pd.get_dummies(document[age_features])
 known_age = age_features_dummies[age_features_dummies.Age.notnull() ].values
unknown_age = age_features_dummies[age_features_dummies.Age.isnull() ].values
X = known_age[:, 1:]
y = known_age[:, 0]
model = RandomForestRegressor(random_state = 0, n_estimators = 2000, n_jobs = -1)
 model.fit(X,y)
 prediction_age = model.predict(unknown_age[:, 1::])
 document.loc[(document.Age.isnull()), 'Age' ] = prediction_age
return document, model
def Cabin_type(document):
document.loc[(document.Cabin.notnull()), 'Cabin' ] = "Yes"
document.loc[(document.Cabin.isnull()), 'Cabin' ] = "No"
return document
train_data, model = file_missing(train_data)
 train_data = Cabin_type(train_data)
 train_data.head(10)
<categorify>
cat_dog_cols = list(df_external_3_dog.columns)+list(set(df_external_3_cat.columns)- set(df_external_3_dog.columns))
df_external_3_all = pd.concat(( df_external_3_dog,df_external_3_cat),axis=0)[cat_dog_cols]
get_features = ["Pclass", "Sex","Age", "Cabin", "Fare", "Embarked","Title", "Familysize", 'GroupTicket','Fare_Category' ]
get_features_dummies = pd.get_dummies(train_data[get_features])
 print(get_features_dummies)
<normalization>
breed_id = {}
for id_,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[name] = id_
scaler = preprocessing.StandardScaler()
age_scale = scaler.fit(get_features_dummies['Age'].values.reshape(-1,1))
get_features_dummies['Age_scaled'] = scaler.fit_transform(get_features_dummies['Age'].values.reshape(-1,1), age_scale)
 fare_scale = scaler.fit(get_features_dummies['Fare'].values.reshape(-1,1))
get_features_dummies['Fare_scaled'] = scaler.fit_transform(get_features_dummies['Fare'].values.reshape(-1,1), fare_scale)
 get_features_dummies.head() 
<split>
df_external_3_all['BreedName']= df_external_3_all.BreedName.map(breed_id )
train_df = get_features_dummies.filter(regex='Age_.*|Fare_.*|Cabin_.*|Embarked_.*|Sex_.*|Pclass_.*|Title_.*|Familysize|GroupTicket|Fare_Category')
 X = train_df.values
y = train_data.Survived
train_X, val_X, train_y, val_y = train_test_split(X, y,test_size=0.2, random_state = 1)
 rf_est = ensemble.RandomForestClassifier(n_estimators = 30, criterion = 'gini', max_features = 'sqrt',
max_depth = 8, min_samples_split = 10, min_samples_leaf = 1,
n_jobs = 50, random_state = 42, verbose = 1)
 gbm_est = ensemble.GradientBoostingClassifier(n_estimators=900, learning_rate=0.0008, loss='exponential',
min_samples_split=3, min_samples_leaf=2, max_features='sqrt',
max_depth=3, random_state=42, verbose=1)
 et_est = ensemble.ExtraTreesClassifier(n_estimators=750, max_features='sqrt', max_depth=35, n_jobs=50,
criterion='entropy', random_state=42, verbose=1)
 voting_est = ensemble.VotingClassifier(estimators = [('rf', rf_est),('gbm', gbm_est),('et', et_est)],
voting = 'soft', weights = [3,5,2],
n_jobs = 50)
 voting_est.fit(train_X, train_y)
 voting_est_predictions = voting_est.predict(val_X)
 voting_est_mae = mean_absolute_error(voting_est_predictions, val_y)
 print(voting_est_mae )<choose_model_class>
df_external_3_all = df_external_3_all.rename(columns={"BreedName":'pred_breed'} )
forest = RandomForestClassifier()
cross_validation = StratifiedKFold(n_splits=10)
 parameter_grid = {
'max_depth' : [8],
'n_estimators': [30],
'max_features': ['sqrt'],
'min_samples_split': [10],
'min_samples_leaf':[1],
'bootstrap': [True],
}
grid_search = GridSearchCV(forest,
scoring='accuracy',
param_grid=parameter_grid,
cv=cross_validation,
verbose=1 
)
 grid_search.fit(train_X, train_y)
 grid_predictions = grid_search.predict(val_X)
 grid_mae = mean_absolute_error(grid_predictions, val_y)
 print(grid_mae )<load_from_csv>
df_all = df_all.merge(df_external_3_all,how='left',on='pred_breed')
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<count_values>
df_all_temp = df_all[['Temperment']].copy()
df_all_temp['Temperment'] = df_all_temp['Temperment'].apply(lambda x: [e.strip(',')for e in str(x ).split() ])
 df_all_temp = df_all_temp.drop('Temperment', 1 ).join(df_all_temp.Temperment.str.join('|' ).str.get_dummies())
 df_all_temp = df_all_temp.add_prefix('Temperment_')
 df_all = pd.concat(( df_all,df_all_temp),axis=1)
 del df_all['Temperment']
test_data['Title'] = test_data['Name'].map(lambda x:x.split(',')[1].split('.')[0].strip())
 test_data['Title']=test_data['Title'].map(TitleDict)
 test_data['Title'].value_counts()<feature_engineering>
rescuer_count = df_all.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
df_all = df_all.merge(rescuer_count, how='left', on='RescuerID' )
test_data, model = file_missing(test_data )<categorify>
groupby_columns = [['is_dog','Breed1'],
['State'],
['is_dog','Gender','Age_bins'],
['is_dog','MaturitySize'],
['is_dog','FurLength'],
['is_dog','Breed1','Vaccinated', 'Dewormed', 'Sterilized', 'Health',]
]
aggregate_columns = ['Age',
'Name_len',
'Color1',
'Color2',
'Color3',
'MaturitySize',
'FurLength',
'Vaccinated',
'Dewormed',
'Sterilized',
'Health',
'Quantity',
'Fee',
'VideoAmt',
'PhotoAmt',
'state_gdp',
'state_population',
'len_words_Description',
'len_isupper_Description_to_characters',
'len_islower_Description_to_characters',
'len_punctuation_Description_to_characters',
'len_letters_Description_to_characters',
'len_characters_Description_to_words',
'RescuerID_COUNT',
]
test_data['Familynum']=test_data['Parch']+test_data['SibSp']+1
test_data['Familysize'] = test_data['Familynum'].map(Family_size )<groupby>
for _groupby in groupby_columns:
_aggregate = [e for e in aggregate_columns if e not in _groupby]
_aggregate = {e:['mean','std','min','max'] for e in _aggregate}
gr = df_all.groupby(_groupby ).agg(_aggregate)
 gr.columns = ['{}_{}_{}'.format("_".join(_groupby),e[0],e[1])for e in gr.columns]
all_columns.extend(gr.columns)
 gr = gr.reset_index()
df_all = pd.merge(df_all,gr,on=_groupby,how='left' )
Ticket_Count = test_data.groupby('Ticket', as_index = False)['PassengerId'].count()
Ticket_Count_0 = Ticket_Count[Ticket_Count.PassengerId == 1]['Ticket']
test_data['GroupTicket'] = np.where(test_data.Ticket.isin(Ticket_Count_0), 0, 1 )<categorify>
other_columns =[
'PetID',
'AdoptionSpeed',
'Age',
'Color1',
'Color2',
'Color3',
'Dewormed',
'Fee',
'FurLength',
'Gender',
'Health',
'MaturitySize',
'PhotoAmt',
'Quantity',
'Sterilized',
'Vaccinated',
'VideoAmt',]
test_data['Fare_Category'] = test_data['Fare'].map(fare_category )<normalization>
categorical_columns = ['Gender', 'Color1', 'Color2', 'Color3','Vaccinated', 'Dewormed',
'Sterilized', 'State','Breed1_le', 'RescuerID']
freq_categorical_columns = []
for c in tqdm(categorical_columns):
df_all[c+'_freq'] = df_all[c].map(df_all.groupby(c ).size() / df_all.shape[0])
 freq_categorical_columns.append(c+'_freq')
 indexer = pd.factorize(df_all[c], sort=True)[1]
df_all[c] = indexer.get_indexer(df_all[c] )
test_features_dummies = pd.get_dummies(test_data[get_features])
 test_features_dummies['Age_scaled'] = scaler.fit_transform(test_features_dummies['Age'].values.reshape(-1,1), age_scale)
 test_features_dummies['Fare_scaled'] = scaler.fit_transform(test_features_dummies['Fare'].values.reshape(-1,1), fare_scale)
 test_features_dummies.head()<save_to_csv>
del train
del test
gc.collect()
test = test_features_dummies.filter(regex='Age_.*|Fare_.*|Cabin_.*|Embarked_.*|Sex_.*|Pclass_.*|Title_.*|Familysize|GroupTicket|Fare_Category')
 predictions = grid_search.predict(test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions},)
 output = output.round(0 ).astype(int)
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
 print(output )<load_from_csv>
excl_columns = list(df_all.dtypes[df_all.dtypes == 'object'].index.values )
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv")
<set_options>
df_all = df_all.loc[:, ~df_all.columns.isin(excl_columns)]
%matplotlib inline
sns.set()<feature_engineering>
train = df_all.loc[df_all.is_train==1, ~df_all.columns.duplicated() ].drop('is_train', 1)
 test = df_all.loc[df_all.is_train==0, ~df_all.columns.duplicated() ].drop('is_train', 1 )
train_test_dataset = [train, test]
for dataset in train_test_dataset:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )<categorify>
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_dataset:
dataset['Title'] = dataset['Title'].map(title_mapping )<count_values>
feat_cols = list(train.columns[ 
(train.columns.values != 'AdoptionSpeed')
 ].values)
 y_col = 'AdoptionSpeed'
train['Title'].value_counts()<drop_column>
X_train = train[feat_cols]
X_test = test[feat_cols]
y_train = train[y_col]
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True)
 train.drop('Embarked', axis=1, inplace=True)
 test.drop('Embarked', axis=1, inplace=True)
 train.drop('Cabin', axis=1, inplace=True)
 test.drop('Cabin', axis=1, inplace=True)
 train.drop('SibSp', axis=1, inplace=True)
 test.drop('SibSp', axis=1, inplace=True)
 train.drop('Parch', axis=1, inplace=True)
 test.drop('Parch', axis=1, inplace=True)
 train.drop('Ticket', axis=1, inplace=True)
 test.drop('Ticket', axis=1, inplace=True )<categorify>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _loss(self, coef, X, y, idx):
X_p = np.array([to_bins(pred, coef)for pred in X])
 ll = -cohen_kappa_score(y, X_p, weights='quadratic')
 return ll
def fit(self, X, y):
coef = [1.5, 2.0, 2.5, 3.0]
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [(1, 2),(1.8, 2.5),(2, 2.8),(2.5, 3.0)]
for it1 in range(10):
for idx in range(4):
a, b = ab_start[idx]
coef[idx] = a
la = self._loss(coef, X, y, idx)
 coef[idx] = b
lb = self._loss(coef, X, y, idx)
 for it in range(20):
if la > lb:
a = b -(b - a)* golden1
coef[idx] = a
la = self._loss(coef, X, y, idx)
 else:
b = b -(b - a)* golden2
coef[idx] = b
lb = self._loss(coef, X, y, idx)
 self.coef_ = {'x': coef}
def predict(self, X, coef):
X_p = np.array([to_bins(pred, coef)for pred in X])
 return X_p
def coefficients(self):
return self.coef_['x']
def to_bins(x, borders):
for i in range(len(borders)) :
if x <= borders[i]:
return i
return len(borders )
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_dataset:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )<categorify>
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold
from bayes_opt import BayesianOptimization
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )<feature_engineering>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'subsample': 0.8,
'tree_method': 'gpu_hist',
'device': 'gpu',
'colsample_bytree': 0.8152926875727884,
'eta': 0.03402795645208036,
'gamma': 0.9209917577458885,
'max_depth': 6,
'silent': 1,
}
for dataset in train_test_dataset:
dataset.loc[ dataset['Age'] <= 12, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 12)&(dataset['Age'] <= 19), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 19)&(dataset['Age'] <= 30), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 30)&(dataset['Age'] <= 45), 'Age'] = 3,
dataset.loc[(dataset['Age'] > 45)&(dataset['Age'] <= 65), 'Age'] = 4,
dataset.loc[ dataset['Age'] > 65, 'Age'] = 5<categorify>
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
oof_coefficients = []
i = 0
for train_idx, valid_idx in kf.split(X_train, y_train.values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = y_train.values[train_idx]
y_val = y_train.values[valid_idx]
d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 optR = OptimizedRounder()
optR.fit(valid_pred, y_val)
 coefficients = optR.coefficients()
oof_coefficients.append(coefficients)
 pred_val_y_k = optR.predict(valid_pred, coefficients)
 qwk = cohen_kappa_score(y_val, pred_val_y_k, weights='quadratic')
 print("QWK = ", qwk)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test, oof_coefficients
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True )<feature_engineering>
model, oof_train, oof_test, oof_coefficients = run_xgb(xgb_params, X_train, X_test )
for dataset in train_test_dataset:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3 
<prepare_x_and_y>
optR = OptimizedRounder()
mean_coeffs = np.mean(np.array(oof_coefficients), axis=0)
 valid_pred = optR.predict(oof_train, mean_coeffs)
 qwk = cohen_kappa_score(y_train.values, valid_pred, weights='quadratic')
 print("QWK = ", qwk )
train = train.drop(['PassengerId'], axis=1)
 train_data = train.drop('Survived', axis=1)
 target = train['Survived']<import_modules>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
import numpy as np<choose_model_class>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )<compute_train_metric>
params = {
'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 60,
'max_depth': 11,
'learning_rate': 0.0076,
'bagging_fraction': 0.9208175607947006,
'feature_fraction': 0.7512774855381017,
'learning_rate': 0.01,
'max_depth': 17,
'min_child_weight': 49.90252232871143,
'min_split_gain': 0.021286271431087116,
'num_leaves': 76,
'reg_alpha': 0.043051815788046566,
'reg_lambda': 0.07902389530425666,
'subsample': 0.9253685895179343,
'verbosity': -1,
'data_random_seed': 17
}
early_stop = 500
verbose_eval = 1000
num_rounds = 10000
n_splits = 5
classifier = KNeighborsClassifier(n_neighbors = 15)
 scoring = 'accuracy'
score = cross_val_score(classifier, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score)
 round(np.mean(score)*100, 2 )<compute_train_metric>
oof_train_lgb = np.zeros(( X_train.shape[0]))
oof_test_lgb = np.zeros(( X_test.shape[0], n_splits))
qwk_scores = []
oof_coefficients_lgb = []
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1337)
 for i,(f_ind, outf_ind)in enumerate(kf.split(X_train, y_train.values)) :
X_tr = X_train.loc[f_ind].copy()
X_val = X_train.loc[outf_ind].copy()
y_tr, y_val = y_train[f_ind], y_train[outf_ind]
print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(val_pred, y_val)
 coefficients = optR.coefficients()
oof_coefficients_lgb.append(coefficients)
 pred_val_y_k = optR.predict(val_pred, coefficients)
 print("Valid Counts = ", Counter(y_val))
print("Predicted Counts = ", Counter(pred_val_y_k))
print("Coefficients = ", coefficients)
 qwk = cohen_kappa_score(y_val, pred_val_y_k, weights='quadratic')
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train_lgb[outf_ind] = val_pred
oof_test_lgb[:, i] = test_pred
print('{} cv mean QWK score : {}, coeffs {}'.format('LGBM', np.mean(qwk_scores)
 ,np.mean(np.array(oof_coefficients_lgb), axis=0)) )
classifier = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(classifier, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 round(np.mean(score)*100, 2 )<compute_train_metric>
optR = OptimizedRounder()
mean_coeffs = np.mean(np.array(oof_coefficients_lgb), axis=0)
 valid_pred = optR.predict(oof_train_lgb, mean_coeffs)
 qwk = cohen_kappa_score(y_train.values, valid_pred, weights='quadratic')
 print("QWK = ", qwk )
classifier = RandomForestClassifier(n_estimators=100)
 scoring = 'accuracy'
score = cross_val_score(classifier, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score)
 round(np.mean(score)*100, 2 )<train_model>
mean_coeff_m =(np.array(oof_coefficients_lgb)+ np.array(oof_coefficients)) / 2
classifier = RandomForestClassifier()
classifier.fit(train_data, target)
 test_data = test.drop("PassengerId", axis=1 ).copy()
prediction = classifier.predict(test_data )<save_to_csv>
optR = OptimizedRounder()
mean_coeffs = np.mean(np.array(mean_coeff_m), axis=0)
 valid_pred = optR.predict(( oof_train_lgb + oof_train)/ 2, mean_coeffs)
 qwk = cohen_kappa_score(y_train.values, valid_pred, weights='quadratic')
 print("QWK = ", qwk )
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
train_predictions = optR.predict(( oof_train_lgb + oof_train)/ 2, mean_coeffs ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(( oof_test_lgb.mean(axis=1)+ oof_test.mean(axis=1)) / 2, mean_coeffs ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
<categorify><EOS> 
test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv', usecols=['PetID'])
 submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()