Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<save_to_csv>
pd.options.display.max_columns = 128
torch.multiprocessing.set_start_method("spawn" )
testData['Survived'] = testData.grouping.apply(lambda x: groupDict[x])
 testData[['PassengerId','Survived']].to_csv("final_submission.csv",index = False)
 print("That's all!" )<import_modules>
def jopen(path):
with open(path, "r", encoding="utf-8")as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(path):
file = jopen(path)
 language: str = file["language"]
sentiment: list = file["documentSentiment"]
entities: list = [x["name"] for x in file["entities"]]
entity = " ".join(entities)
 sentence_sentiment: list = [x["sentiment"] for x in file["sentences"]]
magnitude: np.ndarray = np.array(
 [x["magnitude"] for x in sentence_sentiment])
 score: np.ndarray = np.array([x["score"] for x in sentence_sentiment])
 return_js = {
"magnitude_sum": magnitude.sum() ,
"magnitude_mean": magnitude.mean() ,
"magnitude_var": magnitude.var() ,
"score_sum": score.sum() ,
"score_mean": score.mean() ,
"score_var": score.var() ,
"language": language,
"entity": entity,
"document_magnitude": sentiment["magnitude"],
"document_score": sentiment["score"]
}
return return_js
def parse_metadata(path):
file: dict = jopen(path)
 file_keys = list(file.keys())
 if "labelAnnotations" in file_keys:
file_annots = file["labelAnnotations"]
file_mean_score = np.asarray([x["score"] for x in file_annots] ).mean()
file_desc = " ".join([x["description"] for x in file_annots])
 else:
file_mean_score = np.nan
file_desc = ""
file_colors: list = file["imagePropertiesAnnotation"]["dominantColors"][
"colors"]
file_crops: list = file["cropHintsAnnotation"]["cropHints"]
color_score = np.asarray([x["score"] for x in file_colors] ).mean()
pixel_frac = np.asarray([x["pixelFraction"] for x in file_colors] ).mean()
crop_conf = np.asarray([x["confidence"] for x in file_crops] ).mean()
if "importanceFraction" in file_crops[0].keys() :
crop_importance = np.asarray(
 [x["importanceFraction"] for x in file_crops] ).mean()
else:
crop_importance = np.nan
metadata = {
"annot_score": file_mean_score,
"color_score": color_score,
"pixel_frac": pixel_frac,
"crop_conf": crop_conf,
"crop_importance": crop_importance,
"desc": file_desc
}
return metadata
def additinal_features_per_id(pet_id, sentiment_path: Path, meta_path: Path):
sentiment_path = sentiment_path / f"{pet_id}.json"
try:
sentiment = parse_sentiment_file(sentiment_path)
 sentiment["pet_id"] = pet_id
except FileNotFoundError:
sentiment = {}
meta_files = sorted(meta_path.glob(f"{pet_id}*.json"))
metadata_list = []
if len(meta_files)> 0:
for f in meta_files:
metadata = parse_metadata(f)
 metadata["pet_id"] = pet_id
metadata_list.append(metadata)
 return sentiment, metadata_list
def load_additional_features(ped_ids: list, sentiment_path: Path,
meta_path: Path):
features = Parallel(
 n_jobs=-1, verbose=1 )(
 delayed(additinal_features_per_id )(i, sentiment_path, meta_path)
 for i in ped_ids)
 sentiments = [x[0] for x in features if len(x[0])> 0]
metadatas = [x[1] for x in features if len(x[1])> 0]
sentiment_keys = sentiments[0].keys()
metadata_keys = metadatas[0][0].keys()
sentiment_dict = {}
metadata_dict = {}
for key in sentiment_keys:
sentiment_dict[key] = [x[key] for x in sentiments]
for key in metadata_keys:
meta_list = []
for meta_per_pid in metadatas:
meta_list += [meta[key] for meta in meta_per_pid]
metadata_dict[key] = meta_list
sentiment_df = pd.DataFrame(sentiment_dict)
 metadata_df = pd.DataFrame(metadata_dict)
 return sentiment_df, metadata_df
def aggregate_metadata(metadata_df: pd.DataFrame,
aggregates=["sum", "mean", "var"]):
meta_desc: pd.DataFrame = metadata_df.groupby(["pet_id"])["desc"].unique()
meta_desc = meta_desc.reset_index()
meta_desc["desc"] = meta_desc["desc"].apply(lambda x: " ".join(x))
meta_gr: pd.DataFrame = metadata_df.drop(["desc"], axis=1)
 for i in meta_gr.columns:
if "pet_id" not in i:
meta_gr[i] = meta_gr[i].astype(float)
 meta_gr = meta_gr.groupby(["pet_id"] ).agg(aggregates)
 meta_gr.columns = pd.Index(
 [f"{c[0]}_{c[1].upper() }" for c in meta_gr.columns.tolist() ])
 meta_gr = meta_gr.reset_index()
return meta_gr, meta_desc
def aggregate_sentiment(sentiment_df: pd.DataFrame, aggregates=["sum"]):
sentiment_desc: pd.DataFrame = sentiment_df.groupby(
 ["pet_id"])["entity"].unique()
sentiment_desc = sentiment_desc.reset_index()
sentiment_desc["entity"] = sentiment_desc["entity"].apply(
 lambda x: " ".join(x))
sentiment_lang = sentiment_df.groupby(
 ["pet_id"])["language"].unique()
sentiment_lang = sentiment_lang.reset_index()
sentiment_lang["language"] = sentiment_lang["language"].apply(
 lambda x: " ".join(x))
sentiment_desc = sentiment_desc.merge(
 sentiment_lang, how="left", on="pet_id")
 sentiment_gr: pd.DataFrame = sentiment_df.drop(["entity", "language"],
axis=1)
 for i in sentiment_gr.columns:
if "pet_id" not in i:
sentiment_gr[i] = sentiment_gr[i].astype(float)
 sentiment_gr = sentiment_gr.groupby(["pet_id"] ).agg(aggregates)
 sentiment_gr.columns = pd.Index(
 [f"{c[0]}" for c in sentiment_gr.columns.tolist() ])
 sentiment_gr = sentiment_gr.reset_index()
return sentiment_gr, sentiment_desc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedKFold
from transformers import *
import tokenizers
import tensorflow as tf
import tensorflow.keras.backend as K<import_modules>
input_dir = Path(".. /input/petfinder-adoption-prediction/")
 train = pd.read_csv(input_dir / "train/train.csv")
 test = pd.read_csv(input_dir / "test/test.csv")
 sample_submission = pd.read_csv(input_dir / "test/sample_submission.csv" )
<set_options>
sp_train = input_dir / Path("train_sentiment/")
 mp_train = input_dir / Path("train_metadata/")
 sp_test = input_dir / Path("test_sentiment/")
 mp_test = input_dir / Path("test_metadata/" )
tw_pal = ['
tw_pal_1 = ['
sns.set_style("whitegrid" )<load_from_csv>
train_sentiment_df, train_metadata_df = load_additional_features(
 train_pet_ids, sp_train, mp_train)
 test_sentiment_df, test_metadata_df = load_additional_features(
 test_pet_ids, sp_test, mp_test )
train = pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv" )<count_missing_values>
train_meta_gr, train_meta_desc = aggregate_metadata(train_metadata_df)
 test_meta_gr, test_meta_desc = aggregate_metadata(test_metadata_df)
 train_sentiment_gr, train_sentiment_desc = \
aggregate_sentiment(train_sentiment_df)
 test_sentiment_gr, test_sentiment_desc = \
aggregate_sentiment(test_sentiment_df )
print(train.isna().sum())
 print("")
 print(test.isna().sum())
 print("")
 print(sample.isna().sum() )<filter>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_desc, how="left", left_on="PetID", right_on = "pet_id")
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_desc, how="left", left_on="PetID", right_on = "pet_id" )
train[train.isna().any(axis=1)]<count_missing_values>
train_proc.drop(train_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 test_proc.drop(test_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 train_proc.head()
train = train.dropna()
print(train.isna().sum().sum() )<count_unique_values>
train_proc.language.fillna("", inplace=True)
 test_proc.language.fillna("", inplace=True)
 langs = train_proc.language.unique()
encode_dict = {k: i for i, k in enumerate(langs)}
train_proc.language = train_proc.language.map(encode_dict)
 test_proc.language = test_proc.language.map(encode_dict )
print(train.nunique())
 print('')
 print(test.nunique())
 print('')
 print(sample.nunique() )<feature_engineering>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 X_temp = X.copy()
text_columns = [
"Description",
"entity",
"desc"]
categorical_columns = [
"Type", "Breed1", "Breed2", "Gender",
"Color1", "Color2", "Color3", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized",
"State", "language"
]
drop_columns = [
"PetID", "Name", "RescuerID"
]
train['no_words'] = train['text'].str.split().apply(len)
 train['no_words_sel_text'] = train['selected_text'].str.split().apply(len)
 col = ['no_words', 'no_words_sel_text']
col_names = ['Mean no.of words in text', 'Mean no.of words in selected text']
plot_bar(col, col_names )<string_transform>
rescuer_count = X.groupby(["RescuerID"])["PetID"].count().reset_index()
rescuer_count.columns = ["RescuerID", "RescuerID_COUNT"]
X_temp = X_temp.merge(rescuer_count, how="left", on="RescuerID" )
def hash_count(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('
def mention_count(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('@')])
 def web_add(tweet):
w = tweet.split()
return len([word for word in w if word.startswith('http')] )<import_modules>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna("none" )
print('TF version',tf.__version__ )<load_from_csv>
X_temp["len_description"] = X_text["Description"].map(len)
 X_temp["len_meta_desc"] = X_text["desc"].map(len)
 X_temp["len_entity"] = X_text["entity"].map(len )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
n_components = 16
text_features = []
for i in X_text.columns:
print(f"generating features from: {i}")
 tfv = TfidfVectorizer(
 min_df=2,
strip_accents="unicode",
analyzer="word",
token_pattern=r"(?u)\b\w+\b",
ngram_range=(1, 3),
use_idf=1,
smooth_idf=1,
sublinear_tf=1)
 svd = TruncatedSVD(
 n_components=n_components,
random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix("Tfidf_{}_".format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp.drop(i, axis=1, inplace=True )
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
train_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/train_images/*.jpg"))
test_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/test_images/*.jpg"))
train_df_imgs = pd.DataFrame(train_image_files)
 test_df_imgs = pd.DataFrame(test_image_files)
 train_df_imgs.columns = ["image_file_name"]
test_df_imgs.columns = ["image_file_name"]
train_imgs_pets = train_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 test_imgs_pets = test_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def get_size(filename):
st = os.stat(filename)
 return st.st_size
def get_dimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs["image_size"] = train_df_imgs["image_file_name"].apply(get_size)
 test_df_imgs["image_size"] = test_df_imgs["image_file_name"].apply(get_size)
 train_df_imgs["temp_size"] = train_df_imgs["image_file_name"].apply(get_dimensions)
 test_df_imgs["temp_size"] = test_df_imgs["image_file_name"].apply(get_dimensions)
 train_df_imgs["width"] = train_df_imgs["temp_size"].apply(lambda x: x[0])
 test_df_imgs["width"] = test_df_imgs["temp_size"].apply(lambda x: x[0])
 train_df_imgs["height"] = train_df_imgs["temp_size"].apply(lambda x: x[1])
 test_df_imgs["height"] = test_df_imgs["temp_size"].apply(lambda x: x[1])
 train_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 test_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 aggs = {
"image_size": ["sum", "mean", "var"],
"width": ["sum", "mean", "var"],
"height": ["sum", "mean", "var"]
}
agg_train_imgs = train_df_imgs.groupby("PetID" ).agg(aggs)
 new_columns = [
k + "_" + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs = test_df_imgs.groupby("PetID" ).agg(aggs)
 agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True)
 X_temp = X_temp.merge(agg_imgs, how="left", on="PetID")
 X_temp.head()
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
X_temp.drop(drop_columns, axis=1, inplace=True)
 X_temp.head()
<categorify>
for c in new_columns:
X_temp.loc[:, c] = X_temp[c].map(lambda x: np.log1p(x))
X_temp.head()
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test.drop(["AdoptionSpeed"], axis=1, inplace=True)
 assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1)
 X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
n_splits = 5<predict_on_test>
X_train_non_null.Fee = X_train_non_null.Fee.map(lambda x: np.log1p(x))
X_test_non_null.Fee = X_test_non_null.Fee.map(lambda x: np.log1p(x))
X_train_non_null.RescuerID_COUNT = X_train_non_null.RescuerID_COUNT.map(lambda x: np.log(x))
X_test_non_null.RescuerID_COUNT = X_test_non_null.RescuerID_COUNT.map(lambda x: np.log(x))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(10):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
cat_features = ["Type", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3",
"MaturitySize", "FurLength", "Vaccinated", "Dewormed", "Sterilized",
"Health", "Quantity", "State", "language"]
X_train_cat = X_train_non_null.loc[:, cat_features]
X_test_cat = X_test_non_null.loc[:, cat_features]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
y = X_train_non_null.AdoptionSpeed
X_train_cat_y = X_train_cat.copy()
X_train_cat_y["AdoptionSpeed"] = y
kfold = GroupKFold(n_splits=5)
 X_train_cat_encoded = np.zeros(( X_train_cat.shape[0], len(cat_features)* 2))
X_test_cat_encoded = np.zeros(( X_test_cat.shape[0], len(cat_features)* 2))
for trn_idx, val_idx in kfold.split(X_train_cat, y, train_pet_ids):
X_trn, X_val = X_train_cat_y.loc[trn_idx, :], X_train_cat_y.loc[val_idx, :]
for j, c in enumerate(cat_features):
X_trn_enc = X_trn.groupby(c ).agg({
"AdoptionSpeed": ["mean", "std"]
})
 cte_columns = [f"{c}_{x}" for x in ["mean", "std"]]
X_trn_enc.columns = cte_columns
X_temp = np.zeros(( X_test_cat.shape[0], 2))
X_temp_df = pd.DataFrame(data=X_temp, columns=cte_columns)
 for x in X_trn_enc.columns:
X_val[x] = X_val[c].map(X_trn_enc[x])
 X_temp_df[x] = X_test_cat[c].map(X_trn_enc[x] ).reset_index(drop=True)
 X_train_cat_encoded[val_idx, 2 * j:2 *(j + 1)] = X_val[X_trn_enc.columns].values
X_test_cat_encoded[:, 2 * j: 2 *(j + 1)] += X_temp_df.values / 5
test_df['selected_text'] = all
test_df[['textID','selected_text']].to_csv('submission.csv',index=False )<import_modules>
columns = [f"{c}_{x}" for c in cat_features for x in ["mean", "std"]]
X_train_cat_encoded_df = pd.DataFrame(data=X_train_cat_encoded, columns=columns)
 X_test_cat_encoded_df = pd.DataFrame(data=X_test_cat_encoded, columns=columns)
 X_test_cat_encoded_df.head()
print('TF version',tf.__version__ )<define_variables>
X_train_num = X_train_non_null.drop(cat_features + ["AdoptionSpeed"], axis=1)
 X_test_num = X_test_non_null.drop(cat_features, axis=1)
 target = X_train_non_null["AdoptionSpeed"]
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
X_train_num = pd.concat([X_train_num, X_train_cat_encoded_df], axis=1)
 X_test_cat_encoded_df.index = X_test_num.index
X_test_num = pd.concat([X_test_num, X_test_cat_encoded_df], axis=1 )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
X_all_num = pd.concat([X_train_num, X_test_num], axis=0)
 ss = StandardScaler()
ss.fit(X_all_num)
 X_train_ss = ss.transform(X_train_num)
 X_test_ss = ss.transform(X_test_num )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
X_train_ss[np.isnan(X_train_ss)] = 0.0
X_test_ss[np.isnan(X_test_ss)] = 0.0
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
cat_cat = pd.concat([X_train_cat, X_test_cat])
 n_breed1 = cat_cat["Breed1"].nunique()
n_breed2 = cat_cat["Breed2"].nunique()
n_langs = cat_cat["language"].nunique()
n_color1 = cat_cat["Color1"].nunique()
n_color2 = cat_cat["Color2"].nunique()
n_color3 = cat_cat["Color3"].nunique()
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
for c in X_train_cat.columns:
le = LabelEncoder()
le.fit(cat_cat[c])
 X_train_cat[c] = le.transform(X_train_cat[c])
 X_test_cat[c] = le.transform(X_test_cat[c] )
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y, initial_coef=[]):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = initial_coef
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
assert len(rater_a)== len(rater_b)
 if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_rating = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_rating)] for j in range(num_rating)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):
if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a = y
rater_b = y_pred
min_rating = None
max_rating = None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert len(rater_a)== len(rater_b)
 min_rating = min(min(rater_a), min(rater_b))
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b, min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(
 hist_rater_a[i] * hist_rater_b[j])/ num_scored_items
d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
class PetDataset(data.Dataset):
def __init__(self,
pet_ids,
cat_features,
num_features,
labels,
root_dir,
subset=False,
transform=None):
self.pet_ids = pet_ids
self.cat_features = cat_features
self.num_features = num_features
if labels is not None:
self.labels = labels
else:
self.labels = None
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.pet_ids)
 def __getitem__(self, idx):
img_name = f"{self.pet_ids[idx]}-1.jpg"
fullname = self.root_dir / Path(img_name)
 try:
image = Image.open(fullname ).convert("RGB")
 except FileNotFoundError:
image = np.zeros(( 3, 224, 224), dtype=np.uint8 ).transpose(1, 2, 0)
 image = Image.fromarray(np.uint8(image))
cat_feature = self.cat_features[idx]
num_feature = self.num_features[idx]
if self.transform:
image = self.transform(image)
 if self.labels is not None:
label = self.labels[idx]
return [image, cat_feature, num_feature, label]
else:
return [image, cat_feature, num_feature]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
normalize = transforms.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225] 
)
 ds_trans = transforms.Compose([transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor() ,
normalize] )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def seed_torch(seed=1029):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
def run_xgb(params,
X,
y,
X_test,
resc,
n_splits=10,
num_rounds=60000,
early_stop=500,
verbose_eval=1000):
fold = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
for i,(trn_index, val_index)in enumerate(fold.split(X, y, train_pet_ids)) :
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
d_train = xgb.DMatrix(
 data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(
 data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, "train"),(d_valid, "valid")]
model = xgb.train(
 params=params,
dtrain=d_train,
num_boost_round=num_rounds,
evals=watchlist,
early_stopping_rounds=early_stop,
verbose_eval=verbose_eval)
 valid_pred = model.predict(
 xgb.DMatrix(X_val, feature_names=X_val.columns),
ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(
 xgb.DMatrix(X_test, feature_names=X_test.columns),
ntree_limit=model.best_ntree_limit)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test
class Trainer:
def __init__(self,
model,
resc,
n_splits=5,
seed=42,
device="cuda:0",
train_batch=16,
val_batch=32,
kwargs={}):
self.model = model
self.resc = resc
self.n_splits = n_splits
self.seed = seed
self.device = device
self.train_batch = train_batch
self.val_batch = val_batch
self.kwargs = kwargs
self.fold = GroupKFold(
 n_splits=n_splits)
 self.best_score = None
self.tag = dt.now().strftime("%Y-%m-%d-%H-%M-%S")
 self.loss_fn = nn.MSELoss(reduction="mean" ).to(self.device)
 path = Path(f"bin/{self.tag}")
 path.mkdir(exist_ok=True, parents=True)
 self.path = path
def fit(self, pet_ids, cat_feats, num_feats, answer, n_epochs=30):
self.train_preds = np.zeros(( train.shape[0]))
answer = answer.values
cat_feats = cat_feats.values
for i,(trn_idx, val_idx)in enumerate(self.fold.split(pet_ids, answer, self.resc)) :
self.fold_num = i
print(f"Fold: {i+1}")
 pid_train, pid_val = pet_ids[trn_idx], pet_ids[val_idx]
cat_train, cat_val = cat_feats[trn_idx], cat_feats[val_idx]
num_train, num_val = num_feats[trn_idx], num_feats[val_idx]
y_train, y_val = answer[trn_idx] / 4, answer[val_idx] / 4
valid_preds = self._fit(pid_train,
cat_train,
num_train,
y_train,
n_epochs,
pid_val,
cat_val,
num_val,
y_val 
)
 self.train_preds[val_idx] = valid_preds
def _fit(self, pid, cat, num, y, n_epochs, pid_val, cat_val, num_val, y_val):
seed_torch(self.seed)
 cat_tensor = torch.tensor(cat, dtype=torch.long ).to(self.device)
 num_tensor = torch.tensor(num, dtype=torch.float32 ).to(self.device)
 y_tensor = torch.tensor(y[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 train = PetDataset(pid,
cat_tensor,
num_tensor,
y_tensor,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 train_loader = data.DataLoader(train,
batch_size=self.train_batch, shuffle=True)
 cat_eval = torch.tensor(cat_val, dtype=torch.long ).to(self.device)
 num_eval = torch.tensor(num_val, dtype=torch.float32 ).to(self.device)
 y_eval = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 eval_ = PetDataset(pid_val,
cat_eval,
num_eval,
y_eval,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 eval_loader = data.DataLoader(eval_,
batch_size=self.val_batch, shuffle=False)
 model = self.model(**self.kwargs)
 model = model.to(self.device)
 optimizer = optim.Adam(model.parameters())
 best_score = np.inf
for epoch in range(n_epochs):
model.train()
avg_loss = 0.
 for i_batch, c_batch, n_batch, y_batch in tqdm(train_loader):
i_batch = i_batch.to(self.device)
 y_pred = model(i_batch, c_batch, n_batch)
 loss = self.loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"epoch {epoch+1}/{n_epochs}")
 print(f"avg_loss: {avg_loss:.4f}")
 print(f"avg_val_loss: {avg_val_loss:.4f}")
 if best_score > avg_val_loss:
torch.save(model.state_dict() ,
self.path / f"best{self.fold_num}.pt")
 print(f"Save model on epoch {epoch + 1}")
 best_score = avg_val_loss
model.load_state_dict(torch.load(self.path / f"best{self.fold_num}.pt"))
valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"Validation loss: {avg_val_loss}")
 return valid_preds
def _val(self, loader, model):
model.eval()
valid_preds = np.zeros(loader.dataset.cat_features.size(0))
avg_val_loss = 0.
 for i,(i_batch, c_batch, n_batch, y_batch)in enumerate(loader):
with torch.no_grad() :
i_batch = i_batch.to(self.device)
 y_pred = model(i_batch, c_batch, n_batch ).detach()
avg_val_loss += self.loss_fn(y_pred,
y_batch ).item() / len(loader)
 valid_preds[i * self.val_batch:(i + 1)* self.val_batch] = y_pred.cpu().numpy() [:, 0]
return valid_preds, avg_val_loss
import tokenizers
import torch 
<define_variables>
class NeuralNet(nn.Module):
def __init__(self, path, emb_dims, num_dims, img_linear, linear_size):
super(NeuralNet, self ).__init__()
self.densenet121 = models.densenet121()
self.densenet121.load_state_dict(torch.load(path))
self.densenet121.classifier = nn.Linear(1024, img_linear)
 dense = nn.Sequential(*list(self.densenet121.children())[:-1])
 for param in dense.parameters() :
param.requires_grad = False
self.embeddings = nn.ModuleList(
 [nn.Embedding(x, y)for x, y in emb_dims])
 n_emb_out = sum([y for x, y in emb_dims])
 self.fc1 = nn.Linear(img_linear + n_emb_out + num_dims, linear_size)
 self.bn1 = nn.BatchNorm1d(linear_size)
 self.fc2 = nn.Linear(linear_size, 1)
 self.drop = nn.Dropout(0.2)
 self.relu = nn.ReLU()
def forward(self, i, c, n):
img_feats = self.densenet121(i)
 emb = [
emb_layer(c[:, j])for j, emb_layer in enumerate(self.embeddings)
 ]
emb = torch.cat(emb, 1)
 data = torch.cat([img_feats, emb, n], 1)
 out = self.relu(self.fc1(data))
out = self.bn1(out)
 out = self.drop(out)
 out = self.fc2(out)
 return out
MAX_LEN = 128
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE=32
EPOCHS=10
BERT_PATH='bert-base-uncased'
TRAINING_FILE = '/kaggle/input/tweet-sentiment-extraction/train.csv'
SAMPLE_FILE ='/kaggle/input/tweet-sentiment-extraction/sample_submission.csv'
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 16
EPOCHS = 5
BERT_PATH = ".. /input/bert-base-uncased/"
MODEL_PATH = "model.bin"
TOKENIZER = tokenizers.BertWordPieceTokenizer(
 f"{BERT_PATH}/vocab.txt",
lowercase=True 
)
<import_modules>
emb_dims = [(2, 1),(n_breed1, 3),(n_breed2, 3),(3, 1),(n_color1, 1), 
(n_color2, 1),(n_color3, 1),(4, 1),(3, 1),(3, 1), 
(3, 1),(3, 1),(3, 1),(19, 1),(14, 1),(n_langs, 1)]
num_dims = X_test_ss.shape[1]
trainer = Trainer(
 NeuralNet,
resc=train_pet_ids,
n_splits=4,
train_batch=128,
val_batch=128,
kwargs={
"path": "densenet121.pth",
"emb_dims": emb_dims,
"num_dims": num_dims,
"img_linear": 48,
"linear_size": 120
} )
def param_getter(m): return m.parameters()
def listify(o):
if o is None : return []
if isinstance(o,list): return o
if isinstance(o,str): return [o]
if isinstance(o,Iterable): return list(o)
 return [o]
class DataBunch() :
def __init__(self, train_dl, valid_dl, c=None):
self.train_dl,self.valid_dl,self.c = train_dl,valid_dl,c
@property
def train_ds(self): return self.train_dl.dataset
@property
def valid_ds(self): return self.valid_dl.dataset
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()
class Callback() :
_order=0
def set_runner(self, run): self.run=run
def __getattr__(self, k): return getattr(self.run, k)
 @property
def name(self):
name = re.sub(r'Callback$', '', self.__class__.__name__)
 return camel2snake(name or 'callback')
 def __call__(self, cb_name):
f = getattr(self, cb_name, None)
 if f and f() : return True
return False
class AvgStatsCallback(Callback):
def __init__(self, metrics):
self.train_stats,self.valid_stats = AvgStats(metrics,True),AvgStats(metrics,False)
 def begin_epoch(self):
self.train_stats.reset()
self.valid_stats.reset()
def after_loss(self):
stats = self.valid_stats if not self.in_train else self.train_stats
with torch.no_grad() : stats.accumulate(self.run)
 def after_epoch(self):
print(self.train_stats)
 print(self.valid_stats)
 class Recorder(Callback):
def begin_fit(self):
self.lrs = [[] for _ in self.opt.param_groups]
self.losses = []
def after_batch(self):
if not self.in_train: return
for pg,lr in zip(self.opt.param_groups,self.lrs): lr.append(pg['lr'])
 self.losses.append(self.loss.detach().cpu())
 def plot_lr(self, pgid=-1): plt.plot(self.lrs[pgid])
 def plot_loss(self, skip_last=0): plt.plot(self.losses[:len(self.losses)-skip_last])
 def plot(self, skip_last=0, pgid=-1):
losses = [o.item() for o in self.losses]
lrs = self.lrs[pgid]
n = len(losses)-skip_last
plt.xscale('log')
 plt.plot(lrs[:n], losses[:n])
 class ParamScheduler(Callback):
_order=1
def __init__(self, pname, sched_funcs): self.pname,self.sched_funcs = pname,sched_funcs
def begin_fit(self):
if not isinstance(self.sched_funcs,(list,tuple)) :
self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)
 def set_param(self):
assert len(self.opt.param_groups)==len(self.sched_funcs)
 for pg,f in zip(self.opt.param_groups,self.sched_funcs):
pg[self.pname] = f(self.n_epochs/self.epochs)
 def begin_batch(self):
if self.in_train: self.set_param()
class LR_Find(Callback):
_order=1
def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr
self.best_loss = 1e9
def begin_batch(self):
if not self.in_train: return
pos = self.n_iter/self.max_iter
lr = self.min_lr *(self.max_lr/self.min_lr)** pos
for pg in self.opt.param_groups: pg['lr'] = lr
def after_step(self):
if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
raise CancelTrainException()
if self.loss < self.best_loss: self.best_loss = self.loss
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs=0.
 self.run.n_iter=0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs += 1./self.iters
self.run.n_iter += 1
def begin_epoch(self):
self.run.n_epochs=self.epoch
self.model.train()
self.run.in_train=True
def begin_validate(self):
self.model.eval()
self.run.in_train=False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass
class Learner() :
def __init__(self, model, data, loss_func, optimizer, lr=1e-2, splitter=param_getter,
cbs=None, cb_funcs=None):
self.model,self.data,self.loss_func,self.lr,self.splitter = model,data,loss_func,lr,splitter
self.in_train,self.logger,self.opt = False,print,optimizer
self.cbs = []
self.add_cb(TrainEvalCallback())
 self.add_cbs(cbs)
 self.add_cbs(cbf() for cbf in listify(cb_funcs))
def add_cbs(self, cbs):
for cb in listify(cbs): self.add_cb(cb)
 def add_cb(self, cb):
cb.set_runner(self)
 setattr(self, cb.name, cb)
 self.cbs.append(cb)
 def remove_cbs(self, cbs):
for cb in listify(cbs): self.cbs.remove(cb)
 def one_batch(self, i, xb, yb):
try:
self.iter = i
self.xb,self.yb = xb,yb; self('begin_batch')
 self.pred = self.model(self.xb); self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb); self('after_loss')
 if not self.in_train: return
self.loss.backward() ; self('after_backward')
 self.opt.step() ; self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self):
self.iters = len(self.dl)
 try:
for i,(xb,yb)in enumerate(self.dl): self.one_batch(i, xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def do_begin_fit(self, epochs):
self.epochs,self.loss = epochs,torch.tensor(0.)
self('begin_fit')
 def do_begin_epoch(self, epoch):
self.epoch,self.dl = epoch,self.data.train_dl
return self('begin_epoch')
 def fit(self, epochs, cbs=None, reset_opt=False):
self.add_cbs(cbs)
 try:
self.do_begin_fit(epochs)
 for epoch in range(epochs):
if not self.do_begin_epoch(epoch): self.all_batches()
with torch.no_grad() :
self.dl = self.data.valid_dl
if not self('begin_validate'): self.all_batches()
self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.remove_cbs(cbs)
 ALL_CBS = {'begin_batch', 'after_pred', 'after_loss', 'after_backward', 'after_step',
'after_cancel_batch', 'after_batch', 'after_cancel_epoch', 'begin_fit',
'begin_epoch', 'begin_validate', 'after_epoch',
'after_cancel_train', 'after_fit'}
def __call__(self, cb_name):
res = False
assert cb_name in self.ALL_CBS
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res
def annealer(f):
def _inner(start,end): return partial(f,start,end)
 return _inner
@annealer
def sched_lin(start,end,pos): return start + pos*(end-start)
 @annealer
def sched_cos(start,end,pos):return start+(1+math.cos(math.pi*(1-pos)))*(end-start)/2
@annealer
def sched_no(start,end,pos): return start
@annealer
def sched_exp(start,end,pos): return start*(end/start)**pos
def cos_1cycle_anneal(start,high,end):
return [sched_cos(start,high),sched_cos(high,end)]
def combine_scheds(pcts, scheds):
assert sum(pcts)== 1.
 pcts = torch.tensor([0] + listify(pcts))
assert torch.all(pcts >= 0)
 pcts = torch.cumsum(pcts, 0)
 def _inner(pos):
idx =(pos >= pcts ).nonzero().max()
actual_pos =(pos-pcts[idx])/(pcts[idx+1]-pcts[idx])
 return scheds[idx](actual_pos)
 return _inner
class CudaCallback(Callback):
def begin_fit(self): self.model.cuda()
def begin_batch(self):
if type(self.run.xb)is dict:
for key in self.run.xb.keys() :
self.run.xb[key]=self.run.xb[key].cuda()
if type(self.run.yb)is dict:
for key in self.run.yb.keys() :
if type(self.run.yb[key])is not list:
self.run.yb[key]=self.run.yb[key].cuda()
class AvgStats() :
def __init__(self,metrics,in_train): self.metrics,self.in_train = listify(metrics),in_train
def reset(self):
self.tot_loss,self.count =0.,0.
 self.tot_mets = [0.]*len(self.metrics)
 @property
def all_stats(self):return [self.tot_loss.item() ]+ self.tot_mets
@property
def avg_stats(self): return [o/self.count for o in self.all_stats]
def __repr__(self):
if not self.count: return ''
return f"{'train' if self.in_train else 'valid'}: {self.avg_stats}"
def accumulate(self,run):
bn = run.xb[list(run.xb.keys())[0]].shape[0]
self.tot_loss+=run.loss*bn
self.count+=bn
if not self.in_train:
for i,m in enumerate(self.metrics):
self.tot_mets[i]+= m(run.pred,run.yb)*bn
class ProgressCallback(Callback):
_order=-1
def begin_fit(self):
self.mbar = master_bar(range(self.epochs))
self.mbar.on_iter_begin()
self.run.logger = partial(self.mbar.write, table=True)
 def after_fit(self): self.mbar.on_iter_end()
def after_batch(self): self.pb.update(self.iter)
 def begin_epoch(self): self.set_pb()
def begin_validate(self): self.set_pb()
def after_loss(self): self.pb.comment='loss= ' +str(self.loss.item())[:5]+' lr '+str(self.opt.param_groups[0]['lr'])[:9]
def set_pb(self):
self.pb = progress_bar(self.dl, parent=self.mbar)
 self.mbar.update(self.epoch)
 class EarlyStopingCallback(Callback):
_order=-1
def __init__(self,iters=1,path='bert.pth'):
self.iters=iters
self.bad_metrics =iters
self.path=path

 def begin_fit(self):
self.best_metric=[0]
def after_epoch(self):
mean_metric=self.avg_stats.valid_stats.avg_stats[1]
if mean_metric>self.best_metric:
self.bad_metrics=self.iters
self.best_metric=mean_metric
if self.path:
print('Saving.....',mean_metric)
 torch.save(self.run.model.state_dict() ,self.path)
 else:
self.bad_metrics-=1
if self.bad_metrics==0:
self.run.model.load_state_dict(torch.load(self.path))
raise CancelTrainException() 
<categorify>
trainer.fit(train_pet_ids, X_train_cat, X_train_ss, target, 5 )
config.TOKENIZER.decode([103] )<define_variables>
bin_path = trainer.path
test_preds = np.zeros(( X_test_cat.shape[0]))
c_tensor = torch.tensor(X_test_cat.values, dtype=torch.long ).to(trainer.device)
 n_tensor = torch.tensor(X_test_ss, dtype=torch.float32 ).to(trainer.device)
 test_dataset = PetDataset(test_pet_ids,
c_tensor,
n_tensor,
labels=None,
root_dir=".. /input/petfinder-adoption-prediction/test_images/",
transform=ds_trans)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 for path in bin_path.iterdir() :
print(f"using {str(path)}")
 model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( X_test_cat.shape[0]))
for i,(i_batch, c_batch, n_batch)in enumerate(test_loader):
i_batch = i_batch.to(trainer.device)
 with torch.no_grad() :
y_pred = model(i_batch, c_batch, n_batch ).detach()
temp[i * 128:(i + 1)* 128] = y_pred.cpu().numpy() [:, 0]
test_preds += temp / trainer.n_splits
class TweetDataset:
def __init__(self,tweet,sentiment,selected_text,MAX_LEN,TOKENIZER):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.max_len = MAX_LEN
self.tokenizer = TOKENIZER
def __len__(self):
return len(self.tweet)
 def __getitem__(self,item):
tweet = self.tweet[item]
selected_text =self.selected_text[item]
len_sel_text = len(selected_text)
 idx0 = -1
idx1 = -1
for ind in(i for i,e in enumerate(tweet)if e == selected_text[0]):
if tweet[ind:ind+len_sel_text] == selected_text:
idx0=ind
idx1=ind+len_sel_text-1
break
char_targets = [0]*len(tweet)
 if idx0 != -1 and idx1 !=-1:
for j in range(idx0,idx1+1):
char_targets[j] = 1
sentiment_id = {
'positive': 3893,
'negative': 4997,
'neutral': 8699}
tok_tweet =self.tokenizer.encode(tweet)
 tok_tweet_ids = tok_tweet.ids[1:-1]
tok_tweet_offsets = tok_tweet.offsets[1:-1]
t=True
targets = []
for j ,(offset1,offset2)in enumerate(tok_tweet_offsets):
if sum(char_targets[offset1:offset2])>0:
targets.append(j)
 token_type_ids = [0]*len(tok_tweet_ids)
 token_type_ids=[0]*3 + [1]*len(tok_tweet_ids + [102])
 tok_tweet_ids=[101] + [sentiment_id[self.sentiment[item]]] + [102] + tok_tweet_ids + [102]
tweet_offsets = [(0, 0)] * 3 + tok_tweet_offsets + [(0, 0)]
mask = [1]*len(tok_tweet_ids)
 padding_len = self.max_len - len(tok_tweet_ids)
 if padding_len>1:
ids = tok_tweet_ids+ [0]*padding_len
mask = mask + [0]*padding_len
token_type_ids += [0]*(padding_len)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_len)
 return {
'ids':torch.tensor(ids,dtype=torch.long),
'mask':torch.tensor(mask,dtype=torch.long),
'token_type_ids':torch.tensor(token_type_ids,dtype=torch.long),
'padding_len':torch.tensor(padding_len,dtype=torch.long),
},{'tweet_offsets':torch.tensor(tweet_offsets,dtype=torch.long),
'orig_selected':self.selected_text[item],
'sentiment':self.sentiment[item],
'orig_tweet': self.tweet[item],
'targets_start':torch.tensor(targets[0]+3,dtype=torch.long),
'targets_end':torch.tensor(targets[-1]+3,dtype=torch.long)}<load_from_csv>
class ImageDataset(data.Dataset):
def __init__(self, pet_ids, root_dir, transform=None):
self.pet_ids = pet_ids
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.pet_ids)
 def __getitem__(self, idx):
img_name = f"{self.pet_ids[idx]}-1.jpg"
fullname = self.root_dir / Path(img_name)
 try:
image = Image.open(fullname ).convert("RGB")
 except FileNotFoundError:
image = np.zeros(( 3, 224, 224), dtype=np.uint8 ).transpose(1, 2, 0)
 image = Image.fromarray(np.uint8(image))
if self.transform:
image = self.transform(image)
 return [image]
df=pd.read_csv(TRAINING_FILE ).dropna().reset_index(drop=True)
 dset = TweetDataset(tweet = df.text.values,
sentiment = df.sentiment.values,
selected_text = df.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER)
<import_modules>
train_dataset = ImageDataset(
 train_pet_ids,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 batch = 256
n_img_dim = 48
train_loader = data.DataLoader(train_dataset,
batch_size=batch,
shuffle=False)
 X_train_img = np.zeros(( len(train_pet_ids), n_img_dim))
test_dataset = ImageDataset(
 test_pet_ids,
".. /input/petfinder-adoption-prediction/test_images/",
transform=ds_trans)
 test_loader = data.DataLoader(test_dataset,
batch_size=batch,
shuffle=False)
 X_test_img = np.zeros(( len(test_pet_ids), n_img_dim))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
i_batch = i_batch.to("cuda:0")
 y_pred = model.densenet121(i_batch ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_train_img += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
i_batch = i_batch.to("cuda:0")
 y_pred = model.densenet121(i_batch ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_test_img += temp / trainer.n_splits
from sklearn.model_selection import StratifiedKFold<choose_model_class>
train_img = pd.DataFrame(data=X_train_img, columns=[
f"img{i}" for i in range(X_train_img.shape[1])
 ])
 test_img = pd.DataFrame(data=X_test_img, columns=[
f"img{i}" for i in range(X_test_img.shape[1])
 ] )
skf = StratifiedKFold(n_splits=5)
 ids=skf.get_n_splits(df['text'], df['sentiment'] )<feature_engineering>
num_columns = X_train_num.columns
X_train_num_df = pd.DataFrame(data=X_train_ss, columns=num_columns)
 X_test_num_df = pd.DataFrame(data=X_test_ss, columns=num_columns)
 X_test_num_df.index = X_test_cat.index
test_img.index = X_test_cat.index
X_train_all = pd.concat([X_train_num_df, X_train_cat, train_img], axis=1)
 X_test_all = pd.concat([X_test_num_df, X_test_cat, test_img], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()
df['kfold']=0
for ids,(train_index, test_index)in enumerate(skf.split(df['text'], df['sentiment'])) :
df['kfold'].loc[test_index]=ids<import_modules>
class CategoryDataset(data.Dataset):
def __init__(self, category):
self.category = category
def __len__(self):
return len(self.category)
 def __getitem__(self, idx):
category = self.category[idx, :]
return [category]
import torch.nn as nn
import transformers
import torch
from sklearn import model_selection
from transformers.modeling_bert import BertPreTrainedModel
import scipy<choose_model_class>
c_train = torch.tensor(X_train_cat.values, dtype=torch.long ).to("cuda:0")
 c_test = torch.tensor(X_test_cat.values, dtype=torch.long ).to("cuda:0")
 train_dataset = CategoryDataset(c_train)
 test_dataset = CategoryDataset(c_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_cat_ = np.zeros(( len(train_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred =torch.cat(y_pred, 1 ).detach()
X_train_cat_[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_cat_ = np.zeros(( len(test_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred = torch.cat(y_pred, 1 ).detach()
X_test_cat_[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
class BERTBaseUncased(transformers.BertPreTrainedModel):
def __init__(self, conf,drop_sample=5):
super(BERTBaseUncased, self ).__init__(conf)
 self.bert = transformers.BertModel.from_pretrained(config.BERT_PATH, config=conf)
 self.drop_out = nn.Dropout(0.5)
 self.drop = nn.Dropout(0.2)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 self.drop_sample=drop_sample
def forward(self, x):
_,out1,out= self.bert(
 x['ids'],
attention_mask=x['mask'],
token_type_ids=x['token_type_ids'] 
)
 out = torch.cat(( out1.unsqueeze(1 ).repeat(1,128,1),out[-1]), dim=-1)
 res_start=[]
res_end=[]
if self.training:
for i in range(self.drop_sample):
outt = self.drop_out(out)
 logits = self.l0(outt)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 res_start.append(start_logits)
 res_end.append(end_logits)
 return res_start, res_end
else:
outt = self.drop_out(out)
 logits = self.l0(outt)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits,end_logits<compute_train_metric>
train_emb = pd.DataFrame(data=X_train_cat_, columns=[
f"emb{i}" for i in range(X_train_cat_.shape[1])
 ])
 test_emb = pd.DataFrame(data=X_test_cat_, columns=[
f"emb{i}" for i in range(X_test_cat_.shape[1])
 ] )
def loss_fn(outputs,targets):
loss_=0
if type(outputs[0])is list:
for i in range(len(outputs[0])) :
loss_+= nn.CrossEntropyLoss()(outputs[0][i],targets['targets_start'])+ nn.CrossEntropyLoss()(outputs[1][i],targets['targets_end'])
 loss_=loss_/len(outputs[0])
 else:
loss_=nn.CrossEntropyLoss()(outputs[0],targets['targets_start'])+ nn.CrossEntropyLoss()(outputs[1],targets['targets_end'])
 return loss_
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def calculate_jaccard_score(original_tweet, target_string, sentiment_val, idx_start, idx_end, offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = jaccard(target_string, filtered_output)
 return jac, filtered_output
def metric(outputs,target):
tweet,selected_tweet,tweet_sentiment,offsets =target['orig_tweet'],target['orig_selected'],target['sentiment'],target['tweet_offsets'].cpu().numpy()
outputs_start,outputs_end = outputs[0].softmax(dim=1 ).detach().cpu().numpy() ,outputs[1].softmax(dim=1 ).detach().cpu().numpy()
jac=[]
for ind in range(len(tweet)) :
jac.append(calculate_jaccard_score(tweet[ind], selected_tweet[ind], tweet_sentiment[ind], outputs_start[ind].argmax() ,outputs_end[ind].argmax() , offsets[ind])[0])
 return np.mean(jac )<split>
test_emb.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_emb], axis=1)
 X_test_all = pd.concat([X_test_all, test_emb], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()
TRAIN_BATCH_SIZE=2
MAX_LEN=128
EPOCHS = 2
dfx=pd.read_csv(TRAINING_FILE ).dropna().reset_index(drop=True)
 sample =pd.read_csv(SAMPLE_FILE)
 df_train,df_valid = model_selection.train_test_split(dfx,random_state=42,test_size=0.2)
 df_train = df_train.reset_index(drop=True)
 df_valid = df_valid.reset_index(drop=True)
 target_cols = list(sample.drop('textID',axis=1 ).columns)
 train_targets = df_train[target_cols].values
valid_targets = df_valid[target_cols].values
train_dataset = TweetDataset(df_train.text.values,df_train.sentiment.values,df_train.selected_text.values,MAX_LEN,config.TOKENIZER)
 train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=TRAIN_BATCH_SIZE,shuffle=True)
 valid_dataset = TweetDataset(df_valid.text.values,df_valid.sentiment.values,df_valid.selected_text.values,MAX_LEN,config.TOKENIZER)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=16,shuffle=False )<load_pretrained>
xgb_params = {
"eval_metric": "rmse",
"seed": 1337,
"eta": 0.01,
"subsample": 0.75,
"colsample_bytree": 0.85,
"tree_method": "gpu_hist",
"device": "gpu",
"silent": 1
}
xgb_X = X_train_all
xgb_y = target
xgb_X_test = X_test_all
model, oof_train_xgb, oof_test_xgb= run_xgb(
 xgb_params,
xgb_X,
xgb_y,
xgb_X_test,
resc=train_pet_ids,
n_splits=5,
num_rounds=10000 )
data = DataBunch(train_loader,valid_loader)
<choose_model_class>
def run_lgb(params,
X,
y,
X_test,
resc,
cat_features,
n_splits=10,
early_stop=500):
fold = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
for i,(trn_index, val_index)in enumerate(fold.split(X, y, resc)) :
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
model = lgb.LGBMRegressor(**params)
 model.fit(X_tr,
y_tr,
eval_set=(X_val, y_val),
verbose=500,
early_stopping_rounds=early_stop,
categorical_feature=cat_features)
 valid_pred = model.predict(X_val, num_iteration=model.best_iteration_)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test
model_config = transformers.BertConfig.from_pretrained(config.BERT_PATH)
 model_config.output_hidden_states = True
model = BERTBaseUncased(conf=model_config)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = transformers.AdamW(optimizer_parameters, lr=1e-4/2 )<count_values>
lgb_params = {
"boosting_type": "gbdt",
"num_leaves": 127,
"learning_rate": 0.01,
"n_estimators": 10000,
"subsample": 0.75,
"subsample_freq": 8,
"colsample_bytree": 0.75,
"reg_alpha": 0.01,
"reg_lambda": 0.01,
"n_jobs": -1
}
model, oof_train, oof_test = run_lgb(
 lgb_params,
xgb_X,
xgb_y,
xgb_X_test,
train_pet_ids,
cat_features,
n_splits=5,
early_stop=500 )
df.kfold.value_counts()<load_from_csv>
lgb_xgb_nn = 0.6 * lgb_xgb + 0.4 *(nn_preds * 4)
 lgb_xgb_nn_test = 0.6 * lgb_xgb_test.mean(1)+ 0.4 * nn_preds_test * 4
def get_learner(model,opt,data, loss_func,
cb_funcs=None):
return Learner(model, data, loss_func, opt,cb_funcs=cb_funcs)
 def run_fold(k):
TRAIN_BATCH_SIZE=32
MAX_LEN=128
sample =pd.read_csv(SAMPLE_FILE)
 df_train,df_valid = df[df.kfold!=k],df[df.kfold==k]
df_train = df_train.reset_index(drop=True)
 df_valid = df_valid.reset_index(drop=True)
 target_cols = list(sample.drop('textID',axis=1 ).columns)
 train_targets = df_train[target_cols].values
valid_targets = df_valid[target_cols].values
train_dataset = TweetDataset(df_train.text.values,df_train.sentiment.values,df_train.selected_text.values,MAX_LEN,config.TOKENIZER)
 train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=TRAIN_BATCH_SIZE,shuffle=True)
 valid_dataset = TweetDataset(df_valid.text.values,df_valid.sentiment.values,df_valid.selected_text.values,MAX_LEN,config.TOKENIZER)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=16,shuffle=False)
 data = DataBunch(train_loader,valid_loader)
 model_config = transformers.BertConfig.from_pretrained(config.BERT_PATH)
 model_config.output_hidden_states = True
model = BERTBaseUncased(conf=model_config)
 params = list(model.named_parameters())
 def is_backbone(n):
return "bert" in n
lrr=2e-5
optimizer_grouped_parameters = [
{"params": [p for n, p in params if is_backbone(n)], "lr": lrr},
{"params": [p for n, p in params if not is_backbone(n)], "lr":lrr * 100},
]
optimizer = torch.optim.AdamW(
 optimizer_grouped_parameters, lr=lrr, weight_decay=0 
)
 cbfs = [Recorder,
partial(AvgStatsCallback,metric),
CudaCallback,
ProgressCallback,
partial(EarlyStopingCallback,1)]
learn = get_learner(model,optimizer,data,loss_fn, cb_funcs=cbfs)
 learn.fit(10)
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test['selected_text']=test['text'].apply(lambda x:x)
 dset = TweetDataset(tweet = test.text.values,
sentiment = test.sentiment.values,
selected_text = test.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER)
 res_start=[]
res_end=[]
bs=16
for x,y in tqdm_notebook(torch.utils.data.DataLoader(dset,batch_size=bs,shuffle=False)) :
for key in x.keys() :
x[key]=x[key].cuda()
output = learn.model(x)
 res_start.append(output[0].detach().cpu().numpy())
 res_end.append(output[1].detach().cpu().numpy())
 return np.concatenate(res_start),np.concatenate(res_end)
<feature_engineering>
opt = OptimizedRounder()
opt.fit(lgb_xgb, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_test.mean(1), coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_xgb_lgb.csv", index=False)
 submission.head()
start1,end1=run_fold(0 )<feature_engineering>
opt = OptimizedRounder()
opt.fit(lgb_xgb_nn, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb_nn, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb_nn, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_nn_test, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission.csv", index=False)
 submission.head()
<feature_engineering><EOS> 
opt = OptimizedRounder()
opt.fit(nn_preds * 4, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(nn_preds * 4, coeff)
 qwk = quadratic_weighted_kappa(target, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(nn_preds * 4, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(nn_preds_test * 4, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_nn.csv", index=False)
 submission.head()