Kaggle competition: PetFinder.my Adoption Prediction
data_train = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/train.tsv', delimiter='\t' )<load_from_csv>
np.random.seed(42)
 seed=42
data_test = pd.read_csv('/kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv', delimiter='\t' )<create_dataframe>
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
print("Train DataFrame")
 print(pd.isnull(data_train ).sum())
 print("="*50)
 print("Test DataFrame")
 print(pd.isnull(data_test ).sum() )<count_missing_values>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
print(( pd.isnull(data_train['brand_name'] ).sum())*100/data_train.shape[0])
<count_missing_values>
def get_learner(data,layers,save_name='best_nn',y_range=None):
return tabular_learner(data, layers=layers,
callback_fns=[partial(SaveModelCallback, name=save_name)],
y_range=y_range 
)
 def get_learner_no_cb(data,layers,y_range=None):
return tabular_learner(data, layers=layers, y_range=y_range )
print("{0:.2f}% brand_name has missing value in train data".format(( pd.isnull(data_train['brand_name'] ).sum())*100/data_train.shape[0]))
print("{0:.2f}% category_name has missing value in train data".format(( pd.isnull(data_train['category_name'] ).sum())*100/data_train.shape[0])) 
<count_missing_values>
print("{0:.2f}% brand_name has missing value in test data".format(( pd.isnull(data_test['brand_name'] ).sum())*100/data_test.shape[0]))
print("{0:.2f}% category_name has missing value in test data".format(( pd.isnull(data_test['category_name'] ).sum())*100/data_test.shape[0])) 
<data_type_conversions>
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )
data_train['brand_name'] = data_train['brand_name'].fillna('Unknown')
 data_test['brand_name'] = data_test['brand_name'].fillna('Unknown' )<data_type_conversions>
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )
data_train['category_name'] = data_train['category_name'].fillna('No/No/No')
 data_test['category_name'] = data_test['category_name'].fillna('No/No/No' )<concatenate>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score
def find_sub_cat(X):
try:
return(X.split("/"))
return("jubu")
 except:
return("None","None","None")
<feature_engineering>
train.Name = train.Name.str.lower()
test.Name = test.Name.str.lower()
train.Name.fillna("no name",inplace=True)
 test.Name.fillna("no name",inplace=True)
 train['name_length']= train.Name.str.len()
train.loc[train.Name.str.contains('no name|unknown|unname|noname|none'),'name_length']=0
test['name_length']= test.Name.str.len()
test.loc[test.Name.str.contains('no name|unknown|unname|noname|none'),'name_length']=0
data_train['main_category']=''
data_train['sub_category_1']=''
data_train['sub_category_2']=''
data_train['main_category'],data_train['sub_category_1'],data_train['sub_category_2']\
= zip(*data_train['category_name'].apply(lambda x: find_sub_cat(x)))
<feature_engineering>
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120,random_state=42)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )
data_test['main_category']=''
data_test['sub_category_1']=''
data_test['sub_category_2']=''
data_test['main_category'],data_test['sub_category_1'],data_test['sub_category_2']\
= zip(*data_test['category_name'].apply(lambda x: find_sub_cat(x)))
<count_values>
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores
brands = data_train['brand_name'].value_counts()
print(brands[:10] )<count_values>
train.drop(['Name', 'RescuerID'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID'], axis=1, inplace=True )
( data_train['shipping'].value_counts())*100/data_train.shape[0]<count_unique_values>
train.drop(['Description'],inplace=True,axis=1)
 test.drop(['Description'],inplace=True,axis=1)
 target = target.astype(np.float32)
 train['AdoptionSpeed'] = target
print("There are %d unique main_categories." % data_train['main_category'].nunique() )<count_unique_values>
cont_names=['name_length','Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_names = list(set(train.columns)- set(cont_names)- {'AdoptionSpeed'})
 print(f'
print(f'
dep_var = 'AdoptionSpeed'
procs = [Categorify, Normalize]
print("There are %d unique sub_category_1." % data_train['sub_category_1'].nunique() )<count_unique_values>
train_idxs=[]
val_idxs=[]
skf = StratifiedKFold(n_splits=5, random_state=seed, shuffle=True)
 for train_idx, val_idx in skf.split(train, train.AdoptionSpeed):
train_idxs.append(train_idx)
 val_idxs.append(val_idx)
 val_idxs
print("There are %d unique sub_category_2." % data_train['sub_category_2'].nunique() )<define_variables>
l2str= lambda layers: '_'.join([str(l)for l in layers] )
stopwords= ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've",\
"you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \
'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their',\
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', \
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \
'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\
'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\
'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\
'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \
's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', \
've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn',\
"hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn',\
"mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", \
'won', "won't", 'wouldn', "wouldn't"]<categorify>
def metric(y1,y2):
return cohen_kappa_score(y1,y2, weights='quadratic' )
def decontracted(phrase):
try:
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
except:
return 0<string_transform>
layers=[200,100]
def wordCount_with_cleaning(sentance):
try:
sent = decontracted(sentance)
 sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 sent = sent.strip()
return(len(sent.split()))
except:
return 0<string_transform>
data = get_databunch(0,bs=128 )
def wordCount_without_cleaning(sentance):
try:
sent = ' '.join(e for e in sentance.split())
 sent = sent.strip()
return(len(sent.split()))
except:
return 0<string_transform>
model_name = f'best_{l2str(layers)}_reg_s'
model_name
preprocessed_item_description = []
for sentance in tqdm(data_train['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_item_description.append(sent.lower().strip() )<string_transform>
learn = get_learner_no_cb(data,layers)
 learn.lr_find()
learn.recorder.plot(skip_end=1 )
preprocessed_test_item_description = []
for sentance in tqdm(data_test['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_test_item_description.append(sent.lower().strip() )<drop_column>
learn = get_learner(data,layers,model_name+'1')
 learn.fit_one_cycle(6,1e-01,pct_start=0.5)
 learn.recorder.plot_losses()
data_train = data_train.drop('item_description',axis=1)
 data_train['item_description'] = preprocessed_item_description
data_train.head(2 )<drop_column>
coeff=evaluate_classification(val_preds,data.valid_ds.y.items )
data_test = data_test.drop('item_description',axis=1)
 data_test['item_description'] = preprocessed_test_item_description
data_test.head(2 )<drop_column>
learn = get_learner(data,layers,model_name+'2')
 learn.load(model_name+'1')
 learn.fit_one_cycle(5,max_lr=3e-05)
 learn.recorder.plot_losses()
<prepare_x_and_y>
coeff = evaluate_classification(val_preds,data.valid_ds.y.items)
y = np.log10(np.array(data_train['price'])+1)
 X = data_train.drop('price',axis=1)
<split>
new_coeff=[1.65,coeff[1]+0.01,coeff[2],coeff[3]-0.01]
_=evaluate_classification(val_preds,data.valid_ds.y.items,new_coeff )
X_train,X_cv,Y_train,Y_cv = train_test_split(X, y, test_size=0.20, random_state=42)
<sort_values>
test_preds = np.squeeze(to_np(learn.get_preds(DatasetType.Test)[0]))
def concat_categories(x):
return set(x.values)
 def brandfinder(name, category):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
return brand
return 'Unknown' 
<data_type_conversions>
test_preds = OptimizedRounder().predict(test_preds,new_coeff)
 test_preds=test_preds.astype(np.int32)
<data_type_conversions><EOS> 
submission_df = pd.DataFrame(data={'PetID' : pd.read_csv('.. /input/test/test.csv')['PetID'],
'AdoptionSpeed' : test_preds})
 submission_df.to_csv('submission.csv', index=False )