Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<data_type_conversions>
%matplotlib inline
START_DATE = '2017-11-30'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 train_trn['Date'] = train_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 train_trn['_ymd'] = train_trn['Date'].dt.year.astype(str)+ '-' + train_trn['Date'].dt.month.astype(str)+ '-' + train_trn['Date'].dt.day.astype(str)
 train_trn['_year_month'] = train_trn['Date'].dt.year.astype(str)+ '-' + train_trn['Date'].dt.month.astype(str)
 train_trn['_weekday'] = train_trn['Date'].dt.dayofweek
train_trn['_hour'] = train_trn['Date'].dt.hour
train_trn['_day'] = train_trn['Date'].dt.day
fig,ax = plt.subplots(4, 1, figsize=(16,12))
train_trn.groupby('_weekday')['isFraud'].mean().to_frame().plot.bar(ax=ax[0])
 train_trn.groupby('_hour')['isFraud'].mean().to_frame().plot.bar(ax=ax[1])
 train_trn.groupby('_day')['isFraud'].mean().to_frame().plot.bar(ax=ax[2])
 train_trn.groupby('_year_month')['isFraud'].mean().to_frame().plot.bar(ax=ax[3] )<sort_values>
score=[]
magnitude=[]
petid=[]
for filename in sentimental_analysis:
with open(filename, 'r')as f:
sentiment_file = json.load(f)
 file_sentiment = sentiment_file['documentSentiment']
file_score = np.asarray(sentiment_file['documentSentiment']['score'])
 file_magnitude =np.asarray(sentiment_file['documentSentiment']['magnitude'])
 score.append(file_score)
 magnitude.append(file_magnitude)
 petid.append(filename.replace('.json','' ).replace('.. /input/train_sentiment/', ''))
sentimental_analysis = pd.concat([ pd.DataFrame(petid, columns =['PetID']),pd.DataFrame(score, columns =['sentiment_document_score']),
pd.DataFrame(magnitude, columns =['sentiment_document_magnitude'])],axis =1 )
df = train_trn.groupby(['_ymd'])['isFraud'].agg(['count','mean','sum'])
 df.sort_values(by='mean',ascending=False)[:10].T<sort_values>
image_metadata = sorted(glob.glob('.. /input/train_metadata/*.json'))
print('num of train metadata: {}'.format(len(image_metadata)) )
df.sort_values(by='count',ascending=False)[:10].T<data_type_conversions>
description=[]
topicality=[]
imageid=[]
for filename in image_metadata:
with open(filename, 'r')as f:
d = json.load(f)
 file_keys = list(d.keys())
 if 'labelAnnotations' in file_keys:
file_annots = d['labelAnnotations']
file_topicality = np.asarray([x['topicality'] for x in file_annots])
 file_description = [x['description'] for x in file_annots]
description.append(file_description)
 topicality.append(file_topicality)
 imageid.append(filename.replace('.json','' ).replace('.. /input/train_metadata/',''))
description=pd.DataFrame(description)
 topicality=pd.DataFrame(topicality)
 new_names = [(i,'metadata_description_'+str(i)) for i in description.iloc[:, 0:].columns.values]
description.rename(columns = dict(new_names), inplace=True)
 new_names = [(i,'metadata_topicality_'+str(i)) for i in topicality.iloc[:, 0:].columns.values]
topicality.rename(columns = dict(new_names), inplace=True)
 image_labelannot = pd.concat([ pd.DataFrame(imageid, columns =['ImageId']),topicality,description],axis =1)
 image_labelannot['PetID'] = image_labelannot['ImageId'].str.split('-' ).str[0]
train_trn['_weekday_hour'] = train_trn['_weekday'].astype(str)+ '_' + train_trn['_hour'].astype(str)
 train_trn.groupby('_weekday_hour')['isFraud'].mean().to_frame().plot.line(figsize=(16,3))<sort_values>
image_labelannot['metadata_topicality_mean'] = image_labelannot.iloc[:,1:10].mean(axis=1)
 image_labelannot['metadata_topicality_mean'] = image_labelannot.groupby(['PetID'])['metadata_topicality_mean'].transform('mean')
 image_labelannot['metadata_topicality_max'] = image_labelannot.iloc[:,1:10].max(axis=1)
 image_labelannot['metadata_topicality_max'] = image_labelannot.groupby(['PetID'])['metadata_topicality_max'].transform(max)
 image_labelannot['metadata_topicality_min'] = image_labelannot.iloc[:,1:10].min(axis=1)
 image_labelannot['metadata_topicality_min'] = image_labelannot.groupby(['PetID'])['metadata_topicality_min'].transform(min)
 image_labelannot['metadata_topicality_0_mean'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform('mean')
 image_labelannot['metadata_topicality_0_max'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform(max)
 image_labelannot['metadata_topicality_0_min'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform(min)
 image_labelannot['L_metadata_0_cat']=image_labelannot['metadata_description_0'].str.contains("cat" ).astype(int)
 image_labelannot['L_metadata_0_dog'] =image_labelannot['metadata_description_0'].str.contains("dog" ).astype(int)
 image_labelannot['L_metadata_any_cat']=image_labelannot.apply(lambda row: row.astype(str ).str.contains('cat' ).any() , axis=1)
 image_labelannot['L_metadata_any_dog']=image_labelannot.apply(lambda row: row.astype(str ).str.contains('dog' ).any() , axis=1)
 image_labelannot['L_metadata_0_cat_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_0_cat'].transform('sum')
 image_labelannot['L_metadata_0_dog_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_0_dog'].transform('sum')
 image_labelannot['L_metadata_any_cat_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_any_cat'].transform('sum')
 image_labelannot['L_metadata_any_dog_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_any_dog'].transform('sum')
 image_labelannot = image_labelannot[['PetID','metadata_topicality_max','metadata_topicality_mean','metadata_topicality_min','metadata_topicality_0_mean','metadata_topicality_0_max','metadata_topicality_0_min','L_metadata_0_cat_sum','L_metadata_0_dog_sum','L_metadata_any_cat_sum','L_metadata_any_dog_sum']]
image_labelannot=image_labelannot.drop_duplicates('PetID' )
df = train_trn.groupby('_weekday')['isFraud'].mean().to_frame()
df.sort_values(by='isFraud', ascending=False )<prepare_output>
color_score_mean=[]
color_score_min=[]
color_score_max=[]
color_pixelfrac_mean=[]
color_pixelfrac_min=[]
color_pixelfrac_max=[]
imageid=[]
for filename in image_metadata:
with open(filename, 'r')as f:
d = json.load(f)
 file_keys = list(d.keys())
 if 'imagePropertiesAnnotation' in file_keys:
file_colors = d['imagePropertiesAnnotation']['dominantColors']['colors']
file_color_score_mean = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac_mean = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_score_min = np.asarray([x['score'] for x in file_colors] ).min()
file_color_pixelfrac_min = np.asarray([x['pixelFraction'] for x in file_colors] ).min()
file_color_score_max = np.asarray([x['score'] for x in file_colors] ).max()
file_color_pixelfrac_max = np.asarray([x['pixelFraction'] for x in file_colors] ).max()
imageid.append(filename.replace('.json','' ).replace('.. /input/train_metadata/', ''))
color_score_mean.append(file_color_score_mean)
 color_score_min.append(file_color_score_min)
 color_score_max.append(file_color_score_max)
 color_pixelfrac_mean.append(file_color_pixelfrac_mean)
 color_pixelfrac_min.append(file_color_pixelfrac_min)
 color_pixelfrac_max.append(file_color_pixelfrac_max)
 image_properties = pd.concat([pd.DataFrame({'ImageId':imageid}),pd.DataFrame({'metadata_color_pixelfrac_mean':color_pixelfrac_mean}), pd.DataFrame({'metadata_color_pixelfrac_min':color_pixelfrac_min}),pd.DataFrame({'metadata_color_pixelfrac_max':color_pixelfrac_max}),pd.DataFrame({'metadata_color_score_mean':color_score_mean}),pd.DataFrame({'metadata_color_score_min':color_score_min}),pd.DataFrame({'metadata_color_score_max':color_score_max})],axis=1)
 image_properties['PetID'] = image_properties['ImageId'].str.split('-' ).str[0]
image_properties['metadata_color_pixelfrac_mean'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_mean'].transform('mean')
 image_properties['metadata_color_pixelfrac_min'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_min'].transform(min)
 image_properties['metadata_color_pixelfrac_max'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_max'].transform(max)
 image_properties['metadata_color_score_mean'] = image_properties.groupby(['PetID'])['metadata_color_score_mean'].transform('mean')
 image_properties['metadata_color_score_min'] = image_properties.groupby(['PetID'])['metadata_color_score_min'].transform(min)
 image_properties['metadata_color_score_max'] = image_properties.groupby(['PetID'])['metadata_color_score_max'].transform(max)
 image_properties=image_properties.drop_duplicates('PetID')
 image_properties = image_properties.drop(['ImageId'], 1 )
df = train_trn.groupby('_hour')['isFraud'].mean().to_frame()
df.sort_values(by='isFraud', ascending=False ).head(10 )<define_variables>
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sample_submission = pd.read_csv('.. /input/test/sample_submission.csv')
 breed =pd.read_csv('.. /input/breed_labels.csv',usecols=["BreedID", "BreedName"])
 color =pd.read_csv('.. /input/color_labels.csv')
 state =pd.read_csv('.. /input/state_labels.csv')
 train =(pd.merge(train, breed.rename(columns={"BreedName": "BreedName1"}), how='left', left_on=['Breed1'], right_on = ['BreedID'] ).drop('BreedID', axis=1))
train =(pd.merge(train, breed.rename(columns={"BreedName": "BreedName2"}), how='left', left_on=['Breed2'], right_on = ['BreedID'] ).drop('BreedID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName1"}), how='left', left_on=['Color1'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName2"}), how='left', left_on=['Color2'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName3"}), how='left', left_on=['Color3'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, state, how='inner', left_on=['State'], right_on = ['StateID'] ).drop('StateID', axis=1))
train =(pd.merge(train, sentimental_analysis, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_properties, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_labelannot, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_moments, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_quality, how='left', left_on=['PetID'], right_on = ['PetID']))
fraud_id = train_trn[train_trn['isFraud'] == 1]['TransactionID']
fraud_id_in_trn = [i for i in fraud_id if i in train_id['TransactionID'].values]
print(f'fraud data count:{len(fraud_id)}, and in trn:{len(fraud_id_in_trn)}' )<define_search_space>
train['L_Color1'] =(pd.isnull(train['ColorName3'])& pd.isnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Color2'] =(pd.isnull(train['ColorName3'])& pd.notnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Color3'] =(pd.notnull(train['ColorName3'])& pd.notnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Breed1'] =(pd.isnull(train['BreedName2'])& pd.notnull(train['BreedName1'])).astype(int)
 train['L_Breed2'] =(pd.notnull(train['BreedName2'])& pd.notnull(train['BreedName1'])).astype(int)
 train['Name_Length']=train['Name'].str.len()
train['L_Name_missing'] =(pd.isnull(train['Name'])).astype(int)
 train['L_Breed1_Siamese'] =(train['BreedName1']=='Siamese' ).astype(int)
 train['L_Breed1_Persian']=(train['BreedName1']=='Persian' ).astype(int)
 train['L_Breed1_Labrador_Retriever']=(train['BreedName1']=='Labrador Retriever' ).astype(int)
 train['L_Breed1_Terrier']=(train['BreedName1']=='Terrier' ).astype(int)
 train['L_Breed1_Golden_Retriever ']=(train['BreedName1']=='Golden Retriever' ).astype(int)
 train['Description_Length']=train['Description'].str.len()
train['L_Fee_Free'] =(train['Fee']==0 ).astype(int)
 pets_total = train.groupby(['RescuerID'] ).size().reset_index(name='N_pets_total')
 train= pd.merge(train, pets_total, left_on='RescuerID', right_on='RescuerID', how='inner')
 train.count()
train['L_NoPhoto'] =(train['PhotoAmt']==0 ).astype(int)
 train['L_NoVideo'] =(train['VideoAmt']==0 ).astype(int)
 train['Log_Age']= np.log(train.Age+1)
 train['L_scoreneg'] =(train['sentiment_document_score']<0 ).astype(int)
 train.loc[train['Quantity'] > 5, 'Quantity'] = 5
numid_cols = [f'id_{str(i ).zfill(2)}' for i in range(1,12)]
train_id_trn[numid_cols].isna().sum().to_frame().T / len(train_id )<count_unique_values>
train['Description'] =train['Description'].fillna("<MISSING>")
 train['Description'] = train['Description'].str.replace('\d+', '')
 train['Description'] = train['Description'].str.lower()
train["Description"] = train['Description'].str.replace('[^\w\s]','')
 stop = stopwords.words('english')
 pat = r'\b(?:{})\b'.format('|'.join(stop))
train['Description'] = train['Description'].str.replace(pat, '')
 train['Description'] = train['Description'].str.replace(r'\s+', ' ')
 train['Description'] = train['Description'].astype(str ).str.split()
porter_stemmer = PorterStemmer()
train['Description']=train['Description'].apply(lambda x : [porter_stemmer.stem(y)for y in x])
 train['Description']=train['Description'].apply(lambda x : " ".join(x))
def get_top_n_words(corpus, n=None):
vec = CountVectorizer().fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 test=pd.DataFrame(words_freq[:n], columns=['words','freq'])
 sns.barplot(x='words', y='freq', data=test)
 get_top_n_words(train['Description'],10)
 svd_ = TruncatedSVD(
 n_components=5, random_state=1337)
 nmf_ = NMF(
 n_components=5, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(train['Description'])
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_')
 nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_')
 train = pd.concat([train,nmf_col,svd_col],axis=1)
plotCategoryRateBar('id_19', 20)
 plotHistByFraud('id_19')
 print('unique count:', train_id['id_19'].nunique() )<count_unique_values>
train.loc[train['FurLength'] == 0]
train.loc[train['Health'] == 0 ]
plotCategoryRateBar('id_20', 20)
 plotHistByFraud('id_20')
 print('unique count:', train_id['id_20'].nunique() )<define_variables>
Top4 =rescuer_analysis.sort_values('counts',ascending=False ).head(4)
 Top4
ccols = [f'C{i}' for i in range(1,15)]
dcols = [f'D{i}' for i in range(1,16)]
mcols = [f'M{i}' for i in range(1,10)]
vcols = [f'V{i}' for i in range(1,340)]<feature_engineering>
train.groupby('AdoptionSpeed')['PhotoAmt'].mean()
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 train_date = train_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 test_date = test_trn['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 print('train date:', train_date.min() , '-', train_date.max())
 print('test date:', test_date.min() , '-', test_date.max() )<concatenate>
train.groupby('AdoptionSpeed')['sentiment_document_score'].mean()
def appendLagDT(df):
df = df.assign(_date_lag = df['TransactionDT'] - df.groupby(['card1','card2'])['TransactionDT'].shift(1))
return df
train_trn = appendLagDT(train_trn)
 train_trn_f0 = train_trn[train_trn['isFraud'] == 0]
train_trn_f1 = train_trn[train_trn['isFraud'] == 1]<feature_engineering>
train.groupby('AdoptionSpeed')['sentiment_document_magnitude'].mean()
def appendLagAmt(df):
df = df.assign(_amt_lag = df['TransactionAmt'] - df.groupby(['card1','card2'])['TransactionAmt'].shift(1))
df['_amt_lag_sig'] = df['_amt_lag'].apply(lambda x: '0' if np.isnan(x)else '+' if x >=0 else '-')
 return df
train_trn = appendLagAmt(train_trn)
 train_trn_f0 = train_trn[train_trn['isFraud'] == 0]
train_trn_f1 = train_trn[train_trn['isFraud'] == 1]<feature_engineering>
train_dog=train.loc[train['Type'] == 1]
train_cat=train.loc[train['Type'] == 2]
text_cat = ','.join(str(v)for v in train_cat.Name)
 text_dog = ','.join(str(v)for v in train_dog.Name)
 print("There are {} words in the combination of all Name.".format(len(text_dog)) )
cols = ['addr1','addr2','dist1','dist2']
for f in cols:
train_trn[f + '_isna'] = train_trn[f].isna()<filter>
train_analysis = train.drop(["Name","Description","BreedName2","ColorName3",'Name','Breed1','Breed2','RescuerID','Description',
'BreedName1','Color1', 'Color2', 'Color3','Age','State','ImageId'],axis=1)
 for col in ['sentiment_document_score', 'sentiment_document_magnitude','metadata_topicality_max','metadata_topicality_mean','metadata_topicality_min',
'metadata_topicality_0_mean','metadata_topicality_0_max','metadata_topicality_0_min','L_metadata_0_dog_sum',
'L_metadata_0_cat_sum','L_metadata_any_dog_sum','L_metadata_any_cat_sum','pixel_mean','pixel_min','pixel_max','pixel_sum',
'blur_min','blur_max','blur_sum','blur_mean','metadata_color_pixelfrac_mean','metadata_color_pixelfrac_min',
'metadata_color_pixelfrac_max','metadata_color_score_mean','metadata_color_score_min','metadata_color_score_max',
'Description_Length','Name_Length','huMoments0','huMoments1','huMoments2','huMoments3','huMoments4','huMoments5','huMoments6']:
train_analysis[col].fillna(( train_analysis[col].median()), inplace=True)
 train_analysis["ColorName2"].fillna("No Color", inplace = True )
train_trn[(train_trn['dist1_isna'] == False)&(train_trn['dist2_isna'] == False)][cols]<concatenate>
train_analysis = pd.concat([train_analysis.drop('StateName', axis=1),pd.get_dummies(train_analysis['StateName'], prefix='State')], axis=1)
 col=['ColorName1','ColorName2','Health', 'Gender', 'Dewormed','Type','MaturitySize', 'Sterilized','Vaccinated','FurLength']
for i in col:
train_analysis = pd.concat([train_analysis.drop(i, axis=1),pd.get_dummies(train_analysis[i], prefix=i)], axis=1 )
train_trn = pd.concat([train_trn, pd.get_dummies(train_trn[['ProductCD']])], axis=1)
 train_trn.head(5 )<remove_duplicates>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
train_trn['_amount_max_ProductCD'] = train_trn.groupby(['ProductCD'])['TransactionAmt'].transform('max')
 train_trn[['ProductCD','_amount_max_ProductCD']].drop_duplicates().sort_values(by='_amount_max_ProductCD', ascending=False )<count_missing_values>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.6, 1.7, 2.6, 3.6]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
cols = [f'card{n}' for n in range(1,7)]
train_trn[cols].isnull().sum()<count_unique_values>
def evaluate(y_pred, y_true):
cohen_kappa= cohen_kappa_score(y_true, y_pred)
 accuracy=accuracy_score(y_true,y_pred)
 f1=f1_score(y_true,y_pred,average='micro')
 classification=classification_report(y_true,y_pred)
 cm = confusion_matrix(y_true, y_pred)
 plt.figure(figsize=(20,6))
sns.heatmap(cm, annot=True)
 plt.title('Confusion matrix')
 plt.figure(figsize =(5,4))
plt.show()
print('Cohen Kappa: {:0.2f}.'.format(cohen_kappa))
print('Accuracy Score: {:0.2f}%.'.format(accuracy))
print('F1 Score: {:0.2f}%.'.format(f1))
train_trn[cols].nunique()<data_type_conversions>
ids=train_analysis[['PetID']]
train_analysis=train_analysis.drop(['PetID','State_Labuan'],axis=1 )
train_trn_f0['_card1_card2'] = train_trn_f0['card1'].astype(str)+ '_' + train_trn_f0['card2'].astype(str)
 train_trn_f1['_card1_card2'] = train_trn_f1['card1'].astype(str)+ '_' + train_trn_f1['card2'].astype(str)
 plotTrnCategoryRateBar('_card1_card2', 50, figsize=(15,3))<count_unique_values>
X, y = train_analysis.loc[:, train_analysis.columns != 'AdoptionSpeed'], train_analysis['AdoptionSpeed']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=1, stratify=y)
print(len(train_trn))
print(train_trn['card1'].nunique() , train_trn['card2'].nunique() , train_trn['card3'].nunique() , train_trn['card5'].nunique())
 train_trn['card_n'] =(train_trn['card1'].astype(str)+ '_' + train_trn['card2'].astype(str)\
+ '_' + train_trn['card3'].astype(str)+ '_' + train_trn['card5'].astype(str))
print('unique cards:', train_trn['card_n'].nunique() )<sort_values>
X_train =X_train.loc[:,features_selection]
X_test = X_test.loc[:,features_selection]
train_trn.groupby(['card_n'])['isFraud'].mean().sort_values(ascending=False )<count_unique_values>
print('Original dataset shape %s' % Counter(y_train))
sampling_strategy= {4: 3148, 2: 3028, 3: 2444, 1: 2317, 0: 1000}
ros = RandomOverSampler(sampling_strategy= sampling_strategy, random_state=42)
 X_res, y_res = ros.fit_resample(X_train, y_train)
 print('Resampled dataset shape %s' % Counter(y_res))
train_trn['addr1'].nunique() , train_trn['addr2'].nunique()<count_values>
def cross_val(model,X_train,y_train):
X = X_train
y = y_train
coeff = np.empty(( 1,4))
cv_scores=[]
fold=1
skf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 print(skf.get_n_splits(X, y))
for train_index, val_index in skf.split(X, y):
xtrain, xvalid = X[train_index], X[val_index]
ytrain, yvalid = y[train_index], y[val_index]
model.fit(
 xtrain, ytrain,
eval_set=[(xvalid, yvalid)],
verbose=100,
early_stopping_rounds=100 
)
 valid_preds = model.predict(xvalid, num_iteration=model.best_iteration_)
 yvalid = np.array(yvalid ).tolist()
optR = OptimizedRounder()
optR.fit(valid_preds, yvalid)
 coefficients = optR.coefficients()
valid_p = optR.predict(valid_preds, coefficients)
 scr = quadratic_weighted_kappa(yvalid, valid_p)
 cv_scores.append(scr)
 print("QWK = {}.Coef = {}".format(scr, coefficients))
coefficients.reshape(( 4, 1))
coeff = np.vstack([coeff, coefficients])
 fold += 1
coeff = np.delete(coeff,(0), axis=0)
 global coefficient_mean
coefficient_mean = coeff.mean(axis=0)
 print("Coef Mean ={}".format(coefficient_mean))
train_trn['addr1'].value_counts(dropna=False ).to_frame().iloc[:10]<count_unique_values>
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'n_estimators': 1000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 2018
}
lgb_model = lgb.LGBMRegressor(**lgb_params)
 cross_val(lgb_model,X_res,y_res)
 y_pred=lgb_model.predict(X_train.values)
plotTrnCategoryRateBar('addr2', 10)
 print('addr2 nunique:', train_trn['addr2'].nunique() )<count_values>
print(coefficient_mean )
train_trn['addr2'].value_counts(dropna=False ).to_frame().iloc[:10]<count_missing_values>
optR=OptimizedRounder()
y_true = pd.DataFrame(y_train)
 y_true.reset_index(inplace=True,drop=False)
 predictions = optR.predict(y_pred, coefficient_mean ).astype(int)
 pred_lgb = pd.concat([pd.DataFrame(y_pred),y_true,pd.DataFrame(predictions)],axis=1,ignore_index=True)
 pred_lgb.columns = ['y_pred','index','y_true','y_pred_class']
pred_lgb['y_pred_class'].value_counts()
train_trn[vcols].isnull().sum() / len(train_trn )<create_dataframe>
evaluate(pred_lgb['y_pred_class'].tolist() , pred_lgb['y_true'].tolist() )
del train_trn_f0,train_trn_f1,train_id_f0,train_id_f1
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<load_from_csv>
y_pred=lgb_model.predict(X_test.values)
 predictions = optR.predict(y_pred, coefficient_mean ).astype(int)
 y_test =pd.DataFrame(y_test)
 y_test.reset_index(drop=False, inplace=True)
 pred_lgb = pd.concat([pd.DataFrame(y_pred),y_test,pd.DataFrame(predictions)],axis=1,ignore_index=True)
 pred_lgb.columns = ['y_pred','index','y_true','y_pred_class']
pred_lgb['y_pred_class'].value_counts()
train_id = pd.read_csv('.. /input/train_identity.csv')
 train_trn = pd.read_csv('.. /input/train_transaction.csv')
 test_id = pd.read_csv('.. /input/test_identity.csv')
 test_trn = pd.read_csv('.. /input/test_transaction.csv')
 id_cols = list(train_id.columns.values)
 trn_cols = list(train_trn.drop('isFraud', axis=1 ).columns.values)
 X_train = pd.merge(train_trn[trn_cols + ['isFraud']], train_id[id_cols], how='left')
 X_train = reduce_mem_usage(X_train)
 X_test = pd.merge(test_trn[trn_cols], test_id[id_cols], how='left')
 X_test = reduce_mem_usage(X_test)
 X_train_id = X_train.pop('TransactionID')
 X_test_id = X_test.pop('TransactionID')
 del train_id,train_trn,test_id,test_trn
all_data = X_train.append(X_test, sort=False ).reset_index(drop=True )<normalization>
evaluate(pred_lgb['y_pred_class'].tolist() , pred_lgb['y_true'].tolist() )
vcols = [f'V{i}' for i in range(1,340)]
sc = preprocessing.MinMaxScaler()
pca = PCA(n_components=2)
 vcol_pca = pca.fit_transform(sc.fit_transform(all_data[vcols].fillna(-1)))
 all_data['_vcol_pca0'] = vcol_pca[:,0]
all_data['_vcol_pca1'] = vcol_pca[:,1]
all_data['_vcol_nulls'] = all_data[vcols].isnull().sum(axis=1)
 all_data.drop(vcols, axis=1, inplace=True )<feature_engineering>
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'n_estimators': 1000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 20
}
lgb_model2 = lgb.LGBMRegressor(**lgb_params)
 lgb_model2.fit(X_res, y_res)
 y_pred2=lgb_model2.predict(X_test )
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 all_data['Date'] = all_data['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 all_data['_weekday'] = all_data['Date'].dt.dayofweek
all_data['_hour'] = all_data['Date'].dt.hour
all_data['_day'] = all_data['Date'].dt.day
all_data['_weekday'] = all_data['_weekday'].astype(str)
 all_data['_hour'] = all_data['_hour'].astype(str)
 all_data['_weekday__hour'] = all_data['_weekday'] + all_data['_hour']
cnt_day = all_data['_day'].value_counts()
cnt_day = cnt_day / cnt_day.mean()
all_data['_count_rate'] = all_data['_day'].map(cnt_day.to_dict())
 all_data.drop(['TransactionDT','Date','_day'], axis=1, inplace=True )<data_type_conversions>
optR=OptimizedRounder()
predictions2 = optR.predict(y_pred2, coefficient_mean ).astype(int)
 pred_lgb2 = pd.concat([pd.DataFrame(y_pred2),y_true,pd.DataFrame(predictions2)],axis=1,ignore_index=True)
 pred_lgb2.columns = ['y_pred_lgb2','index','y_true','y_pred_class']
pred_lgb2['y_pred_class'].value_counts()
all_data['_P_emaildomain__addr1'] = all_data['P_emaildomain'] + '__' + all_data['addr1'].astype(str)
 all_data['_card1__card2'] = all_data['card1'].astype(str)+ '__' + all_data['card2'].astype(str)
 all_data['_card1__addr1'] = all_data['card1'].astype(str)+ '__' + all_data['addr1'].astype(str)
 all_data['_card2__addr1'] = all_data['card2'].astype(str)+ '__' + all_data['addr1'].astype(str)
 all_data['_card12__addr1'] = all_data['_card1__card2'] + '__' + all_data['addr1'].astype(str)
 all_data['_card_all__addr1'] = all_data['_card1__card2'] + '__' + all_data['addr1'].astype(str )<feature_engineering>
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'n_estimators': 1000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 2019
}
lgb_model3 = lgb.LGBMRegressor(**lgb_params)
 lgb_model3.fit(X_res, y_res)
 y_pred=lgb_model3.predict(X_test.values )
<categorify><EOS> 
optR=OptimizedRounder()
predictions = optR.predict(y_pred, coefficient_mean ).astype(int)
 pred_lgb3 = pd.concat([pd.DataFrame(y_pred),y_true,pd.DataFrame(predictions)],axis=1,ignore_index=True)
 pred_lgb3.columns = ['y_pred_lgb3','index','y_true','y_pred_class']
pred_lgb3['y_pred_class'].value_counts()