Kaggle competition: Titanic - Machine Learning from Disaster
rescuers=df_all0.groupby(by='RescuerID')['RescuerID'].count()
df_rescuers=pd.DataFrame(rescuers)
 df_rescuers.columns=['ResLev']
df_rescuers.reset_index(inplace=True)
 df_rescuers.head()<merge>
submission_format = pd.read_csv("/kaggle/input/titanic/gender_submission.csv" )
dfm=df_all0.merge(df_rescuers,on='RescuerID')
 numeric_cols=['Age','PhotoAmt','Quantity','Fee','DescriptionLength','ResLev']
categorical_cols=['Sterilized','FurLength','Breed1','State','AdoptionSpeed','Breed2','MaturitySize','Gender','Dewormed','Color1','Color2','Color3','Health']
cols=['PetID']+numeric_cols+categorical_cols
dfm=dfm[cols]
dfm.shape<merge>
train_data_orig = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data=train_data_orig.copy()
train_data.head()
dfm=dfm.merge(txt_data,on='PetID', how='left')
 categorical_cols=categorical_cols+['sent_language']
dfm.shape<merge>
test_data_orig = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data=test_data_orig.copy()
test_data.head()
n_svd=32
svd=des_svd_df.iloc[:,0:n_svd+3]
svd.drop('Description',axis=1,inplace=True)
 svd.drop('AdoptionSpeed',axis=1,inplace=True)
 dfm=dfm.merge(svd,on='PetID', how='left')
 dfm.shape<groupby>
distinct_classes,value_counts=np.unique(train_data.Survived,return_counts=True )
img_df1ad=img_df1a.groupby(['PetID'])['img_met_description'].apply(', '.join ).reset_index()<feature_engineering>
train_data.isnull().sum()
img_df1ad['img_met_description1']=img_df1ad['img_met_description'].apply(lambda s:s.split(',')).apply(set ).apply(','.join )<filter>
test_data.isnull().sum()
img_df1ad['img_met_description1'].loc[0]<filter>
train_data.drop('Cabin',axis=1,inplace=True)
 test_data.drop('Cabin',axis=1,inplace=True )
img_df1ad['img_met_description'].loc[0]<drop_column>
train_data.isnull().sum()
img_df1ad.drop('img_met_description',axis=1,inplace=True)
 img_df1ad.head()<import_modules>
test_data.isnull().sum()
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD<train_on_grid>
train_data.drop('PassengerId',axis=1,inplace=True )
def find_svd(df,txt_col_name,n_comp):
tfv = TfidfVectorizer(analyzer='word', stop_words = 'english', token_pattern=r'\b[a-zA-Z]\w+\b',
min_df=1, max_features=10000, strip_accents='unicode',
ngram_range=(1, 32), use_idf=1, smooth_idf=1, sublinear_tf=1,)
 corpus=list(df[txt_col_name])
 txt_trasf=tfv.fit_transform(corpus)
 svd = TruncatedSVD(n_components=n_comp, n_iter=10, tol=1.0)
 svd.fit(txt_trasf)
 txt_svd=svd.transform(txt_trasf)
 txt_svd_df=pd.DataFrame(txt_svd)
 return(txt_svd_df )<feature_engineering>
testPIds=test_data['PassengerId']
test_data.drop('PassengerId',axis=1,inplace=True )
nc=16
img_met_svd=find_svd(img_df1ad,'img_met_description1',nc)
 img_met_svd.columns=['SVD_'+str(c)for c in range(0,nc)]
img_met_svd['PetID']=img_df1ad['PetID']<groupby>
def change_gender_num(s):
if(s=='male'):
return 1
else:
return 0
train_data['Sex']=train_data['Sex'].apply(change_gender_num)
 test_data['Sex']=test_data['Sex'].apply(change_gender_num )
img_df1ar=img_met_svd
img_df1ar['img_met_score_max']=img_df1a.groupby(by=['PetID','ImageID'],as_index=False ).agg({'img_met_score': 'max'})['img_met_score']
img_df1ar['img_met_score_min']=img_df1a.groupby(by=['PetID','ImageID'],as_index=False ).agg({'img_met_score': 'min'})['img_met_score']
img_df1ar.head()<drop_column>
train_data['relatives']=train_data['SibSp']+train_data['Parch']+1
train_data.drop(['SibSp','Parch'],axis=1,inplace=True )
img_df1cr1=img_df1c[img_df1c['ImageID']==1]
img_df1cr1.drop('ImageID', axis=1, inplace=True)
 img_df1cr1.columns=['PetID','img_crp_x1','img_crp_y1','img_crp_conf1','img_crp_if1']
img_df1cr2=img_df1c[img_df1c['ImageID']==2]
img_df1cr2.drop('ImageID', axis=1, inplace=True)
 img_df1cr2.columns=['PetID','img_crp_x2','img_crp_y2','img_crp_conf2','img_crp_if2']
img_df1cr3=img_df1c[img_df1c['ImageID']==3]
img_df1cr3.drop('ImageID', axis=1, inplace=True)
 img_df1cr3.columns=['PetID','img_crp_x3','img_crp_y3','img_crp_conf3','img_crp_if3']
img_df1cr=img_df1cr1
img_df1cr.head()<merge>
test_data['relatives']=test_data['SibSp']+test_data['Parch']+1
test_data.drop(['SibSp','Parch'],axis=1,inplace=True )
img_df1cr=img_df1cr.merge(img_df1cr2,on='PetID')
 img_df1cr=img_df1cr.merge(img_df1cr3,on='PetID')
 img_df1cr.head()<count_missing_values>
len(train_data.Ticket.unique() )
img_df1cr.isna().sum()<correct_missing_values>
train_data.drop('Ticket',axis=1,inplace=True )
img_df1cr.fillna(-1, inplace=True )<groupby>
test_data.drop('Ticket',axis=1,inplace=True )
img_df1pg=img_df1p.groupby(by=['PetID','ImageID'],as_index=False ).agg({'img_par_score': 'max',
'img_par_red':'first',
'img_par_green':'first',
'img_par_blue':'first',
'img_par_pf':'first'})
 img_df1pg.head()<drop_column>
def get_title(name):
return name.strip().split(',')[1].split('.')[0]
train_data['title']=train_data['Name'].apply(get_title)
 train_data.drop('Name',axis=1,inplace=True)
 test_data['title']=test_data['Name'].apply(get_title)
 test_data.drop('Name',axis=1,inplace=True )
img_df1pr1=img_df1pg[img_df1pg['ImageID']==1]
img_df1pr1.drop('ImageID', axis=1, inplace=True)
 img_df1pr1.columns=['PetID','img_par_red1','img_par_green1','img_par_blue1','img_par_pf1','img_par_score1']
img_df1pr2=img_df1pg[img_df1pg['ImageID']==2]
img_df1pr2.drop('ImageID', axis=1, inplace=True)
 img_df1pr2.columns=['PetID','img_par_red2','img_par_green2','img_par_blue2','img_par_pf2','img_par_score2']
img_df1pr3=img_df1pg[img_df1pg['ImageID']==3]
img_df1pr3.drop('ImageID', axis=1, inplace=True)
 img_df1pr3.columns=['PetID','img_par_red3','img_par_green3','img_par_blue3','img_par_pf3','img_par_score3']
img_df1pr=img_df1pr1
img_df1pr.head()<merge>
matplotlib.rcParams['figure.figsize']=(15,10 )
img_df1pr=img_df1pr.merge(img_df1pr2,on='PetID')
 img_df1pr=img_df1pr.merge(img_df1pr3,on='PetID')
 img_df1pr.head()<merge>
( keys,values)=np.unique(train_data[train_data.Survived==1].title,return_counts=True )
img=img_df1ar
img=img.merge(img_df1cr,on=['PetID'], how='left')
 img=img.merge(img_df1pr,on=['PetID'], how='left')
 img.shape<correct_missing_values>
train_data.Age.fillna(train_data.Age.mean() ,inplace=True )
img.fillna(-1,inplace=True)
 img.head()<merge>
test_data.Age.fillna(test_data.Age.mean() ,inplace=True )
dfm=dfm.merge(img,on='PetID', how='left')
 dfm.shape<data_type_conversions>
train_data.isnull().sum()
df_all['sent_magnitude'].fillna(-1, inplace=True)
 df_all['sent_score'].fillna(-1, inplace=True)
 df_all['sent_language'].fillna('en', inplace=True )<correct_missing_values>
test_data.isnull().sum()
df_all.fillna(-1, inplace=True )<data_type_conversions>
train_data.fillna(train_data.Embarked.value_counts().idxmax() ,inplace=True )
df_all[categorical_cols]=df_all[categorical_cols].apply(lambda c : c.astype('category'))<save_to_csv>
test_data.fillna(test_data.Fare.mean() ,inplace=True )
df_all.to_csv('df_all.csv')
 df_all.head()<filter>
survived_train_data=train_data[train_data.Survived==1]
df_all.dtypes[df_all.dtypes=='object']<filter>
( keys,values)=np.unique(survived_train_data.Embarked,return_counts=True )
dftrain=df_all[np.invert(df_all['AdoptionSpeed']==-1)].copy()
dftest=df_all[df_all['AdoptionSpeed']==-1].copy()<drop_column>
dummies=pd.get_dummies(train_data.Embarked )
dftrain = dftrain.drop(['PetID'],axis=1)
 dftest = dftest.drop(['PetID'],axis=1 )<prepare_x_and_y>
dummies_test=pd.get_dummies(test_data.Embarked )
XT = dftest.drop('AdoptionSpeed',axis=1)
 y = dftrain['AdoptionSpeed']
X = dftrain.drop('AdoptionSpeed',axis=1 )<import_modules>
train_data2=pd.concat([train_data,dummies],axis='columns' )
from sklearn.model_selection import train_test_split
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import KFold<import_modules>
test_data2=pd.concat([test_data,dummies_test],axis='columns' )
import lightgbm as lgb<define_variables>
train_data2.drop('Embarked',axis=1,inplace=True )
cat_features=[x for x in categorical_cols if x!='AdoptionSpeed']<init_hyperparams>
test_data2.drop('Embarked',axis=1,inplace=True )
parameters = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'max_bin' : 8,
'num_leaves': 12,
'max_depth': 4,
'learning_rate': 0.01,
'bagging_fraction': 0.8,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 128,
'min_child_weight': 0.1,
'data_random_seed': 123,
'verbosity': -1,
'early_stopping_rounds': 50,
'num_rounds': 3000}
evals_result={}<compute_test_metric>
thresh_title_counts=10
title_names=(train_data2.title.value_counts() < thresh_title_counts )
def qks(a,b):
return cohen_kappa_score(np.round(a), np.round(b), weights='quadratic' )<split>
train_data2['title']=train_data2.title.apply(lambda x:'Misc' if title_names.loc[x]==True else x )
kf_splits=10
k_fold = KFold(n_splits=kf_splits, shuffle=True)
 k=0
df_qks=pd.DataFrame(columns=['best_round','qks_train','qks_valid'])
 df_y=pd.DataFrame(index=XT.index)
 perf_list=[]
for train_idx, valid_idx in k_fold.split(X,y):
k=k+1
print('Step k={}'.format(k))
X_train = X.iloc[train_idx, :]
X_valid = X.iloc[valid_idx, :]
y_train = y.iloc[train_idx]
y_valid = y.iloc[valid_idx]
lgb_train_set = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features,free_raw_data=False)
 lgb_valid_set = lgb.Dataset(X_valid, label=y_valid,free_raw_data=False)
 lgbm=lgb.LGBMRegressor()
lgbm = lgb.train(parameters,
train_set=lgb_train_set,
valid_sets=[lgb_train_set,lgb_valid_set],
verbose_eval=100,
evals_result=evals_result)
 best_round=lgbm.best_iteration
y_train_pred = lgbm.predict(X_train,num_iteration=best_round)
 y_valid_pred = lgbm.predict(X_valid,num_iteration=best_round)
 y_test_pred = lgbm.predict(XT,num_iteration=best_round)
 qks1=qks(y_train_pred,y_train)
 qks2=qks(y_valid_pred,y_valid)
 df_qks.loc[k]=[best_round,qks1,qks2]
perf_list=perf_list+[evals_result]
df_y[k]=y_test_pred<count_values>
distinct_keys=list(train_data2['title'].value_counts().keys() )
df_all[df_all['AdoptionSpeed'].astype(int)>=0]['AdoptionSpeed'].value_counts()<count_values>
distinct_keys.remove('Misc' )
sum(ym>3.5 )<statistical_test>
test_data2['title']=test_data2.title.apply(lambda x:'Misc' if x not in distinct_keys else x )
def distrib_err(coef,test_proba,train_label):
test_predictions = pd.cut(test_proba, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 N_CLASS=5
freq_train=np.zeros(N_CLASS)
 freq_test=np.zeros(N_CLASS)
 delta_freq=np.zeros(N_CLASS)
 for i in range(0,N_CLASS):
freq_train[i]=100*Counter(train_label)[i]/len(train_label)
 freq_test[i]=100*Counter(test_predictions)[i]/len(test_predictions)
 delta_freq[i]=freq_test[i]-freq_train[i]
return np.sum(delta_freq**2 )<find_best_params>
test_data2['title'].value_counts()
initial_coef = [2.0, 2.5, 3.0, 3.5]
distrib_err_partial = partial(distrib_err, test_proba=ym, train_label=y)
 final_coef = sp.optimize.minimize(distrib_err_partial, initial_coef, method='nelder-mead')
 final_coef<categorify>
dummies_embarked_train=pd.get_dummies(train_data2['title'] )
def apply_lim(y_calc,limits):
y_round=np.zeros(len(y_calc))
for i,yc in enumerate(y_calc):
if(yc<=limits[0]):
y_round[i]=0
if(( yc>limits[0])&(yc<=limits[1])) :
y_round[i]=1
if(( yc>limits[1])&(yc<=limits[2])) :
y_round[i]=2
if(( yc>limits[2])&(yc<=limits[3])) :
y_round[i]=3
if(yc>limits[3]):
y_round[i]=4
return y_round<compute_test_metric>
dummies_embarked_test=pd.get_dummies(test_data2['title'] )
y_test_pred_r = apply_lim(ym,final_coef['x'] )<data_type_conversions>
train_data3=pd.concat([train_data2,dummies_embarked_train],axis='columns' )
y_pred = y_test_pred_r.astype('int' )<prepare_output>
test_data3=pd.concat([test_data2,dummies_embarked_test],axis='columns' )
subm=pd.DataFrame({'PetID': dftest_ids,'AdoptionSpeed': y_pred})
 subm.head()<count_values>
train_data3.drop('title',axis=1,inplace=True )
subm['AdoptionSpeed'].value_counts()<save_to_csv>
test_data3.drop('title',axis=1,inplace=True )
subm.to_csv('submission.csv', index=False )<import_modules>
train_data3.isnull().sum()
np.random.seed(369 )<compute_test_metric>
test_data3.isnull().sum()
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
yTrain=train_data3.Survived
train_data4=train_data3.drop('Survived',axis=1)
 xTrain=train_data4.values
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
xTest=test_data3.values
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_csv>
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix,classification_report
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )<prepare_x_and_y>
xTrain_small,xTest_small,yTrain_small,yTest_small=train_test_split(xTrain,yTrain )
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score 
<train_on_grid>
clf_best_fit=SVC(kernel='linear' )
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=200)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 print("train:", train.shape )<train_on_grid>
clf=SVC(kernel='linear' )
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1)
 print("train:", train.shape )<prepare_x_and_y>
clf.fit(xTrain_small,yTrain_small )
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<drop_column>
clf.score(xTest_small,yTest_small )
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
yTest_predicted_small=clf.predict(xTest_small )
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<filter>
confusion_matrix(yTest_small,yTest_predicted_small )
foo = train.dtypes
cat_feature_names = foo[foo == "category"]
cat_features = [train.columns.get_loc(c)for c in train.columns if c in cat_feature_names]<find_best_model_class>
print(classification_report(yTest_small,yTest_predicted_small))
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 8,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
categorical_feature=list(cat_features),
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
yPredicted=clf.predict(xTest )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.64
coefficients_[1] = 2.11
coefficients_[3] = 2.85
train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<compute_train_metric>
clf = RandomForestClassifier(random_state=0)
 clf.fit(xTrain_small,yTrain_small )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<count_values>
clf.score(xTest_small,yTest_small )
print("True Distribution:")
 print(pd.value_counts(target, normalize=True ).sort_index())
 print("Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index())
 print("Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index())
<create_dataframe>
clf_logistic_regression=LogisticRegression(max_iter=1000)
 clf_logistic_regression.fit(xTrain_small,yTrain_small )
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
clf.score(xTest_small,yTest_small )
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
final_dataFrame=pd.DataFrame()
final_dataFrame['PassengerId']=testPIds
submission.to_csv('submission.csv', index=False )<save_to_csv>
from sklearn.neural_network import MLPClassifier
submission.to_csv('submission.csv', index=False )<set_options>
clf=MLPClassifier(hidden_layer_sizes=(20,10),max_iter=1000,activation='logistic')
 clf.fit(xTrain_small,yTrain_small )
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic' )<load_from_csv>
clf.score(xTest_small,yTest_small )
breeds = pd.read_csv('.. /input/breed_labels.csv')
 colors = pd.read_csv('.. /input/color_labels.csv')
 states = pd.read_csv('.. /input/state_labels.csv')
 train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sub = pd.read_csv('.. /input/test/sample_submission.csv')
 train['dataset_type'] = 'train'
test['dataset_type'] = 'test'
all_data = pd.concat([train, test] )<set_options>
yPredicted_MLP_Small=clf.predict(xTest_small )
ax.patches<count_values>
print(classification_report(yTest_small,yPredicted_MLP_Small))
print('Most popular pet names and AdoptionSpeed')
 for n in train['Name'].value_counts().index[:5]:
print(n)
 print(train.loc[train['Name'] == n, 'AdoptionSpeed'].value_counts().sort_index())
 print('' )<feature_engineering>
print(confusion_matrix(yTest_small,yPredicted_MLP_Small))
train['Name'] = train['Name'].fillna('Unnamed')
 test['Name'] = test['Name'].fillna('Unnamed')
 all_data['Name'] = all_data['Name'].fillna('Unnamed')
 train['No_name'] = 0
train.loc[train['Name'] == 'Unnamed', 'No_name'] = 1
test['No_name'] = 0
test.loc[test['Name'] == 'Unnamed', 'No_name'] = 1
all_data['No_name'] = 0
all_data.loc[all_data['Name'] == 'Unnamed', 'No_name'] = 1
print(f"Rate of unnamed pets in train data: {train['No_name'].sum() * 100 / train['No_name'].shape[0]:.4f}%.")
 print(f"Rate of unnamed pets in test data: {test['No_name'].sum() * 100 / test['No_name'].shape[0]:.4f}%." )<count_values>
yPredicted_MLP=clf.predict(xTest )
all_data[all_data['Name'].apply(lambda x: len(str(x)))== 3]['Name'].value_counts().tail()<count_values>
final_dataFrame['Survived']=yPredicted_MLP
train['Age'].value_counts().head(10 )<feature_engineering>
final_dataFrame.to_csv('titanic.csv', index=False)
 print("Your submission was successfully saved!" )