Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<import_modules>
Dropout, PReLU, LeakyReLU, Embedding
Embedding, SpatialDropout1D, concatenate, CuDNNGRU, CuDNNLSTM, Dropout, TimeDistributed, Activation
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
from sklearn.model_selection import train_test_split, validation_curve, learning_curve, GridSearchCV
from sklearn.linear_model import LinearRegression, Ridge, Lasso, LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor, plot_tree
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor, ExtraTreesClassifier
from sklearn.ensemble import BaggingClassifier, BaggingRegressor, GradientBoostingClassifier, GradientBoostingRegressor
from xgboost import XGBClassifier, XGBRegressor
from sklearn.metrics import log_loss, roc_auc_score, accuracy_score, precision_score, recall_score, roc_curve, precision_recall_curve, auc, make_scorer, confusion_matrix, f1_score, fbeta_score
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score 
<load_from_csv>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 len_train = len(train )
train_data = pd.read_csv(".. /input/titanic/train.csv")
 test_data = pd.read_csv(".. /input/titanic/test.csv")
 train_data.groupby('Survived')['PassengerId'].nunique()
train_data.groupby('Pclass')['PassengerId'].nunique() 
<categorify>
HD_cate = [
'Type',
'Breed1', 'Breed2', 'Color1', 'Color2', 'Color3', 'FurLength',
'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'Quantity', 'State',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName', 'breed_id_cross'
,'breed_type_cross'
]
keep_cate = [
'Type',
'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2', 'Color3',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health', 'Quantity', 'State',
'main_breed_Type', 'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName', 'breed_id_cross'
,'breed_type_cross'
]
keep_float = [
'Age',
'MaturitySize',
'Fee',
'VideoAmt', 'PhotoAmt','sentiment_magnitude',
'sentiment_score', 'sentiment_magnitude_sum', 'sentiment_score_sum', 'sentiment_magnitude_mean', 'sentiment_score_mean',
'sentiment_magnitude_var', 'sentiment_score_var', 'metadata_annots_score_SUM', 'metadata_annots_score_MEAN', 'metadata_annots_score_VAR',
'metadata_color_score_SUM', 'metadata_color_score_MEAN', 'metadata_color_score_VAR', 'metadata_color_pixelfrac_SUM', 'metadata_color_pixelfrac_MEAN',
'metadata_color_pixelfrac_VAR', 'metadata_crop_conf_SUM', 'metadata_crop_conf_MEAN', 'metadata_crop_conf_VAR', 'metadata_crop_importance_SUM',
'metadata_crop_importance_MEAN', 'metadata_crop_importance_VAR',
'TFIDF_Description_0', 'TFIDF_Description_1', 'TFIDF_Description_2', 'TFIDF_Description_3', 'TFIDF_Description_4',
'TFIDF_Description_5', 'TFIDF_Description_6', 'TFIDF_Description_7', 'TFIDF_Description_8', 'TFIDF_Description_9',
'TFIDF_Description_10', 'TFIDF_Description_11', 'TFIDF_Description_12', 'TFIDF_Description_13', 'TFIDF_Description_14',
'TFIDF_Description_15', 'TFIDF_metadata_annots_top_desc_0', 'TFIDF_metadata_annots_top_desc_1', 'TFIDF_metadata_annots_top_desc_2',
'TFIDF_metadata_annots_top_desc_3', 'TFIDF_metadata_annots_top_desc_4', 'TFIDF_metadata_annots_top_desc_5', 'TFIDF_metadata_annots_top_desc_6',
'TFIDF_metadata_annots_top_desc_7', 'TFIDF_metadata_annots_top_desc_8', 'TFIDF_metadata_annots_top_desc_9',
'TFIDF_metadata_annots_top_desc_10', 'TFIDF_metadata_annots_top_desc_11', 'TFIDF_metadata_annots_top_desc_12',
'TFIDF_metadata_annots_top_desc_13', 'TFIDF_metadata_annots_top_desc_14', 'TFIDF_metadata_annots_top_desc_15',
'TFIDF_sentiment_entities_0', 'TFIDF_sentiment_entities_1', 'TFIDF_sentiment_entities_2', 'TFIDF_sentiment_entities_3',
'TFIDF_sentiment_entities_4', 'TFIDF_sentiment_entities_5', 'TFIDF_sentiment_entities_6', 'TFIDF_sentiment_entities_7',
'TFIDF_sentiment_entities_8', 'TFIDF_sentiment_entities_9', 'TFIDF_sentiment_entities_10', 'TFIDF_sentiment_entities_11',
'TFIDF_sentiment_entities_12', 'TFIDF_sentiment_entities_13', 'TFIDF_sentiment_entities_14', 'TFIDF_sentiment_entities_15',
'IMG_SVD_0', 'IMG_SVD_1', 'IMG_SVD_2', 'IMG_SVD_3', 'IMG_SVD_4', 'IMG_SVD_5', 'IMG_SVD_6', 'IMG_SVD_7', 'IMG_SVD_8', 'IMG_SVD_9', 'IMG_SVD_10', 'IMG_SVD_11',
'IMG_SVD_12', 'IMG_SVD_13', 'IMG_SVD_14', 'IMG_SVD_15', 'IMG_SVD_16', 'IMG_SVD_17', 'IMG_SVD_18', 'IMG_SVD_19', 'IMG_SVD_20', 'IMG_SVD_21', 'IMG_SVD_22',
'IMG_SVD_23', 'IMG_SVD_24', 'IMG_SVD_25', 'IMG_SVD_26', 'IMG_SVD_27', 'IMG_SVD_28', 'IMG_SVD_29', 'IMG_SVD_30', 'IMG_SVD_31',
'average_blue_sum', 'average_blue_mean', 'average_blue_var', 'average_green_sum', 'average_green_mean', 'average_green_var',
'average_red_sum', 'average_red_mean', 'average_red_var', 'img_average_sum', 'img_average_mean', 'img_average_var',
'blur_sum', 'blur_mean', 'blur_var', 'image_size_sum', 'image_size_mean', 'image_size_var', 'width_sum', 'width_mean',
'width_var', 'height_sum', 'height_mean', 'height_var',
'RescuerID_COUNT', 'Length_Description', 'Length_metadata_annots_top_desc', 'Lengths_sentiment_entities',
'Length_Description_2', 'Length_metadata_annots_top_desc_2', 'Lengths_sentiment_entities_2',
'resultion','ppi','txt_pred','img_pred','tablur_pred'
]
train = train_data.copy()
train['Cab'] = pd.Series(['cab' + str(i)[0] for i in train.Cabin], index=train.index)
 del train['PassengerId']
del train['Name']
del train['Ticket']
del train['Cabin']
del train['Parch']
dummy = pd.get_dummies(train['Sex'])
 train = train.join(dummy)
 del train['Sex']
dummy = pd.get_dummies(train['Embarked'])
 train = train.join(dummy)
 del train['Embarked']
del train['Cab']
train["Age"].fillna(train["Age"].mean() , inplace=True)
 print(train.head())
 print(train.info() )<categorify>
def agg_helper(X, base, other):
tmp2 = X.groupby(base ).agg({other:['max','min','mean']})
 tmp2.columns = pd.Index([base +'_' + e[0] + "_" + e[1].upper() for e in tmp2.columns.tolist() ])
 X = X.merge(tmp2, on = [base],how = 'left')
 return X
test = test_data.copy()
del test['PassengerId']
del test['Name']
del test['Ticket']
del test['Cabin']
del test['Parch']
dummy = pd.get_dummies(test['Sex'])
 test = test.join(dummy)
 del test['Sex']
dummy = pd.get_dummies(test['Embarked'])
 test = test.join(dummy)
 del test['Embarked']
test["Age"].fillna(test["Age"].mean() , inplace=True)
 test["Fare"].fillna(test["Fare"].mean() , inplace=True)
 test["SibSp"].fillna(0, inplace=True)
 print(test.info() )<split>
from keras.applications.densenet import preprocess_input, DenseNet121
X = train.drop(['Survived'], axis=1)
 y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.05,
random_state=4 )<train_model>
img_size = 256
batch_size = 256
XGBC_model = XGBClassifier(max_depth= 4, n_estimators=20)
 XGBC_model.fit(X_train, y_train)
 print('XGBC_model Training score:', XGBC_model.score(X_train,y_train))
print('XGBC_model Test score: ', XGBC_model.score(X_test,y_test),'
')
 extratree_model = ExtraTreesClassifier(max_depth=5)
 extratree_model.fit(X_train, y_train)
 print('ExtraTrees Training score:', extratree_model.score(X_train,y_train))
print('ExtraTrees Test score: ', extratree_model.score(X_test,y_test),'
')
 bagging_model = BaggingClassifier(n_estimators = 10, random_state=0)
 bagging_model.fit(X_train, y_train)
 print('Bagging Training score:', bagging_model.score(X_train,y_train))
print('Bagging Test score: ', bagging_model.score(X_test,y_test),'
')
 decision_tree_model = DecisionTreeClassifier(random_state=0, max_depth = 5)
 decision_tree_model.fit(X_train, y_train)
 print('Decision Tree Training score:', decision_tree_model.score(X_train,y_train))
print('Decision Tree Test score: ', decision_tree_model.score(X_test,y_test),'
')
 random_forest_model = RandomForestClassifier(random_state=0, max_depth = 5)
 random_forest_model.fit(X_train, y_train)
 print('Random Forest Training score:', random_forest_model.score(X_train,y_train))
print('Random Forest Test score: ', random_forest_model.score(X_test,y_test),'
')
 gradient_boosting_model = GradientBoostingClassifier(random_state= 0, n_estimators = 150, max_depth = 2)
 gradient_boosting_model.fit(X_train, y_train)
 print('Gradient Boosting Training score:', gradient_boosting_model.score(X_train,y_train))
print('Gradient Bosting Test score: ', gradient_boosting_model.score(X_test,y_test),'
' )<compute_train_metric>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
scores = cross_val_score(estimator=XGBC_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
 scores = cross_val_score(estimator=extratree_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
 scores = cross_val_score(estimator=bagging_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
 scores = cross_val_score(estimator=decision_tree_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
 scores = cross_val_score(estimator=random_forest_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
 scores = cross_val_score(estimator=gradient_boosting_model, X=X, y=y, cv=10)
 print(scores, scores.mean() , scores.std() , "
")
<predict_on_test>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
survived1 = gradient_boosting_model.predict(test)
 survived2 = random_forest_model.predict(test)
 survived3 = decision_tree_model.predict(test)
 survived4 = extratree_model.predict(test)
 survived5 = bagging_model.predict(test)
 survived6 = XGBC_model.predict(test)
 survived =(survived1 + survived2 + survived3 + survived4 + survived5 + survived6)/6
s = []
for i in survived:
if i >= 0.5:
s.append(1)
 else:
s.append(0)
 passenger_id = test_data['PassengerId']
d = pd.DataFrame(columns = ['PassengerId', 'Survived'])
 d['PassengerId'] = passenger_id
d['Survived'] = s
print(d)
 d.groupby('Survived')['PassengerId'].nunique()
d.to_csv('gradient_boosting.csv',index=False )<load_from_csv>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
train_data=pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data=pd.read_csv("/kaggle/input/titanic/test.csv" )<count_missing_values>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train_data.isna().sum()<count_missing_values>
del m
gc.collect()
test_data.isna().sum()<drop_column>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
train_data.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked'], inplace=True, axis=1)
 test_data.drop([ 'Name', 'Ticket', 'Cabin', 'Embarked'], inplace=True, axis=1 )<data_type_conversions>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
train_data['Age'].fillna(24.0,inplace=True)
<data_type_conversions>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
test_data['Age'].fillna(24.0,inplace=True )<count_missing_values>
features_df = pd.concat([train_feats, test_feats], axis=0 )
train_data.isna().sum()<set_options>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
test_data['Fare'].fillna(7.75,inplace=True )<count_missing_values>
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
test_data.isna().sum()<categorify>
img_row = pd.concat([all_ids.reset_index(drop = True), features_df[[f'pic_{i}' for i in range(256)]].reset_index(drop = True)], axis=1 )
genderMap={
'male':0,
'female':1
}
train_data.Sex=train_data.Sex.map(genderMap)
 test_data.Sex=test_data.Sex.map(genderMap )<prepare_x_and_y>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
X=np.array(train_data.drop('Survived',axis=1))
y=np.array(train_data['Survived'])
 X.shape<split>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=47 )<train_model>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_max': file_sentences_sentiment['magnitude'].max(axis=0),
'score_max': file_sentences_sentiment['score'].max(axis=0),
'magnitude_min': file_sentences_sentiment['magnitude'].min(axis=0),
'score_min': file_sentences_sentiment['score'].min(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 try:
metadata_file = pet_parser.open_json_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
if dfs_metadata:
dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
lr=LogisticRegression(solver='liblinear',multi_class='ovr')
 lr.fit(X_train,y_train )<compute_test_metric>
aggregates = ['sum', 'mean', 'var', 'min', 'max']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
lr.score(X_test,y_test )<predict_on_test>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
test_data['Survived']=lr.predict(test_data.drop(['PassengerId'],axis=1))<load_from_csv>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
pd.read_csv('/kaggle/input/titanic/gender_submission.csv' )<prepare_output>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
Submission=test_data[['PassengerId','Survived']]
Submission.set_index('PassengerId',inplace=True )<save_to_csv>
X['breed_type_cross'] = X['main_breed_BreedName'].fillna('none')+'_' + X['second_breed_BreedName'].fillna('none')
 X['breed_id_cross'] = X['main_breed_Type'].fillna('none' ).astype(str)+'_' + X['second_breed_Type'].fillna('none' ).astype(str )
Submission.to_csv('Submission.csv' )<set_options>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName','breed_id_cross','breed_type_cross']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
%matplotlib inline<load_from_csv>
def no_name(x):
if x == 'n' or 'No_name' in x:
return 1
return 0
X_temp['Name'] = X_temp['Name'].fillna('n')
 X_temp['Name'] = X_temp['Name'].apply(lambda x:no_name(str(x)))
 X_temp['Name_length'] = X_temp['Name'].apply(lambda x:len(str(x)))
 X_temp['Name_length_split'] = X_temp['Name'].apply(lambda x:len(str(x ).split()))
def have_num(x):
if any(char.isdigit() for char in x):
return 1
else:
return 0
X_temp['Name_length_if_num'] = X_temp['Name'].apply(lambda x:have_num(str(x)))
 def kitty(x):
if 'kitty' in x.lower() :
return 1
else:
return 0
X_temp['Name_if_kitty'] = X_temp['Name'].apply(lambda x:kitty(str(x)))
 def puppy(x):
if 'puppy' in x.lower() :
return 1
else:
return 0
X_temp['Name_if_puppy'] = X_temp['Name'].apply(lambda x:puppy(str(x)))
train= pd.read_csv('/kaggle/input/titanic/train.csv' )<categorify>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass==1:
return 37
elif Pclass == 2:
return 29
else :
return 24
else:
return Age<feature_engineering>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
train['Age']=train[['Age','Pclass']].apply(impute_age,axis = 1 )<drop_column>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
train.drop('Cabin',inplace = True,axis =1)
<categorify>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len)
 X_TXT = X_text[['Description', 'metadata_annots_top_desc', 'sentiment_entities']].copy()
sex = pd.get_dummies(train['Sex'],drop_first=True)
 embark = pd.get_dummies(train['Embarked'],drop_first=True )<concatenate>
X_temp['Length_Description_2'] = X_text['Description'].apply(lambda x: len(x.split()))
X_temp['Length_metadata_annots_top_desc_2'] = X_text['metadata_annots_top_desc'].apply(lambda x: len(x.split()))
X_temp['Lengths_sentiment_entities_2'] = X_text['sentiment_entities'].apply(lambda x: len(x.split()))
train = pd.concat([train,sex,embark],axis=1 )<drop_column>
def spacing_punctuation(text):

 regular_punct = list(string.punctuation)
 extra_punct = [
',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤']
all_punct = ''.join(sorted(list(set(regular_punct + extra_punct))))
re_tok = re.compile(f'([{all_punct}])')
 return re_tok.sub(r' \1 ', text)
 def tokenize(df_text, max_features):
df_text = df_text.apply(spacing_punctuation)
 tokenizer = Tokenizer(
 num_words=max_features,
filters='',
lower=False,
split=' ')
 tokenizer.fit_on_texts(list(df_text))
return tokenizer
train.drop(['Sex','Embarked','PassengerId','Name','Ticket'],axis=1,inplace=True )<normalization>
df_text = pd.concat([X_TXT['Description'], X_TXT['metadata_annots_top_desc'],X_TXT['sentiment_entities']],axis=0 ).reset_index(drop=True)
 tokenizer = tokenize(df_text, max_features=160000 )
st = StandardScaler()<categorify>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.005,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test
feature_scale = ['Age','Fare']
train[feature_scale] = st.fit_transform(train[feature_scale])
<prepare_x_and_y>
tabular_temp = X_temp[[i for i in X_temp.columns if i not in to_drop_columns +['Description','metadata_annots_top_desc', 'sentiment_entities'] ] ]
for i in HD_cate:
if i in tabular_temp.columns:
tabular_temp[i] = tabular_temp.groupby(i)[i].transform('count')
 X_tabular_temp = tabular_temp[len_train:].drop(['AdoptionSpeed'], axis = 1 )
x = train.drop(['Survived'],axis=1)
 y = train['Survived']<import_modules>
tablur_train, tablur_test = run_xgb(xgb_params,tabular_temp[:len_train], X_tabular_temp)
 X_temp['tablur_pred'] = np.concatenate([tablur_train,tablur_test.mean(axis = 1)] )
from sklearn.model_selection import GridSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier<train_model>
del tabular_temp, X_tabular_temp
gc.collect()
tree = DecisionTreeClassifier()
tree.fit(x,y)
 tree.score(x,y )<load_from_csv>
print('Tfidf word vector')
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 3),
max_features=20000)
 word_vectorizer.fit(df_text)
 train_desc_word_features = word_vectorizer.transform(X_TXT[:len_train]['Description'])
 test_desc_word_features = word_vectorizer.transform(X_TXT[len_train:]['Description'])
 train_meta_word_features = word_vectorizer.transform(X_TXT[:len_train]['metadata_annots_top_desc'])
 test_meta_word_features = word_vectorizer.transform(X_TXT[len_train:]['metadata_annots_top_desc'])
 train_sent_word_features = word_vectorizer.transform(X_TXT[:len_train]['sentiment_entities'])
 test_sent_word_features = word_vectorizer.transform(X_TXT[len_train:]['sentiment_entities'])
 print('Tfidf char vector')
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 5),
max_features=50000)
 char_vectorizer.fit(df_text)
 train_desc_char_features = char_vectorizer.transform(X_TXT[:len_train]['Description'])
 test_desc_char_features = char_vectorizer.transform(X_TXT[len_train:]['Description'])
 train_meta_char_features = char_vectorizer.transform(X_TXT[:len_train]['metadata_annots_top_desc'])
 test_meta_char_features = char_vectorizer.transform(X_TXT[len_train:]['metadata_annots_top_desc'])
 train_sent_char_features = char_vectorizer.transform(X_TXT[:len_train]['sentiment_entities'])
 test_sent_char_features = char_vectorizer.transform(X_TXT[len_train:]['sentiment_entities'])
 train_features = hstack([
train_desc_word_features, train_desc_char_features,
train_meta_word_features, train_meta_char_features,
train_sent_word_features, train_sent_char_features, ] ).tocsr()
test_features = hstack([
test_desc_word_features, test_desc_char_features,
test_meta_word_features, test_meta_char_features,
test_sent_word_features, test_sent_char_features, ] ).tocsr()
def run_txt(X_train, X_test):
n_splits = 10
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(train_features, X_train['AdoptionSpeed'].values):
X_tr,X_val= train_features[train_idx], train_features[valid_idx]
y_tr = X_train.iloc[train_idx, :]['AdoptionSpeed'].values
y_val = X_train.iloc[valid_idx, :]['AdoptionSpeed'].values
model = Ridge(alpha=20, copy_X=True, fit_intercept=True, solver='auto',max_iter=100,normalize=False, random_state=0, tol=0.0025)
 model.fit(X_tr,y_tr)
 valid_pred = model.predict(X_val)
 test_pred = model.predict(test_features)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
oof_test = oof_test.mean(1)
 return oof_train, oof_test
test = pd.read_csv('/kaggle/input/titanic/test.csv' )<create_dataframe>
oof_train_txt, oof_test_txt = run_txt(X_temp[:len_train][['AdoptionSpeed']], X_temp[len_train:][['AdoptionSpeed']])
 X_temp['txt_pred'] = np.concatenate([oof_train_txt,oof_test_txt] )
test2 = test.copy()<feature_engineering>
del train_features, test_features, train_desc_word_features,test_desc_char_features,train_meta_char_features,test_meta_char_features
del train_sent_char_features,test_desc_word_features, test_meta_word_features, test_sent_word_features, test_sent_char_features,
gc.collect()
test['Age']=test[['Age','Pclass']].apply(impute_age,axis = 1 )<drop_column>
def load_word_embedding(filepath):

 def _get_vec(word, *arr):
return word, np.asarray(arr, dtype='float32')
 print('load word embedding...... ')
 try:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))
except UnicodeDecodeError:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(
 filepath, encoding="utf8", errors='ignore'))
words_to_del = []
for word, vec in word_embedding.items() :
if len(vec)!= 300:
words_to_del.append(word)
 for word in words_to_del:
del word_embedding[word]
return word_embedding
test.drop('Cabin',inplace = True,axis =1 )<categorify>
def create_embedding_weights(word_index, word_embedding,
max_features, paragram=False):
print('create word embedding weights...... ')
 count = 0
stemmer = PorterStemmer()
mat_embedding = np.stack(word_embedding.values())
 a, b = min(max_features, len(word_index)) + 1, mat_embedding.shape[1]
print('embedding weights matrix with shape:({}, {})'.format(a, b))
embedding_mean, embedding_std = mat_embedding.mean() , mat_embedding.std()
del mat_embedding
gc.collect()
embedding_weights = np.random.normal(embedding_mean, embedding_std,(a, b))
for word, idx in word_index.items() :
if idx >= a:
continue
word_vec = word_embedding.get(word, None)
 if word_vec is None:
word_vec = word_embedding.get(word.lower() , None)
 if word_vec is None:
word_vec = word_embedding.get(stemmer.stem(word), None)
 if word_vec is not None:
count += 1
if word_vec is not None:
embedding_weights[idx] = word_vec
print("{} number has been replaced".format(count))
return embedding_weights
sex = pd.get_dummies(test['Sex'],drop_first=True)
 embark = pd.get_dummies(test['Embarked'],drop_first=True )<concatenate>
GLOVE_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
glove_word_embed = load_word_embedding(GLOVE_PATH )
test = pd.concat([test,sex,embark],axis=1 )<drop_column>
embedding_matrix = create_embedding_weights(tokenizer.word_index, glove_word_embed, 160000, False )
test.drop(['Sex','Embarked','PassengerId','Name','Ticket'],axis=1,inplace=True )<correct_missing_values>
del glove_word_embed
gc.collect()
test['Fare'].fillna(test['Fare'].mean() ,inplace=True )<categorify>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='char',
ngram_range=(2, 5), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_char_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
feature_scale = ['Age','Fare']
test[feature_scale] = st.fit_transform(test[feature_scale] )<import_modules>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier<choose_model_class>
X_img_row_temp = X_temp[['PetID','AdoptionSpeed']].merge(img_row, how='left', on='PetID' )
level1 = LogisticRegression()
model = StackingClassifier(estimators=level0,final_estimator=level1,cv=5 )<choose_model_class>
X_img_row_temp = X_img_row_temp.drop(['PetID'], axis = 1)
 X_text_image = X_img_row_temp[len_train:].drop(['AdoptionSpeed'], axis = 1 )
level1 = LogisticRegression()
model = StackingClassifier(estimators=level0,final_estimator=level1,cv=5)
<train_model>
img_train, img_test = run_xgb(xgb_params,X_img_row_temp[:len_train],X_text_image)
 X_temp['img_pred'] = np.concatenate([img_train,img_test.mean(axis = 1)] )
model.fit(x,y )<predict_on_test>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
y_predicted = model.predict(test )<create_dataframe>
avg_blur_result = Parallel(n_jobs=-1, verbose=1 )(
 delayed(get_average_color )(i)for i in train_df_imgs['image_filename'])
 train_df_imgs['average_red'] = [x[0] for x in avg_blur_result]
train_df_imgs['average_green'] = [x[1] for x in avg_blur_result]
train_df_imgs['average_blue'] = [x[2] for x in avg_blur_result]
train_df_imgs['blur'] = [x[3] for x in avg_blur_result]
submission = pd.DataFrame({
"PassengerId":test2['PassengerId'],
"Survived":y_predicted
} )<save_to_csv>
w_h_average = Parallel(n_jobs=-1, verbose=1 )(
 delayed(average_pixel_width )(i)for i in train_df_imgs['image_filename'])
 train_df_imgs['width'] = [x[0] for x in w_h_average]
train_df_imgs['height'] = [x[1] for x in w_h_average]
train_df_imgs['resultion'] = train_df_imgs['width'] * train_df_imgs['height']
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['ppi'] = train_df_imgs['resultion'] / train_df_imgs['image_size']
submission.to_csv('first_kaggale_titanic_submission.csv',index=False )<import_modules>
avg_blur_result = Parallel(n_jobs=-1, verbose=1 )(
 delayed(get_average_color )(i)for i in test_df_imgs['image_filename'])
 test_df_imgs['average_red'] = [x[0] for x in avg_blur_result]
test_df_imgs['average_green'] = [x[1] for x in avg_blur_result]
test_df_imgs['average_blue'] = [x[2] for x in avg_blur_result]
test_df_imgs['blur'] = [x[3] for x in avg_blur_result]
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as tud<load_from_csv>
w_h_average = Parallel(n_jobs=-1, verbose=1 )(
 delayed(average_pixel_width )(i)for i in test_df_imgs['image_filename'])
 test_df_imgs['width'] = [x[0] for x in w_h_average]
test_df_imgs['height'] = [x[1] for x in w_h_average]
test_df_imgs['resultion'] = test_df_imgs['width'] * test_df_imgs['height']
test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['ppi'] = test_df_imgs['resultion'] / test_df_imgs['image_size']
trainRaw = pd.read_csv('/kaggle/input/titanic/train.csv')
 testRaw = pd.read_csv('/kaggle/input/titanic/test.csv')
 answer = pd.read_csv('/kaggle/input/titanic-answer/answers.csv' )<feature_engineering>
aggs = {
'average_blue': ['sum', 'mean', 'var','min','max'],
'average_green': ['sum', 'mean', 'var','min','max'],
'average_red': ['sum', 'mean', 'var','min','max'],
'blur': ['sum', 'mean', 'var','min','max'],
'image_size': ['sum', 'mean', 'var','min','max'],
'width': ['sum', 'mean', 'var','min','max'],
'height': ['sum', 'mean', 'var','min','max'],
'ppi': ['sum', 'mean', 'var','min','max'],
'resultion': ['sum', 'mean', 'var','min','max'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
Pclass_dic = ['Pclass_1', 'Pclass_2', 'Pclass_3']
all_data[Pclass_dic] = all_data[Pclass_dic]/3
Sex_dic = ['Sex_female', 'Sex_male']
all_data[Sex_dic] = all_data[Sex_dic]/2
Embarked_dic = ['Embarked_C', 'Embarked_Q', 'Embarked_S']
all_data[Embarked_dic] = all_data[Embarked_dic]/3
Title_dic = ['Title_Master', 'Title_Miss', 'Title_Mr', 'Title_Mrs', 'Title_Officer', 'Title_Royalty']
all_data[Title_dic] = all_data[Title_dic]/6
FamilyLabel_dic = ['FamilyLabel_0', 'FamilyLabel_1', 'FamilyLabel_2']
all_data[FamilyLabel_dic] = all_data[FamilyLabel_dic]/3
Deck_dic = ['Deck_A', 'Deck_B', 'Deck_C', 'Deck_D', 'Deck_E', 'Deck_F', 'Deck_G', 'Deck_T','Deck_U' ]
all_data[Deck_dic] = all_data[Deck_dic]/9
TicketGroup_dic = ['TicketGroup_0', 'TicketGroup_1', 'TicketGroup_2']
all_data[TicketGroup_dic] = all_data[TicketGroup_dic]/3<normalization>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
def minmaxscaler(data):
min = np.amin(data)
 max = np.amax(data)
 return(data - min)/(max-min)
 def feature_normalize(data):
mu = np.mean(data,axis=0)
 std = np.std(data,axis=0)
 return(data - mu)/std
def unnormalized_show(img):
img = img * std + mu
npimg = img.numpy()
plt.figure()
plt.imshow(np.transpose(npimg,(1, 2, 0)) )<normalization>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
all_data.Age = minmaxscaler(all_data.Age)
 all_data.Fare = minmaxscaler(all_data.Fare )<prepare_x_and_y>
for x in [
'Breed1','Breed2',
'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName',
'breed_type_cross'
]:
X_temp = agg_helper(X_temp, x,'txt_pred')
 X_temp = agg_helper(X_temp, x,'img_pred')
 X_temp = agg_helper(X_temp, x,'tablur_pred')
 X_temp = agg_helper(X_temp, x,'height_mean')
 X_temp = agg_helper(X_temp, x,'Age')
 X_temp = agg_helper(X_temp, x,'image_size_mean')
 X_temp = agg_helper(X_temp, x,'resultion_mean')
 X_temp = agg_helper(X_temp, x,'ppi_mean')
train=all_data[all_data['Survived'].notnull() ]
test=all_data[all_data['Survived'].isnull() ].drop('Survived',axis=1)
 test_submssion=testRaw[['PassengerId']]
y = train.Survived
X = train.drop(['Survived'],axis=1 )<prepare_x_and_y>
for i in HD_cate:
if i in X_temp.columns:
X_temp[i] = X_temp.groupby(i)[i].transform('count' )
class DATASET(tud.Dataset):
def __init__(self, train, test):
self.X = torch.from_numpy(np.asarray(train)).float()
self.y= torch.from_numpy(np.asarray(test)).float()
def __getitem__(self, index):
return self.X[index], self.y[index]
def __len__(self):
return len(self.y)
 D_IN, H1, H2, H3, D_OUT = 31, 100, 100, 10, 1
DROP_PROB1, DROP_PROB2 = 0.5, 0.5
class Base_net(nn.Module):
def __init__(self):
super(Base_net, self ).__init__()
self.linear1 = nn.Linear(D_IN, H1)
 self.linear2 = nn.Linear(H1, H2)
 self.linear3 = nn.Linear(H2, H3)
 self.linear4 = nn.Linear(H3, D_OUT)
 self.dp1 = nn.Dropout(DROP_PROB1)
 self.dp2 = nn.Dropout(DROP_PROB2)
 def forward(self, x):
x = F.relu(self.linear1(x))
x = F.leaky_relu(self.linear2(x))
x = self.dp1(x)
 x = F.leaky_relu(self.linear3(x))
x = self.dp2(x)
 x = F.leaky_relu(self.linear4(x))
x = torch.sigmoid(x)
 return x<create_dataframe>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
full_dataset = DATASET(X, y)
 answer_dataset = DATASET(test, answer.Survived)
 full_dataloador = tud.DataLoader(full_dataset, batch_size=64, shuffle=False, drop_last=False)
 LEN_FULL_TRAIN = len(full_dataset.y)
 LEN_ANSWER = len(answer_dataset.y)
 LEARNING_RATE = 1e-3
WEIGHT_DECAY = 2e-4
model = Base_net()
optimizer = torch.optim.Adam(model.parameters() , weight_decay=WEIGHT_DECAY, lr=LEARNING_RATE)
 loss_func = nn.BCELoss()
for epoch in range(1000):
model.train()
for batch_x, batch_y in full_dataloador:
barch_y_pred = model(batch_x)
 loss = loss_func(barch_y_pred, batch_y.unsqueeze(dim=1))
optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
train_pred = model(full_dataset.X)
 train_pred = train_pred.detach().apply_(lambda x : 1 if x > 0.6 else 0)
 train_acc = np.sum(train_pred.numpy().flatten() == full_dataset.y.numpy().flatten())/ LEN_FULL_TRAIN
answer_pred = model(answer_dataset.X)
 answer_pred = answer_pred.detach().apply_(lambda x : 1 if x > 0.6 else 0)
 answer_acc = np.sum(answer_pred.numpy().flatten() == answer_dataset.y.numpy().flatten())/ LEN_ANSWER
if epoch % 20 == 0 :
print("epoch: {} | loss: {} | train acc: {} | answer acc: {}".format(epoch, loss.item() , train_acc, answer_acc))
if answer_acc > 0.81 :
print("epoch: {} | loss: {} | train acc: {} | answer acc: {}".format(epoch, loss.item() , train_acc, answer_acc))
break<feature_engineering>
len_train = len(X_train_non_null)
 data = X_train_non_null.append(X_test_non_null ).reset_index(drop=True )
test = torch.from_numpy(np.asarray(test)).float()
model.eval()
test_submssion.loc[:,'Survived'] = model(test ).detach().numpy().flatten()
test_submssion.loc[:,'Survived'] = test_submssion.loc[:,'Survived'].apply(lambda x : 1 if x > 0.5 else 0 )<save_to_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
test_submssion.to_csv('/kaggle/working/submssion.csv',index=False )<load_from_csv>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()<load_from_csv>
keep_cate = [i for i in keep_cate if i in data.columns]
keep_float = [i for i in data.columns if i not in keep_cate + ['Description','metadata_annots_top_desc','sentiment_entities','AdoptionSpeed']]
pad_title = 200
pad_desc = 200
pad_sent = 50
for i in keep_cate:
encoder = preprocessing.LabelEncoder()
data[i] = encoder.fit_transform(data[i].astype(str))
for i in keep_float:
stanscaler = preprocessing.StandardScaler()
stanscaler.fit(data[i].values.reshape(-1, 1 ).astype(np.float32))
data[i] = stanscaler.transform(data[i].values.reshape(-1, 1 ).astype(np.float32))
def get_keras_data(data, y=None):
X = {}
for feat in keep_cate + keep_float:
if feat in data.columns:
X[feat] = np.array(data[feat])
 X['Description'] = pad_sequences(tokenizer.texts_to_sequences(data['Description'].apply(spacing_punctuation)) , maxlen=pad_title)
 X['metadata_annots_top_desc'] = pad_sequences(tokenizer.texts_to_sequences(data['metadata_annots_top_desc'].apply(spacing_punctuation)) , maxlen=pad_desc)
 X['sentiment_entities'] = pad_sequences(tokenizer.texts_to_sequences(data['sentiment_entities'].apply(spacing_punctuation)) , maxlen=pad_sent)
 if y is not None:
y = y.values.reshape(-1, 1)
 return X, y
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()<count_missing_values>
class RMSEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0, batch_size=2000)
 score = np.sqrt(mean_squared_error(self.y_val, y_pred))
print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch + 1, score))
class ModelWrapper(object):
def __init__(self):
pass
def fit_predict(self, X_train, y_train, X_val, y_val):
pass
def predict(self, X_test):
pass
class DNNModelWrapper(ModelWrapper):
def __init__(self, get_dnn_model, batch_size,
epochs, get_keras_data):
self.get_dnn_model = get_dnn_model
self.batch_size = batch_size
self.epochs = epochs
self.get_keras_data = get_keras_data
def fit_predict(self, X_train, y_train, X_val, y_val):
self.dnn_model = self.get_dnn_model(len(X_train), self.batch_size, self.epochs)
 print("get_keras_data.....")
 X_train, y_train = self.get_keras_data(X_train, y_train)
 X_val, y_val = self.get_keras_data(X_val, y_val)
 RMSE = RMSEvaluation(validation_data=(X_val, y_val), interval=1)
 ES = EarlyStopping(monitor='val_loss',
patience=2,
verbose=0, mode='auto')
 mcp_save = ModelCheckpoint('mdl_wts.h5', save_best_only=True, monitor='val_loss', mode='min')
 self.dnn_model.fit(X_train, y_train,
batch_size=self.batch_size,
callbacks=[RMSE, ES, mcp_save],
validation_data=(X_val, y_val),
verbose=1, epochs=self.epochs)
 self.dnn_model.load_weights('mdl_wts.h5')
 val_pred = self.dnn_model.predict(X_val)
 return val_pred.reshape(-1)
 def predict(self, X):
X, _ = self.get_keras_data(X)
 y_pred = self.dnn_model.predict(X, batch_size=2000)
 return y_pred.reshape(-1)
categorical_cols = [col for col in train_data.columns if train_data[col].dtype == object
and train_data[col].nunique() <= 10]
print("Categorical columns: ",categorical_cols)
 categorical_cols_missing_val = [col for col in train_data[categorical_cols].columns
if train_data[col].isnull().any() ]
print('
Categorical columns with missing values: ')
 for col in categorical_cols_missing_val:
print(col,'-', train_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "train_data":',train_data.shape )<count_missing_values>
max_dict = dict(( feat, np.max(data[feat])+ 1)for feat in keep_cate )
categorical_test_cols = [col for col in test_data.columns
if test_data[col].nunique() <= 10 and test_data[col].dtype == object]
print("Categorical columns: ",categorical_test_cols)
 categorical_test_cols_missing_val = [col for col in test_data[categorical_test_cols].columns
if test_data[col].isnull().any() ]
print('
Categorical columns with missing values: None')
 for col in categorical_test_cols_missing_val:
print(col,'-', test_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "test_data":',test_data.shape )<count_missing_values>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
numerical_cols = [col for col in train_data.columns if train_data[col].dtype != object]
numerical_cols.remove('Survived')
 numerical_cols.remove('PassengerId')
 print("Numerical columns: ",numerical_cols)
 num_cols_with_missing = [col for col in train_data[numerical_cols].columns
if train_data[col].isnull().any() ]
print('
Nummerical columns with missing values:')
 for col in num_cols_with_missing:
print(col,'-', train_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "train_data"',train_data.shape )<count_missing_values>
def get_dnn_model2(n_train, batch_size, epochs):
conts_size = 12
inputs = [Input(shape=[1], name=feat)for feat in keep_cate + keep_float]
embeds = []
for i, feat in enumerate(keep_cate):
embeds.append(Flatten()(Embedding(max_dict[feat], conts_size )(inputs[i])))
 y_dot = []
for field1, field2 in itertools.combinations([i for i in range(len(keep_cate)) ], 2):
embed1 = embeds[field1]
embed2 = embeds[field2]
y_dot.append(
 Dot(1, normalize=False )([embed1, embed2])
)
 y_dot_cat = concatenate(y_dot)
 for i, feat in enumerate(keep_float):
embeds.append(inputs[i + len(keep_cate)])
 title = Input(shape=[pad_title], name="Description")
 embed_title = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable= False )(title)
 embed_title = SpatialDropout1D(0.2 )(embed_title)
 embed_title = Bidirectional(CuDNNGRU(96, return_sequences=True))(embed_title)
 att_pool = Attention(pad_title )(embed_title)
 max_pool2 = GlobalMaxPooling1D()(embed_title)
 embed_title = concatenate([att_pool, max_pool2])
 description = Input(shape=[pad_desc], name="metadata_annots_top_desc")
 embed_description = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable= False )(description)
 embed_description = SpatialDropout1D(0.2 )(embed_description)
 embed_description = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_description)
 att_pool = Attention(pad_desc )(embed_description)
 max_pool2 = GlobalMaxPooling1D()(embed_description)
 embed_description = concatenate([att_pool,max_pool2])
 sentiment_entities = Input(shape=[pad_sent], name="sentiment_entities")
 embed_sentiment_entities = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable= False )(sentiment_entities)
 embed_sentiment_entities = SpatialDropout1D(0.2 )(embed_sentiment_entities)
 embed_sentiment_entities = Bidirectional(CuDNNGRU(128, return_sequences=True))(embed_sentiment_entities)
 att_pool = Attention(pad_sent )(embed_sentiment_entities)
 max_pool2 = GlobalMaxPooling1D()(embed_sentiment_entities)
 embed_sentiment_entities = concatenate([att_pool,max_pool2])
 inputs += [title, description,sentiment_entities]
embeds += [embed_title, embed_description, embed_sentiment_entities]
out = concatenate(embeds)
 out = concatenate([out, y_dot_cat])
 out = Dense(300 )(out)
 out = Activation('relu' )(out)
 out = Dropout(0.2 )(out)
 out = BatchNormalization()(out)
 out = Dense(1, activation='linear', name='output' )(out)
 model = Model(inputs, out)
 exp_decay = lambda init, fin, steps:(init / fin)**(1 /(steps - 1)) - 1
steps = int(n_train / batch_size)* epochs
lr_init, lr_fin = 0.001, 0.0005
lr_decay = exp_decay(lr_init, lr_fin, steps)
 optimizer = Adam(lr=lr_init, decay=lr_decay)
 model.compile(optimizer=optimizer, loss="mse")
 return model
def get_dnn_model(n_train, batch_size, epochs):
conts_size = 10
inputs = [Input(shape=[1], name=feat)for feat in keep_cate + keep_float]
embeds = []
for i, feat in enumerate(keep_cate):
if feat in data.columns:
embeds.append(Flatten()(Embedding(max_dict[feat], conts_size )(inputs[i])))
 for i, feat in enumerate(keep_float):
if feat in data.columns:
embeds.append(inputs[i + len(keep_cate)])

out = concatenate(embeds)
 out = Dense(512 )(out)
 out = BatchNormalization()(out)
 out = Activation('relu' )(out)
 out = Dense(1, activation='linear', name='output' )(out)
 model = Model(inputs, out)
 exp_decay = lambda init, fin, steps:(init / fin)**(1 /(steps - 1)) - 1
steps = int(n_train / batch_size)* epochs
lr_init, lr_fin = 0.001, 0.0005
lr_decay = exp_decay(lr_init, lr_fin, steps)
 optimizer = Adam(lr=lr_init, decay=lr_decay)
 model.compile(optimizer=optimizer, loss="mse")
 return model
def rmse(y_true, y_pred):
return mean_squared_error(y_true, y_pred)
 def get_oof(model, X, y, X_test,
k_fold, metric=rmse, save=False,
name=None, is_df=False):
ntrain, ntest = X.shape[0], X_test.shape[0]
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
kf = StratifiedKFold(n_splits=k_fold, shuffle=True, random_state=1337)
 oof_test_skf = np.empty(( k_fold, ntest))
errors = []
for i,(train_index, val_index)in enumerate(kf.split(X,y)) :
print(f'Fold {i}.......... ')
 if is_df:
X_train, y_train, X_val, y_val = \
X.loc[train_index], y.loc[train_index], X.loc[val_index], y.loc[val_index]
else:
X_train, y_train, X_val, y_val = \
X[train_index], y[train_index], X[val_index],y[val_index]
val_pred = model.fit_predict(X_train, y_train, X_val, y_val)
 error = metric(val_pred, y_val)
 print(f"{i}th error is {error}")
 errors.append(error)
 oof_train[val_index] = val_pred
oof_test_skf[i, :] = model.predict(X_test)
 print(f"mean error is {np.mean(errors)}")
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
numerical_test_cols = [col for col in test_data.columns if test_data[col].dtype != object]
numerical_test_cols.remove('PassengerId')
 print("Numerical columns: ",numerical_test_cols)
 num_test_cols_with_missing = [col for col in test_data[numerical_test_cols].columns
if test_data[col].isnull().any() ]
print('
Nummerical columns with missing values:')
 for col in num_test_cols_with_missing:
print(col,'-', test_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "test_data"',test_data.shape )<define_variables>
X = data[:len_train]
X_test = data[len_train:]
X = pd.concat([X,X_TXT[:len_train]], axis = 1)
 X_test = pd.concat([X_test,X_TXT[len_train:]], axis = 1)
 print(len(X), type(X))
print(len(X_test))
y = X_train['AdoptionSpeed']
k_fold = 10
batch_size = 300
epochs = 10
dnn_wrapper = DNNModelWrapper(get_dnn_model2, batch_size, epochs, get_keras_data)
 dnn_oof_train, dnn_oof_test = get_oof(dnn_wrapper, X, y,
X_test, k_fold,
save=True, name="dnn_v1",
is_df=True)
my_cols = numerical_cols + categorical_cols<count_unique_values>
def get_fea_importance(clf, feature_name):
gain = clf.feature_importance('gain')
 importance_df = pd.DataFrame(
 {'feature':clf.feature_name() ,
'split':clf.feature_importance('split'),
'gain':100 * gain / gain.sum() ,
} ).sort_values('gain',ascending=False)
 return importance_df
def run_lgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
feature_importance_df = pd.DataFrame()
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 lgb_train = lgb.Dataset(X_tr, y_tr)
 lgb_val = lgb.Dataset(X_val, y_val)
 clf = lgb.train(params, lgb_train , valid_sets = lgb_val,feature_name = list(X_tr.columns),
verbose_eval=verbose_eval, early_stopping_rounds=early_stop, num_boost_round=num_rounds)
 valid_pred = clf.predict(X_val, num_iteration=clf.best_iteration)
 test_pred = clf.predict(X_test, num_iteration=clf.best_iteration)
 fold_importance_df_fold = get_fea_importance(clf, list(X_tr.columns))
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df_fold], axis=0)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
feature_importance_df = feature_importance_df[
["feature", "split","gain"]
].groupby("feature",as_index=False ).mean().sort_values(by="gain", ascending=False)
 oof_test = oof_test.mean(axis=1)
 return oof_train, oof_test, feature_importance_df
print('Number of unique values in X
',train_data[my_cols].nunique() ,'
' )<define_variables>
lgb_params = {
'learning_rate': 0.005,
'eval_metric': 'l2',
'seed': 1337,
'bagging_fraction': 0.8,
'feature_fraction': 0.75,
'boosting': 'gbdt',
'max_depth':-1,
'num_leaves': 32,
"objective": "regression",
}
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women)
 men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )<drop_column>
oof_train_lgb, oof_test_lgb,feature_importance_df = run_lgb(lgb_params, X_train_non_null, X_test_non_null )
numerical_cols.remove('Age' )<drop_column>
drop_f = list(feature_importance_df[feature_importance_df.gain == 0].feature )
categorical_cols.remove('Embarked' )<split>
X_train_non_null.drop(drop_f, inplace = True, axis = 1)
 X_test_non_null.drop(drop_f, inplace = True, axis = 1 )
y = train_data.Survived
X = train_data.drop(['Survived','Name'], axis=1)
 train_X_full, val_X_full, train_y, val_y = train_test_split(X, y,
train_size=0.8, test_size=0.2,
random_state=0)
 train_X = train_X_full[my_cols].copy()
val_X = val_X_full[my_cols].copy()<categorify>
oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
numerical_transformer = SimpleImputer(strategy='median')
 categorical_transformer = Pipeline(steps=
[('imputer', SimpleImputer(strategy='constant')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 rf_model = RandomForestClassifier(n_estimators=72,
random_state=0,
max_leaf_nodes=50,
max_depth=10)
 pipe_rf = Pipeline(steps=
[('preprocessor', preprocessor), 
('model', rf_model)])
 pipe_rf.fit(train_X, train_y)
 preds_rf = pipe_rf.predict(val_X)
 mae_rf = mean_absolute_error(val_y, preds_rf)
 print("Validation MAE for Random Forest Classifier: %0.4f"%(mae_rf))
xgb_model = XGBClassifier(n_estimators=800,
learning_rate=0.01)
 preps = Pipeline(steps=
[('preprocessor', preprocessor), 
('xgbrg', xgb_model)])
 preps.fit(train_X, train_y)
 preds_xgb = preps.predict(val_X)
 mae_xgb = mean_absolute_error(preds_xgb, val_y)
 print("Validation MAE for XGBoost Classifier: %0.4f" %mae_xgb )<train_model>
X_train_non_null['nn_pred'] = dnn_oof_train
X_test_non_null['nn_pred'] = dnn_oof_test
X_train_non_null['lgb_pred'] = oof_train_lgb
X_test_non_null['lgb_pred'] = oof_test_lgb
X_train_non_null['xgb_pred'] = oof_train
X_test_non_null['xgb_pred'] = oof_test.mean(axis=1)
 dnn_oof_train = X_train_non_null['nn_pred'].values
dnn_oof_test = X_test_non_null['nn_pred'].values
lgb_oof_train = X_train_non_null['lgb_pred'].values
lgb_oof_test = X_test_non_null['lgb_pred'].values
my_cols = numerical_cols + categorical_cols
my_model_full_data = RandomForestClassifier(n_estimators=72,
random_state=0,
max_leaf_nodes=50,
max_depth=10)
 prep_full_data = Pipeline(steps=
[('preprocessor', preprocessor), 
('model', my_model_full_data)])
 prep_full_data.fit(X[my_cols], y )<create_dataframe>
oof_train_ensemble = oof_train * 0.45 + dnn_oof_train * 0.1 + lgb_oof_train * 0.45
oof_test_ensemble = oof_test.mean(axis=1)* 0.45 + dnn_oof_test * 0.1 + lgb_oof_test * 0.45
test_X = test_data[my_cols].copy()<save_to_csv>
optR = OptimizedRounder()
optR.fit(oof_train_ensemble, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train_ensemble, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 test_predictions = optR.predict(oof_test_ensemble, coefficients)
 Counter(test_predictions)
 plot_pred(test_predictions)
 submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False )
test_preds = prep_full_data.predict(test_X)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived':test_preds})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
optR = OptimizedRounder()
optR.fit(oof_train_lgb, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train_lgb, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK LGB = ", qwk )
mainData = pd.read_csv('/kaggle/input/titanic/train.csv')
 testData = pd.read_csv('/kaggle/input/titanic/test.csv' )<drop_column>
optR = OptimizedRounder()
optR.fit(dnn_oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(dnn_oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK DNN= ", qwk )
mainData = mainData[mainData.Embarked.notna() ]
mainData.drop(['Cabin'], axis='columns', inplace=True)
 testData.drop(['Cabin'], axis='columns', inplace=True )<feature_engineering>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK XGB= ", qwk )
def rank(row):
a = ['Mrs.','Mr.','Miss','Master']
ret = []
for b in a:
if b in row.Name:
ret.append(b)
 if len(ret)==0:
if row.Sex == 'male':
ret = ['Mr.']
else:
ret = ['Miss']
return ' '.join(ret)
 mainData['rank'] = mainData.apply(rank, axis = 1)
 testData['rank'] = testData.apply(rank, axis = 1 )<drop_column>
coefficients_ = coefficients.copy()
train_predictions = optR.predict(oof_train_ensemble, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test_ensemble, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
mainData = mainData.drop(['Name','Ticket','Sex'], axis = 1)
 testData = testData.drop(['Name','Ticket','Sex'], axis = 1 )<categorify>
Counter(train_predictions )
mainData["Embarked"] = mainData["Embarked"].map({'S': 1,'C':2,'Q':3})
 testData["Embarked"] = testData["Embarked"].map({'S': 1,'C':2,'Q':3})
 mainData["rank_num"] = mainData["rank"].map({'Mrs.': 1,'Mr.':2,'Miss':3,'Master':4})
 testData["rank_num"] = testData["rank"].map({'Mrs.': 1,'Mr.':2,'Miss':3,'Master':4})
 def grouping(row):
a = row.Pclass
a = a * 10 + row.rank_num
a = a * 10 + row.Embarked
return a
mainData['grouping'] = mainData.apply(grouping, axis = 1 ).astype(int)
 testData['grouping'] = testData.apply(grouping, axis = 1 ).astype(int )<groupby>
Counter(test_predictions )
<feature_engineering><EOS> 
oof_train_lgb_stacking, oof_test_lgb_stacking, feature_importance_df = run_lgb(lgb_params, X_train_non_null, X_test_non_null)
 print(feature_importance_df)
 optR = OptimizedRounder()
optR.fit(oof_train_lgb_stacking, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train_lgb_stacking, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK XGB= ", qwk )