Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<compute_train_metric>
!pip install --use-feature=2020-resolver https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-1.3.0_SNAPSHOT%2B00b0ad1293b4fa74d6aca5da4e9ab7a9d16777f0-py3-none-manylinux2010_x86_64.whl
data = open_data("train.csv")
 X, Y = param_label(data)
 x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2)
 model = svm.SVC()
model.fit(x_train, y_train)
 y_predict = model.predict(x_test)
 acc = metrics.accuracy_score(y_predict, y_test)
 print("Accuracy:",acc )<compute_train_metric>
xgb.__version__
kernels = ["rbf", "linear", "sigmoid", "poly"]
for kernel in kernels:
model = svm.SVC(kernel = kernel)
 model.fit(x_train, y_train)
 y_predict = model.predict(x_test)
 acc = metrics.accuracy_score(y_predict, y_test)
 print("Accuracy with kernel =", kernel, ": ",acc )<save_to_csv>
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.linear_model import LogisticRegression
import gc
import os
model = svm.SVC(kernel = 'linear')
 model.fit(X,Y)
 data_test = open_data("test.csv")
 prediction = model.predict(data_test.drop(["PassengerId"], 1))
solution = pd.DataFrame(np.array([[data_test.PassengerId.iloc[i], prediction[i]] for i in range(len(data_test)) ]),
columns=['PassengerId', 'Survived'])
 solution.to_csv("solution_SVM.csv", index=False )<find_best_params>
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()
cs = [1, 5, 10, 15, 20]
gammas = [0.005, 0.01, 0.02, 0.05, 0.1]
for gamma in gammas:
for c in cs:
model = svm.SVC(kernel = 'rbf', C = c, gamma = gamma)
 model.fit(x_train, y_train)
 acc = metrics.accuracy_score(model.predict(x_test), y_test)
 print("Accuracy with gamma = ",gamma,"c = ",c,": ",acc)
 print("" )<save_to_csv>
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224
model = svm.SVC(kernel = 'rbf', gamma = 0.01, C = 10)
 model.fit(X,Y)
 data_test = open_data("test.csv")
 prediction = model.predict(data_test.drop(["PassengerId"], 1))
solution = pd.DataFrame(np.array([[data_test.PassengerId.iloc[i], prediction[i]] for i in range(len(data_test)) ]),
columns=['PassengerId', 'Survived'])
 solution.to_csv("solution_SVM_para.csv", index=False )<import_modules>
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )
from sklearn.ensemble import RandomForestClassifier<split>
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]
data = open_data("train.csv")
 X, Y = param_label(data)
 x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2 )<compute_train_metric>
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )
model = RandomForestClassifier()
model.fit(x_train, y_train)
 y_predict = model.predict(x_test)
 y_predict
acc = metrics.accuracy_score(y_predict, y_test)
 print(acc )<find_best_model_class>
train_EB7_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB7_ns.npy')
 test_EB7_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB7_ns.npy')
 train_EB4_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB4_ns.npy')
 test_EB4_ns = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB4_ns.npy' )
trees = [5, 10, 20, 50, 100]
averages = []
mins = []
maxs = []
for tree in trees:
average_acc = 0
min_acc = 1
max_acc = 0
for i in range(100):
x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2)
 model = RandomForestClassifier(n_estimators = tree)
 model.fit(x_train, y_train)
 y_predict = model.predict(x_test)
 y_predict
acc = metrics.accuracy_score(y_predict, y_test)
 average_acc = average_acc + acc
if acc > max_acc: max_acc = acc
if acc < min_acc: min_acc = acc
averages = averages + [average_acc/100]
mins = mins + [min_acc]
maxs = maxs + [max_acc]
plt.figure(figsize=(24,8))
plt.plot(averages, color = 'r', linewidth=2)
 plt.plot(mins, color = 'r', linestyle='--')
 plt.plot(maxs, color = 'r', linestyle='--' )<find_best_model_class>
X_train, X_val, y_train, y_val = train_test_split(train_EB7_ns, target, test_size=0.1, random_state=42 )
depths = [1, 2, 5, 10, 15, 20]
averages = []
mins = []
maxs = []
for depth in depths:
average_acc = 0
min_acc = 1
max_acc = 0
for i in range(100):
x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2)
 model = RandomForestClassifier(n_estimators = 20, max_depth = depth)
 model.fit(x_train, y_train)
 y_predict = model.predict(x_test)
 y_predict
acc = metrics.accuracy_score(y_predict, y_test)
 average_acc = average_acc + acc
if acc > max_acc: max_acc = acc
if acc < min_acc: min_acc = acc
averages = averages + [average_acc/100]
mins = mins + [min_acc]
maxs = maxs + [max_acc]
plt.figure(figsize=(24,8))
plt.plot(averages, color = 'r', linewidth=2)
 plt.plot(mins, color = 'r', linestyle='--')
 plt.plot(maxs, color = 'r', linestyle='--' )<save_to_csv>
lr = LogisticRegression(C=0.026, max_iter=10000)
 lr.fit(X_train, y_train)
 val_preds_EB7_lr = lr.predict_proba(X_val)[:,1]
test_preds_EB7_lr = lr.predict_proba(test_EB7_ns)[:,1]
print(roc_auc_score(y_val, val_preds_EB7_lr))
print(log_loss(y_val, val_preds_EB7_lr))
model = RandomForestClassifier(n_estimators = 20, max_depth = 10)
 model.fit(X,Y)
 data_test = open_data("test.csv")
 prediction = model.predict(data_test.drop(["PassengerId"], 1))
solution = pd.DataFrame(np.array([[data_test.PassengerId.iloc[i], prediction[i]] for i in range(len(data_test)) ]),
columns=['PassengerId', 'Survived'])
 solution.to_csv("solution_Random_Forest.csv", index=False )<save_to_csv>
xgb_params = {
'eta': 0.05,
'max_depth': 4,
'subsample': 0.8,
'colsample_bytree': 0.6,
'alpha': 0.01,
'lambda': 1.00,
'gamma' : 0.02,
'max_bin': 256,
'objective': 'reg:logistic',
'eval_metric': 'auc',
'verbosity': 0,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
}
dtrain = xgb.DMatrix(X_train, y_train)
 dval = xgb.DMatrix(X_val, y_val)
 dtest = xgb.DMatrix(test_EB7_ns )
model = RandomForestClassifier(n_estimators = 20, max_depth = 10)
 model.fit(X[["Sex", "Age", "Fare", "Pclass"]],Y)
 data_test = open_data("test.csv")
 prediction = model.predict(data_test[["Sex", "Age", "Fare", "Pclass"]])
 solution = pd.DataFrame(np.array([[data_test.PassengerId.iloc[i], prediction[i]] for i in range(len(data_test)) ]),
columns=['PassengerId', 'Survived'])
 solution.to_csv("solution_Random_Forest_4features.csv", index=False )<import_modules>
del train_EB7_ns, test_EB7_ns, X_train, X_val
gc.collect()
import tensorflow as tf
from tensorflow import keras<split>
%%time
model = xgb.train(xgb_params, dtrain, num_boost_round=100)
 val_preds_xgb_1 = model.predict(dval)
 test_preds_xgb_1 = model.predict(dtest )
data = open_data("train.csv")
 X, Y = param_label(data)
 x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2 )<choose_model_class>
roc_auc_score(y_val, val_preds_xgb_1 )
def build_model() :
model = keras.Sequential()
model.add(keras.layers.Dense(32, activation='relu', kernel_initializer = 'uniform', input_shape=[len(X.keys())]))
model.add(keras.layers.Dense(12, activation='relu', kernel_initializer = 'uniform'))
model.add(keras.layers.Dense(1, activation='sigmoid', kernel_initializer = 'uniform'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = build_model()
model.summary()<train_model>
log_loss(y_val, val_preds_xgb_1 )
model = build_model()
class PrintDot(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs):
if epoch % 100 == 0: print('')
 print('.', end='')
 EPOCHS = 1000
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=50)
 history = model.fit(normed_x_train, y_train, epochs=EPOCHS,
validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot() ])
 plot_history(history )<predict_on_test>
roc_auc_score(y_val, 0.5*val_preds_xgb_1+0.5*val_preds_EB7_lr )
y_pred = model.predict(x_test)
 y_pred =(y_pred > 0.5 ).astype(int ).reshape(x_test.shape[0])
 metrics.accuracy_score(y_pred, y_test )<define_search_space>
log_loss(y_val, 0.5*val_preds_xgb_1+0.5*val_preds_EB7_lr )
architectures = [[12, 6],
[32, 16],
[64, 32],
[12, 12, 6],
[32, 16, 8],
[64, 32, 16],
[12, 12, 6, 6],
[32, 32, 16, 8],
[64, 32, 16, 8],
[64, 64, 32, 16, 8]]<choose_model_class>
def build_model(architecture):
model = keras.Sequential()
model.add(keras.layers.Dense(architecture[0], activation='relu', kernel_initializer = 'uniform', input_shape=[len(X.keys())]))
for i in range(1, len(architecture)) :
n = architecture[i]
model.add(keras.layers.Dense(n, activation='relu', kernel_initializer = 'uniform'))
model.add(keras.layers.Dense(1, activation='sigmoid', kernel_initializer = 'uniform'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
EPOCHS = 1000
averages = []
mins = []
maxs = []
for architecture in architectures:
model = build_model(architecture)
 average_acc = 0
min_acc = 1
max_acc = 0
for i in range(100):
x_train, x_test, y_train, y_test = subset_data(X, Y, 0.2)
 history = model.fit(normed_x_train, y_train, epochs=EPOCHS,
validation_split = 0.2, verbose=0, callbacks=[early_stop])
 y_pred = model.predict(x_test)
 y_pred =(y_pred > 0.5 ).astype(int ).reshape(x_test.shape[0])
 acc = metrics.accuracy_score(y_pred, y_test)
 average_acc = average_acc + acc
if acc > max_acc: max_acc = acc
if acc < min_acc: min_acc = acc
averages = averages + [average_acc/100]
mins = mins + [min_acc]
maxs = maxs + [max_acc]
print("Accuracy with architecture ", architecture, ": ", average_acc/100)
 plt.figure(figsize=(24,8))
plt.plot(averages, color = 'r', linewidth=2)
 plt.plot(mins, color = 'r', linestyle='--')
 plt.plot(maxs, color = 'r', linestyle='--' )<load_from_csv>
tra=pd.read_csv('.. /input/train.csv')
 tes=pd.read_csv('.. /input/test.csv' )<drop_column>
X_train, X_val, y_train, y_val = train_test_split(train_EB4_ns, target, test_size=0.1, random_state=42 )
x=tra.drop(['Name','PassengerId','Ticket','Survived'],axis=1)
 x_t=tes.drop(['Name','PassengerId','Ticket'],axis=1)
<count_missing_values>
lr = LogisticRegression(C=0.019, max_iter=10000)
 lr.fit(X_train, y_train)
 val_preds_EB4_lr = lr.predict_proba(X_val)[:,1]
test_preds_EB4_lr = lr.predict_proba(test_EB4_ns)[:,1]
print(roc_auc_score(y_val, val_preds_EB4_lr))
print(log_loss(y_val, val_preds_EB4_lr))
x.isna().sum()<count_missing_values>
xgb_params = {
'eta': 0.05,
'max_depth': 4,
'subsample': 0.85,
'colsample_bytree': 0.6,
'alpha': 0.01,
'lambda': 1.00,
'gamma' : 0.02,
'max_bin': 256,
'objective': 'reg:logistic',
'eval_metric': 'auc',
'verbosity': 0,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
}
dtrain = xgb.DMatrix(X_train, y_train)
 dval = xgb.DMatrix(X_val, y_val)
 dtest = xgb.DMatrix(test_EB4_ns )
x_t.isna().sum()<data_type_conversions>
del train_EB4_ns, test_EB4_ns, X_train, X_val
gc.collect()
x.Age=x.Age.fillna(x.Age.mean())
 x_t.Age=x_t.Age.fillna(x_t.Age.mean() )<data_type_conversions>
%%time
val_preds_xgb_2 = 0
test_preds_xgb_2 = 0
n_seed = 10
for j in range(n_seed):
xgb_params['seed'] = 3*j**2+154
model = xgb.train(xgb_params, dtrain, num_boost_round=100)
 val_preds_xgb_2 += model.predict(dval)/n_seed
test_preds_xgb_2 += model.predict(dtest)/n_seed
x.Cabin=x.Cabin.fillna('U')
 x_t.Cabin=x_t.Cabin.fillna('U' )<data_type_conversions>
sample_submission['id'] = ids
sample_submission['label'] = test_preds_xgb_1
sample_submission.to_csv('submission_xgb_1.csv', index=False)
 sample_submission.head()
x.Embarked=x.Embarked.fillna('S')
 x_t.Embarked=x_t.Embarked.fillna('S' )<feature_engineering>
sample_submission['id'] = ids
sample_submission['label'] = test_preds_xgb_2
sample_submission.to_csv('submission_xgb_2.csv', index=False)
 sample_submission.head()
x_t.Fare=x_t.Fare.fillna(x_t.Fare.mean() )<categorify>
sample_submission['id'] = ids
sample_submission['label'] = 0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr
sample_submission.to_csv('submission_0.csv', index=False)
 sample_submission.head()
x.Cabin = x.Cabin.map(lambda z: z[0])
 x_t.Cabin = x_t.Cabin.map(lambda z: z[0] )<feature_engineering>
sample_submission['id'] = ids
sample_submission['label'] =(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)**0.99
sample_submission.to_csv('submission_0_b.csv', index=False)
 sample_submission.head()
<categorify>
sample_submission['id'] = ids
sample_submission['label'] =(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)**0.95
sample_submission.to_csv('submission_0_c.csv', index=False)
 sample_submission.head()
x= pd.get_dummies(x)
 x_t=pd.get_dummies(x_t )<drop_column>
sample_submission['id'] = ids
sample_submission['label'] = 0.47*test_preds_xgb_1+0.53*test_preds_EB7_lr
sample_submission.to_csv('submission_1.csv', index=False)
 sample_submission.head()
x=x.drop(['Cabin_T'],axis=1 )<prepare_x_and_y>
sample_submission['id'] = ids
sample_submission['label'] = 0.52*test_preds_xgb_1+0.48*test_preds_EB7_lr
sample_submission.to_csv('submission_2.csv', index=False)
 sample_submission.head()
y=tra['Survived']<split>
sample_submission['id'] = ids
sample_submission['label'] = 0.55*test_preds_xgb_1+0.45*test_preds_EB7_lr
sample_submission.to_csv('submission_3.csv', index=False)
 sample_submission.head()
x_train,x_val,y_train,y_val=train_test_split(x,y,test_size=0.2,random_state=0 )<train_model>
sample_submission['id'] = ids
sample_submission['label'] = 0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr
sample_submission.to_csv('submission_EB4_0.csv', index=False)
 sample_submission.head()
reg=RandomForestClassifier(n_estimators=100000,random_state=0)
 reg.fit(x_train,y_train )<choose_model_class>
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.01*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.01*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_0.csv', index=False)
 sample_submission.head()
<predict_on_test>
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.05*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.05*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_1.csv', index=False)
 sample_submission.head()
y_pred=reg.predict(x_val )<compute_test_metric>
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.08*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.08*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_2.csv', index=False)
 sample_submission.head()
accuracy_score(y_val, y_pred )<save_to_csv>
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.08*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.08*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_2.csv', index=False)
 sample_submission.head()
pred=reg.predict(x_t)
 new_pred=pred.astype(int)
 output=pd.DataFrame({'PassengerId':tes['PassengerId'],'Survived':new_pred})
 output.to_csv('Titanic.csv', index=False )<load_from_csv>
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3.csv', index=False)
 sample_submission.head()
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv" )<drop_column>
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9976] = 1
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_b.csv', index=False)
 sample_submission.head()
train = train.drop(columns=['Name','Cabin','Ticket'])
 test = test.drop(columns=['Name','Cabin','Ticket'] )<data_type_conversions>
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9973] = 1
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_c.csv', index=False)
 sample_submission.head()
train['Embarked_S'] =(train['Embarked'] == 'S' ).astype(int)
 train['Embarked_C'] =(train['Embarked'] == 'C' ).astype(int)
 train['Embarked_Q'] =(train['Embarked'] == 'Q' ).astype(int)
 train['Gender'] =(train['Sex'] == 'male' ).astype(int )<data_type_conversions>
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9973] = 1
subs[subs < 0.0018] = 0
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_d.csv', index=False)
 sample_submission.head()
test['Embarked_S'] =(test['Embarked'] == 'S' ).astype(int)
 test['Embarked_C'] =(test['Embarked'] == 'C' ).astype(int)
 test['Embarked_Q'] =(test['Embarked'] == 'Q' ).astype(int)
 test['Gender'] =(test['Sex'] == 'male' ).astype(int )<drop_column>
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.12*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.12*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_4.csv', index=False)
 sample_submission.head()
<train_model><EOS> 
sample_submission['id'] = ids
subs = np.clip(1.12*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.12*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission['label'] = subs
subs[subs > 0.9974] = 1
subs[subs < 0.0018] = 0
sample_submission.to_csv('submission_EB7_EB4_4_b.csv', index=False)
 sample_submission.head()