Kaggle competition: Exam for Students20200129
<SOS> metric: rmsle Kaggle data source: exam-for-students20200129<save_to_csv>
%matplotlib inline
plt.style.use('ggplot')
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 20 )
sample['has_cactus'] = test_pred
sample.to_csv('densenet_model.csv', index=False )<import_modules>
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import roc_auc_score
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.feature_extraction.text import TfidfVectorizer
from category_encoders import OrdinalEncoder, OneHotEncoder, TargetEncoder, CatBoostEncoder
from tqdm import tqdm_notebook as tqdm
import lightgbm as lgb
from lightgbm import LGBMClassifier
from tensorflow.keras import layers, optimizers, models, callbacks, applications
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from shutil import copy2<load_from_csv>
df_train = pd.read_csv('.. /input/exam-for-students20200129/train.csv', index_col=0)
train_dir = '.. /input/train/train/'
test_dir = '.. /input/test/test/'
df = pd.read_csv('.. /input/train.csv')
 df.head()<define_variables>
df_test = pd.read_csv('.. /input/exam-for-students20200129/test.csv', index_col=0)
labels = df['has_cactus'].tolist()
img_paths = df['id'].tolist()<choose_model_class>
tgt_col = 'ConvertedSalary'
def build_model() :
model_input = layers.Input(shape = [32, 32, 3])
 X = applications.densenet.DenseNet121(weights = 'imagenet', include_top = False, classes = 1 )(model_input)
 X = layers.Flatten()(X)
 model_output = layers.Dense(1, activation = 'sigmoid' )(X)
 return models.Model(model_input, model_output)
<choose_model_class>
df_train2 = df_train.copy()
df_test2 = df_test.copy()
df_train2['missing_amnt'] = df_train2.isnull().sum(axis=1)
 df_test2['missing_amnt'] = df_test2.isnull().sum(axis=1 )
model = build_model()
model.compile(loss = 'binary_crossentropy', optimizer = optimizers.Adam(lr = 0.0001), metrics = ['accuracy'])
 model.summary() 
<load_from_csv>
df_train2[tgt_col] = np.log1p(df_train2[tgt_col] )
def split_dataframe(csv_file):
df = pd.read_csv(csv_file)
 train_df = df.iloc[:14000, :]
train_df['has_cactus'] = train_df['has_cactus'].apply(str)
 val_df = df.iloc[14000:, :]
val_df['has_cactus'] = val_df['has_cactus'].apply(str)
 return train_df, val_df
train_df, val_df = split_dataframe('.. /input/train.csv' )<load_from_csv>
y_train = df_train2[tgt_col]
X_train = df_train2.drop([tgt_col], axis=1)
 X_test = df_test2
train_df, val_df = split_dataframe('.. /input/train.csv')
 train_gen = ImageDataGenerator(
 rescale = 1./255,
validation_split = 0.15,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True,
vertical_flip = True,
rotation_range = 10 
)
 train_gen = train_gen.flow_from_dataframe(
 dataframe = train_df,
directory = train_dir,
x_col = 'id',
y_col = 'has_cactus',
target_size =(32, 32),
color_mode = 'rgb',
class_mode = 'binary',
batch_size = 128,
shuffle = True, 
)
 val_gen = ImageDataGenerator(rescale = 1./255, validation_split = 0.15)
 val_gen = val_gen.flow_from_dataframe(
 dataframe = val_df,
directory = train_dir,
x_col = 'id',
y_col = 'has_cactus',
target_size =(32, 32),
color_mode = 'rgb',
class_mode = 'binary',
batch_size = 128,
shuffle = True,
)<train_model>
num_cols = []
for col in X_train.columns:
if X_train[col].dtype == 'int64' or X_train[col].dtype == 'float64':
num_cols.append(col)
 scaler = StandardScaler()
scaler.fit(X_train[num_cols])
 X_train[num_cols] = scaler.transform(X_train[num_cols])
 X_test[num_cols] = scaler.transform(X_test[num_cols] )
checkpoint = callbacks.ModelCheckpoint(
 'model.h5',
monitor = 'val_loss',
verbose = 0,
save_best_only = True,
save_weights_only = False,
mode = 'auto' 
)
 regulate_lr = callbacks.ReduceLROnPlateau(monitor = 'val_loss', min_lr = 1e-5, patience = 3)
 model.fit_generator(
 train_gen,
steps_per_epoch = 14000 // 128,
validation_data = val_gen,
validation_steps = 14000 // 128,
epochs = 60,
callbacks = [checkpoint, regulate_lr] 
)
<load_from_csv>
def student_enc(x):
if x == 'No':
return 3
elif x == 'Yes, part-time':
return 2
elif x == 'Yes, full-time':
return 1
submit = pd.read_csv('.. /input/sample_submission.csv')
 test = []
for image in tqdm(submit['id']):
test.append(cv2.imread('.. /input/test/test/' + image))
test = np.array(test)
 print(test[0].shape)
<save_to_csv>
X_train['Student'] = X_train['Student'].apply(student_enc)
 X_test['Student'] = X_test['Student'].apply(student_enc )
test = test / 255
preds = final_model.predict(test, verbose = 1)
 submit['has_cactus'] = preds
submit.head(10)
 submit.to_csv('sample_submission.csv', index = False )<load_from_csv>
def company_size_enc(x):
if x == 'Fewer than 10 employees':
return 1
elif x == '10 to 19 employees':
return 2
elif x == '20 to 99 employees':
return 3
elif x == '100 to 499 employees':
return 4
elif x == '500 to 999 employees':
return 5
elif x == '1,000 to 4,999 employees':
return 6
elif x == '5,000 to 9,999 employees':
return 7
elif x == '10,000 or more employees':
return 8
train_dir = ".. /input/aerial-cactus-identification/train/train/"
test_dir = ".. /input/aerial-cactus-identification/test/test/"
train_df = pd.read_csv('.. /input/aerial-cactus-identification/train.csv')
 train_df.head()
label = train_df['has_cactus'].tolist()
fn = train_df['id'].tolist()<define_variables>
X_train['CompanySize'] = X_train['CompanySize'].apply(company_size_enc)
 X_test['CompanySize'] = X_test['CompanySize'].apply(company_size_enc )
train_fn = fn[:14000]
val_fn = fn[14000:]
train_labels = label[:14000]
val_labels = label[14000:]
for i in zip(train_labels,train_fn):
if i[0] == 1:
copy2(str(train_dir+i[1]),str('.. /img_train/yes/'+i[1]))
elif i[0] == 0:
copy2(str(train_dir+i[1]),str('.. /img_train/no/'+i[1]))
for i in zip(val_labels,val_fn):
if i[0] == 1:
copy2(str(train_dir+i[1]),str('.. /img_val/yes/'+i[1]))
elif i[0] == 0:
copy2(str(train_dir+i[1]),str('.. /img_val/no/'+i[1])) 
<define_variables>
def years_coding_enc(x):
if x == '0-2 years':
return 1
elif x == '3-5 years':
return 2
elif x == '6-8 years':
return 3
elif x == '9-11 years':
return 4
elif x == '12-14 years':
return 5
elif x == '15-17 years':
return 6
elif x == '18-20 years':
return 7
elif x == '21-23 years':
return 8
elif x == '24-26 years':
return 9
elif x == '27-29 years':
return 10
elif x == '30 or more years':
return 11
train_datagen = ImageDataGenerator(
 rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True 
)
 val_datagen = ImageDataGenerator(rescale=1./255)
 val_generator = val_datagen.flow_from_directory(
 '.. /img_val/',
target_size=(32,32),
color_mode="rgb",
batch_size=64,
class_mode='binary')
 train_generator = train_datagen.flow_from_directory(
 '.. /img_train/',
target_size=(32,32),
color_mode="rgb",
batch_size=64,
class_mode='binary')
 step_train = train_generator.n//train_generator.batch_size
step_val= val_generator.n//val_generator.batch_size<choose_model_class>
X_train['YearsCoding'] = X_train['YearsCoding'].apply(years_coding_enc)
 X_test['YearsCoding'] = X_test['YearsCoding'].apply(years_coding_enc )
base_model = ResNet50(weights=".. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5",include_top=False,input_shape=(32,32,3))
for layer in base_model.layers:
layer.trainable = True 
<choose_model_class>
def years_coding_prof_enc(x):
if x == '0-2 years':
return 1
elif x == '3-5 years':
return 2
elif x == '6-8 years':
return 3
elif x == '9-11 years':
return 4
elif x == '12-14 years':
return 5
elif x == '15-17 years':
return 6
elif x == '18-20 years':
return 7
elif x == '21-23 years':
return 8
elif x == '24-26 years':
return 9
elif x == '27-29 years':
return 10
elif x == '30 or more years':
return 11
x = base_model.output
x = GlobalAveragePooling2D()(x)
 x = Dropout(rate=0.6 )(x)
 predictions = Dense(1, activation = 'sigmoid',activity_regularizer=regularizers.l1(0.01))(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 model.compile(optimizer='Adam', loss='binary_crossentropy',metrics=['accuracy'])
<train_model>
X_train['YearsCodingProf'] = X_train['YearsCodingProf'].apply(years_coding_prof_enc)
 X_test['YearsCodingProf'] = X_test['YearsCodingProf'].apply(years_coding_prof_enc )
cp = ModelCheckpoint('ResNet50.hdf5' , monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=3, min_lr=0.0001)
 model.fit_generator(
 train_generator,
steps_per_epoch=step_train,
validation_data= val_generator,
validation_steps = step_val,
epochs=20,
callbacks=[cp,reduce_lr] )<load_pretrained>
def job_satisfaction_enc(x):
if x == 'Extremely dissatisfied':
return 1
elif x == 'Moderately dissatisfied':
return 2
elif x == 'Slightly dissatisfied':
return 3
elif x == 'Neither satisfied nor dissatisfied':
return 4
elif x == 'Slightly satisfied':
return 5
elif x == 'Moderately satisfied':
return 6
elif x == 'Extremely satisfied':
return 7
model.load_weights("ResNet50.hdf5")
 print(os.listdir(".. /input/aerial-cactus-identification/"))<save_to_csv>
X_train['JobSatisfaction'] = X_train['JobSatisfaction'].apply(job_satisfaction_enc)
 X_test['JobSatisfaction'] = X_test['JobSatisfaction'].apply(job_satisfaction_enc )
test_set = pd.read_csv('.. /input/aerial-cactus-identification/sample_submission.csv')
 pred = np.empty(( test_set.shape[0],))
for n in range(4000):
data = np.array(Image.open('.. /input/aerial-cactus-identification/test/test/'+test_set.id[n]))
data = data.astype(np.float32)/ 255.
 pred[n] = model.predict(data.reshape(( 1, 32, 32, 3)))
 test_set['has_cactus'] = pred
test_set.to_csv('sample_submission.csv', index=False )<load_from_csv>
def career_satisfaction_enc(x):
if x == 'Extremely dissatisfied':
return 1
elif x == 'Moderately dissatisfied':
return 2
elif x == 'Slightly dissatisfied':
return 3
elif x == 'Neither satisfied nor dissatisfied':
return 4
elif x == 'Slightly satisfied':
return 5
elif x == 'Moderately satisfied':
return 6
elif x == 'Extremely satisfied':
return 7
print(os.listdir(".. /input"))
train_csv = pd.read_csv('.. /input/train.csv' ).sample(frac=1 ).reset_index(drop=True)
 images = train_csv.id.values.tolist()
target = train_csv.has_cactus.values.tolist()
train_X, test_X, train_Y, test_Y = train_test_split(images, target, test_size=0.1, random_state=42)
 del train_csv, images, target<define_variables>
X_train['CareerSatisfaction'] = X_train['CareerSatisfaction'].apply(career_satisfaction_enc)
 X_test['CareerSatisfaction'] = X_test['CareerSatisfaction'].apply(career_satisfaction_enc )
batch_img = []
batch_tar = []
val_x = []
val_y = []
for i in range(len(train_X)) :
batch_img.append(np.reshape(get_image(train_X[i]),(32,32,3)))
 batch_tar.append(train_Y[i])
 for i in range(len(test_X)) :
val_x.append(np.reshape(get_image(test_X[i]),(32,32,3)))
 val_y.append(test_Y[i])
 batch_img, val_x = np.array(batch_img), np.array(val_x )<choose_model_class>
def age_enc(x):
if x == 'Under 18 years old':
return 1
elif x == '18 - 24 years old':
return 2
elif x == '25 - 34 years old':
return 3
elif x == '35 - 44 years old':
return 4
elif x == '45 - 54 years old':
return 5
elif x == '55 - 64 years old':
return 6
elif x == '65 years or older':
return 7
model = Sequential()
model.add(Conv2D(16, kernel_size=3, padding='same', activation='relu', input_shape=(32,32,3)))
 model.add(Conv2D(16, kernel_size=3, padding='same', activation='relu'))
model.add(Conv2D(8, kernel_size=3, padding='same', activation='relu'))
model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'] )<train_model>
X_train['Age'] = X_train['Age'].apply(age_enc)
 X_test['Age'] = X_test['Age'].apply(age_enc )
model.fit(batch_img, batch_tar, validation_data =(val_x, val_y), epochs = 20 )<train_model>
def edu_enc(x):
if x == 'They never completed any formal education':
return 1
elif x == 'Primary/elementary school':
return 2
elif x == 'Secondary school(e.g.American high school, German Realschule or Gymnasium, etc.) ':
return 3
elif x == 'Some college/university study without earning a degree':
return 4
elif x == 'Bachelor’s degree(BA, BS, B.Eng., etc.) ':
return 6
elif x == 'Master’s degree(MA, MS, M.Eng., MBA, etc.) ':
return 7
elif x == 'Associate degree':
return 5
elif x == 'Other doctoral degree(Ph.D, Ed.D., etc.) ':
return 8
elif x == 'Professional degree(JD, MD, etc.) ':
return 9
test_list = os.listdir('.. /input/test/test')
 def get_test_image(imname):
name = os.path.join('.. /input/test/test', imname)
 img = cv2.imread(name, 1)
 img = cv2.resize(img,(32,32)) /255
return img<predict_on_test>
cat_cols = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cat_cols.append(col)
test_imgs = []
for i in range(len(test_list)) :
test_imgs.append(np.reshape(get_test_image(test_list[i]),(32,32,3)))
 test_imgs = np.array(test_imgs)
 pred = model.predict(test_imgs )<save_to_csv>
for i in cat_cols:
summary = X_train[i].value_counts() / len(X_train)
 X_train[i] = X_train[i].map(summary)
 X_test[i] = X_test[i].map(summary )
pred = [i[0] for i in pred]
res_dict = {'id': test_list, 'has_cactus' : pred}
res_df = pd.DataFrame(res_dict)
 res_df.to_csv('result.csv', index=False )<import_modules>
X_train['Country_LastNewJob'] = X_train['Country'] * X_train['LastNewJob']
X_train['Country_YearsCodingProf'] = X_train['Country'] * X_train['YearsCodingProf']
X_train['LastNewJob_YearsCodingProf'] = X_train['LastNewJob'] * X_train['YearsCodingProf']
import cv2
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import json
import os
from tqdm import tqdm, tqdm_notebook
from keras.models import Sequential
import tensorflow.keras.backend as K
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.applications import VGG16
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator<load_from_csv>
X_test['Country_LastNewJob'] = X_test['Country'] * X_test['LastNewJob']
X_test['Country_YearsCodingProf'] = X_test['Country'] * X_test['YearsCodingProf']
X_test['LastNewJob_YearsCodingProf'] = X_test['LastNewJob'] * X_test['YearsCodingProf']
train_dir = ".. /input/train/train/"
test_dir = ".. /input/test/test/"
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()<load_pretrained>
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )
im = cv2.imread(".. /input/train/train/01e30c0ba6e91343a12d2126fcafc0dd.jpg")
 plt.imshow(im )<choose_model_class>
scores = []
kf = KFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(kf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
model = lgb.LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=9999, n_jobs=-1, num_leaves=15, objective=None,
random_state=71, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 model.fit(X_train_, y_train_, early_stopping_rounds=20, eval_metric='rmse', eval_set=[(X_val, y_val)])
 y_pred = model.predict(X_val)
 mse = mean_squared_error(y_val, y_pred)
 rmse = np.sqrt(mse)
 scores.append(rmse)
 print('CV Score of Fold_%d is %f' %(i, rmse))
def baseline_model() :
model = Sequential()
model.add(Conv2D(64,(5, 5), input_shape=(32, 32, 3), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(64,(5, 5), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(128,(5, 5), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(128,(5, 5), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(256,(3, 3), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(256,(3, 3), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(512,(3, 3), padding='same', kernel_regularizer=l2(1e-4)))
 model.add(BatchNormalization())
 model.add(LeakyReLU(alpha=0.3))
model.add(GlobalAveragePooling2D())
 model.add(Dense(1, activation='sigmoid'))
return model<compute_test_metric>
model.fit(X_train, y_train)
 y_pred = model.predict(X_test )
<choose_model_class><EOS> 
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission[tgt_col] = y_pred
submission[tgt_col] = np.exp(submission[tgt_col])- 1
submission.to_csv('submission.csv')