Kaggle competition: Exam for Students20200129
<SOS> metric: rmsle Kaggle data source: exam-for-students20200129<save_model>
plt.style.use('ggplot')
 %matplotlib inline
pd.set_option('display.max_columns', 500 )
learn.save('step-1' )<choose_model_class>
df_train = pd.read_csv('.. /input/exam-for-students20200129/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/exam-for-students20200129/test.csv', index_col=0)
 df_train.ConvertedSalary = np.log1p(df_train.ConvertedSalary )
learn = cnn_learner(data,
models.resnet34,
metrics=[accuracy, AUROC() ],
callback_fns=[partial(SaveModelCallback)],
wd=0.1,
ps=[0.9, 0.6, 0.4],
path = '.')
 learn = learn.load('step-1' )<find_best_params>
df_train.isnull().sum()
learn.freeze_to(1)
 learn.lr_find()
learn.recorder.plot()<train_model>
df_test.isnull().sum()
learn.fit_one_cycle(20, slice(4e-4, lr/5))<train_model>
cats = []
for col in df_train.columns:
if df_train[col].dtype == 'object':
cats.append(col)
 print(col, df_train[col].nunique() )
unfrozen_validation = learn.validate()
print("Final model validation loss: {0}".format(unfrozen_validation[0]))<predict_on_test>
df_train.groupby(u ).size()
probability, classification = learn.get_preds(ds_type=DatasetType.Test)
 test_df.has_cactus = probability.numpy() [:, 0]
test_df.head()<save_to_csv>
df_train.groupby(u ).size()
test_df.to_csv("submission.csv", index=False )<define_variables>
df_train.groupby(u ).size().sort_values()
DEV_MODE = False
data_root_path = pathlib.Path(".. /input" )<normalization>
df_test.groupby(u ).size().sort_values()
def load_and_preprocess_image(path):
image = tf.io.read_file(path)
 image = tf.image.decode_jpeg(image, channels=3)
 image = tf.image.resize(image, [32, 32])
 image /= 255.0
return image<load_from_csv>
df_train.drop('Country',axis=1,inplace=True)
 df_test.drop('Country',axis=1,inplace=True )
train_df = pd.read_csv(data_root_path/'train.csv', dtype={'id': 'str', 'has_cactus': np.int32})
 DATASET_SIZE = len(train_df)
 print("n =", DATASET_SIZE )<randomize_order>
df_train.groupby(u ).size()
no_cactus, yes_cactus = train_df.has_cactus.value_counts().sort_values().values
no_multiplier = int(yes_cactus/no_cactus)-1
no_cactus_rows = train_df[train_df.has_cactus == 0]
for i in range(no_multiplier):
train_df = train_df.append(no_cactus_rows )<define_variables>
df_train.groupby(u ).size()
ROOT_TRAIN_PATH = '.. /input/train/train/'
train_paths = ROOT_TRAIN_PATH + train_df['id']<prepare_x_and_y>
path_ds = tf.data.Dataset.from_tensor_slices(train_paths)
 image_ds = path_ds.map(load_and_preprocess_image)
 label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(train_df['has_cactus'].values, tf.int32))<define_variables>
nf = 'count_Employment'
df_train[nf] = df_train[u].map(df_train.groupby(u ).ConvertedSalary.count())
 df_test[nf] = df_test[u].map(df_train.groupby(u ).ConvertedSalary.count() )
IMAGE_SHAPE =(32, 32, 3)
 image_label_ds = tf.data.Dataset.zip(( image_ds, label_ds)).shuffle(DATASET_SIZE)
 BATCH_SIZE = 16
if DEV_MODE:
train_size, val_size = int(0.7 * DATASET_SIZE), int(0.3 * DATASET_SIZE)
 else:
train_size, val_size = DATASET_SIZE, 0
train_ds = image_label_ds.map(augment_image ).take(train_size ).batch(BATCH_SIZE ).repeat()
val_ds = image_label_ds.skip(train_size ).batch(BATCH_SIZE ).repeat()<choose_model_class>
df_train.groupby(u ).size()
def get_model() :
model = Sequential()
model.add(Conv2D(32,(2, 2), input_shape=IMAGE_SHAPE, activation='relu'))
model.add(Conv2D(64,(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(4, 4), activation='relu'))
model.add(Conv2D(64,(5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation="sigmoid"))
return model
model = get_model()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
 model.summary()<train_model>
u = 'CompanySize'
df_train.groupby(u ).size()
train_steps = 2*(train_size//BATCH_SIZE)
 val_steps = val_size//BATCH_SIZE
if val_steps == 0:
val_steps = None
val_ds = None
history = model.fit(train_ds,
steps_per_epoch=train_steps,
epochs=50,
validation_data=val_ds,
validation_steps=val_steps,
callbacks=None )<train_model>
df_test.groupby(u ).size()
training_eval = tf.data.Dataset.zip(( image_ds, label_ds)).shuffle(DATASET_SIZE)
 training_eval = training_eval.take(DATASET_SIZE ).batch(DATASET_SIZE)
 model.evaluate(training_eval, steps=1 )<define_variables>
df_train.groupby(u ).size()
test_paths = [path for path in sorted(pathlib.Path('.. /input/test/test/' ).glob('*.jpg')) ]
TEST_SIZE = len(test_paths)
 test_path_ds = tf.data.Dataset.from_tensor_slices([str(path)for path in test_paths])
 test_image_ds = test_path_ds.map(load_and_preprocess_image)
 test_image_ds = test_image_ds.take(TEST_SIZE ).batch(TEST_SIZE )<predict_on_test>
u = 'YearsCoding'
df_train.groupby(u ).size()
preds=model.predict(test_image_ds, steps = 1 )<save_to_csv>
df_test.groupby(u ).size()
test_df=pd.DataFrame({'id': [path.name for path in test_paths] })
 test_df['has_cactus']=preds
test_df.to_csv("submission.csv",index=False )<import_modules>
df_train[u].replace({'0-2 years':0,
'12-14 years':12,
'15-17 years':15,
'18-20 years':18,
'21-23 years':21,
'24-26 years':24,
'27-29 years':27,
'3-5 years':3,
'30 or more years':30,
'6-8 years':6,
'9-11 years':9},inplace=True )
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import tensorflow as tf
from tensorflow import keras
import os
from shutil import copyfile, move
from tqdm import tqdm
import h5py<set_options>
df_train.groupby(u ).size()
print(tf.__version__)
 print(tf.test.is_gpu_available() )<load_from_csv>
df_test[u].replace({'0-2 years':0,
'12-14 years':12,
'15-17 years':15,
'18-20 years':18,
'21-23 years':21,
'24-26 years':24,
'27-29 years':27,
'3-5 years':3,
'30 or more years':30,
'6-8 years':6,
'9-11 years':9},inplace=True )
training_df = pd.read_csv(".. /input/train.csv")
 training_df.head()<import_modules>
u = 'YearsCodingProf'
df_train.groupby(u ).size()
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import InputLayer, Input
from tensorflow.keras.layers import Conv2D, Dense, Flatten, Dropout, Activation
from tensorflow.keras.layers import BatchNormalization, Reshape, MaxPooling2D, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint<define_variables>
df_train.groupby(u ).size()
batch_size = 64
train_datagen = ImageDataGenerator(
 rescale=1./ 255,
horizontal_flip=True,
vertical_flip=True)
 train_data_dir = ".. /sorted_training"
train_generator = train_datagen.flow_from_directory(
 train_data_dir,
shuffle=True,
target_size=(32, 32),
batch_size=batch_size,
class_mode='binary')
 validation_datagen = ImageDataGenerator(rescale=1./ 255)
 validation_data_dir = ".. /sorted_validation"
validation_generator = validation_datagen.flow_from_directory(
 validation_data_dir,
target_size=(32, 32),
batch_size=batch_size,
class_mode='binary')
 input_shape =(32,32,3)
 num_classes = 2 
<choose_model_class>
u = 'Age'
df_train.groupby(u ).size()
dropout_dense_layer = 0.6
model = Sequential()
model.add(Conv2D(32,(3, 3), input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(32,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Conv2D(64,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Flatten())
 model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(dropout_dense_layer))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(dropout_dense_layer))
model.add(Dense(1))
model.add(Activation('sigmoid'))<choose_model_class>
df_test.groupby(u ).size()
model.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.Adam(lr=0.001),
metrics=['accuracy'] )<choose_model_class>
df_train.groupby(u ).size()
callbacks = [EarlyStopping(monitor='val_loss', patience=25),
ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]<train_model>
u = 'LastNewJob'
df_train.groupby(u ).size()
epochs = 100
history = model.fit_generator(train_generator,
validation_data=validation_generator,
epochs=epochs,
verbose=1,
shuffle=True,
callbacks=callbacks )<load_pretrained>
df_train.groupby(u)['ConvertedSalary'].mean()
model.load_weights("best_model.h5" )<define_variables>
u = 'Currency'
df_train.groupby(u ).size()
test_folder = ".. /input/test/"
test_datagen = ImageDataGenerator(
 rescale=1./ 255)
 test_generator = test_datagen.flow_from_directory(
 directory=test_folder,
target_size=(32,32),
batch_size=1,
class_mode='binary',
shuffle=False
)<predict_on_test>
df_test.groupby(u ).size().index
pred=model.predict_generator(test_generator,verbose=1)
 pred_binary = [0 if value<0.50 else 1 for value in pred]<save_to_csv>
csv_file = open("sample_submission_cnn.csv","w")
 csv_file.write("id,has_cactus
")
 for filename, prediction in zip(test_generator.filenames,pred_binary):
name = filename.split("/")[1].replace(".tif","")
 csv_file.write(str(name)+","+str(prediction)+"
")
 csv_file.close()<set_options>
df_train[u].nunique()
%matplotlib inline
%reload_ext autoreload
%autoreload 2
InteractiveShell.ast_node_interactivity = "all"<import_modules>
df_test[u].nunique()
from fastai.vision import *
from pathlib import Path<load_from_csv>
u = 'CurrencySymbol'
df_train.groupby(u ).size()
train_df = pd.read_csv(root/"train.csv")
 test_df = pd.read_csv(root/"sample_submission.csv" )<define_variables>
df_test.groupby(u ).size().index
test_set = ImageList.from_df(test_df, path=root/'test', cols='id', folder='test' )<feature_engineering>
df_train.groupby(u)['ConvertedSalary'].mean()
tsfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75 )<define_variables>
nf = 'count_CurrencySymbol'
df_train[nf] = df_train[u].map(df_train.groupby(u ).ConvertedSalary.count())
 df_test[nf] = df_test[u].map(df_train.groupby(u ).ConvertedSalary.count() )
SZ=128
BS=64<define_variables>
cats = []
for col in df_train.columns:
if df_train[col].dtype == 'object':
cats.append(col)
 print(col, df_train[col].nunique())
 encoder = OrdinalEncoder(cols=cats)
 df_train[cats] = encoder.fit_transform(df_train[cats])
 df_test[cats] = encoder.transform(df_test[cats] )
data.show_batch(rows=3, figsize=(6,6))<choose_model_class>
feature = ['CompanySize',
'LastNewJob',
'CurrencySymbol',
'YearsCoding',
'count_Employment',
'SalaryType',
'Currency',
'Employment',
'MilitaryUS',
'YearsCodingProf',
'count_CurrencySymbol',
'Age',
'RaceEthnicity',
'DevType',
'Student',
'AssessBenefits2',
'JobContactPriorities3',
'CareerSatisfaction',
'FrameworkWorkedWith',
'NumberMonitors',
'CheckInCode',
'AssessBenefits6',
'AssessJob1',
'AssessBenefits11',
'JobEmailPriorities5',
'AssessBenefits8',
'OperatingSystem',
'AssessBenefits9',
'FormalEducation',
'AssessBenefits7',
'AssessJob5',
'AssessJob4',
'AssessBenefits4',
'EducationParents',
'WakeTime',
'AssessBenefits1',
'JobEmailPriorities6',
'AssessBenefits10',
'AssessJob10',
'CommunicationTools',
'JobContactPriorities4',
'JobContactPriorities1',
'JobContactPriorities2',
'AdsPriorities5',
'UndergradMajor',
'AssessJob3',
'AssessJob8',
'AssessJob6',
'AssessJob2',
'AssessJob7',
'AssessBenefits5',
'AdsActions',
'AssessBenefits3',
'Exercise',
'AdsPriorities2',
'AdsPriorities3',
'JobEmailPriorities1',
'AdsAgreeDisagree3',
'AssessJob9',
'StackOverflowVisit',
'JobEmailPriorities2',
'AdsPriorities1',
'AdsPriorities6',
'Gender',
'JobEmailPriorities7',
'JobEmailPriorities3',
'AdsPriorities7',
'UpdateCV',
'AdsPriorities4',
'StackOverflowDevStory',
'HopeFiveYears',
'ErgonomicDevices']
arch = models.densenet169 
<choose_model_class>
y_train = df_train.ConvertedSalary
X_train = df_train.drop(['ConvertedSalary'],axis=1)
 X_test = df_test.copy()
learn = cnn_learner(data, arch, metrics=[error_rate, accuracy] )<train_model>
scores = []
y_pred_test = 0
skf = KFold(n_splits=5, random_state=60, shuffle=True)
 for i,(train_ix, test_ix)in tqdm(enumerate(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
clf = LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=9999, n_jobs=-1, num_leaves=15, objective=None,
random_state=71, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 clf.fit(X_train_, y_train_, early_stopping_rounds=50, eval_metric='rmse', eval_set=[(X_val, y_val)])
 y_pred = clf.predict(X_val)
 y_pred_test += clf.predict(X_test)
 score = mean_squared_error(y_val, y_pred)
 scores.append(score)
 print('CV Score of Fold_%d is %f' %(i, score))
lr = 1e-03
learn.fit_one_cycle(5, slice(lr))<predict_on_test>
skf = KFold(n_splits=5, random_state=40, shuffle=True)
 for i,(train_ix, test_ix)in tqdm(enumerate(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
clf = LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=9999, n_jobs=-1, num_leaves=15, objective=None,
random_state=71, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 clf.fit(X_train_, y_train_, early_stopping_rounds=50, eval_metric='rmse', eval_set=[(X_val, y_val)])
 y_pred = clf.predict(X_val)
 y_pred_test += clf.predict(X_test)
 score = mean_squared_error(y_val, y_pred)
 scores.append(score)
 print('CV Score of Fold_%d is %f' %(i, score))
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
y_pred = y_pred_test/10
y_pred = np.expm1(y_pred)
 submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission.ConvertedSalary = y_pred
submission.to_csv('submission.csv' )