Kaggle competition: Exam for Students20200129
data_folder = Path(".. /input" )<load_from_csv>
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 %matplotlib inline
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<define_variables>
df_train_original = pd.read_csv('.. /input/exam-for-students20200129/train.csv', index_col=0)
 df_test_original = pd.read_csv('.. /input/exam-for-students20200129/test.csv',index_col=0)
 target = 'ConvertedSalary'
df_country_original = pd.read_csv('.. /input/exam-for-students20200129/country_info.csv')
 df_train_original.shape, df_test_original.shape
train_img.show_batch(rows=3, figsize=(7,6))<choose_model_class>
df_train = df_train_original.copy()
df_test = df_test_original.copy()
df_country = df_country_original.copy()
df_test_with_targetcol = df_test_original.copy()
df_test_with_targetcol[target] = -1
df_all = pd.concat([df_train, df_test_with_targetcol], axis=0 )
learn = cnn_learner(train_img, models.densenet161, metrics=[error_rate, accuracy] )<train_model>
df_GDP = df_country[['Country', 'GDP($ per capita)']]
df_train = df_train.merge(df_GDP, on = ['Country'], how = 'left')
 df_test = df_test.merge(df_GDP, on = ['Country'], how = 'left')
 df_all = df_all.merge(df_GDP, on = ['Country'], how = 'left' )
lr = 3e-02
learn.fit_one_cycle(6, slice(lr))<predict_on_test>
c = 'Country'
df_train[c].value_counts()
preds,_ = learn.get_preds(ds_type=DatasetType.Test )<filter>
df_test[c].value_counts()
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
df_train[df_train['Country'] == 'Germany']
df_train[df_train['Country'] == 'France']
test_df.to_csv('submission.csv', index=False )<import_modules>
for col in df_all.columns:
print(df_all[col].value_counts())
 print('____')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg<prepare_x_and_y>
first_log = True
path = '.. /input/train/train/'
df = pd.read_csv('.. /input/train.csv')
 n = len(df)
 y = np.zeros(shape=(n,1),dtype=int)
 X = np.zeros(shape=(n,32,32,3))
for i,v in df.iterrows() :
y[i] = v[1]
X[i] =(mpimg.imread(f'{path}/{v[0]}')) /255<train_model>
y_train = df_train[target]
X_train = df_train.drop([target], axis=1)
 X_test = df_test
X_all = pd.concat([X_train, X_test], axis = 0)
 X_train = X_train[y_train.isnull() ==False]
y_train = y_train[y_train.isnull() ==False]
if(first_log == True):
y_train = y_train.apply(np.log1p )
<concatenate>
TXT_train_DevType = X_train.DevType.copy()
TXT_test_DevType = X_test.DevType.copy()
TXT_train_CommunicationTools = X_train.CommunicationTools.copy()
TXT_test_CommunicationTools = X_test.CommunicationTools.copy()
TXT_train_FrameworkWorkedWith = X_train.FrameworkWorkedWith.copy()
TXT_test_FrameworkWorkedWith = X_test.FrameworkWorkedWith.copy()
TXT_train_DevType.fillna('
TXT_test_DevType.fillna('
TXT_train_CommunicationTools.fillna('
TXT_test_CommunicationTools.fillna('
TXT_train_FrameworkWorkedWith.fillna('
TXT_test_FrameworkWorkedWith.fillna('
X_all = np.concatenate(( X,X_vflip,X_hflip,X_rot))
y_all = np.concatenate(( y,y,y,y))<import_modules>
X_train['missing_sum'] = X_train.isnull().sum(axis=1)
 X_test['missing_sum'] = X_test.isnull().sum(axis=1 )
from keras.models import Model
from keras.layers import Input, Dense, Convolution2D, Flatten, MaxPooling2D, Dropout<choose_model_class>
X_train['DevType'].fillna('
X_test['DevType'].fillna('
X_train['CommunicationTools'].fillna('
X_test['CommunicationTools'].fillna('
X_train['FrameworkWorkedWith'].fillna('
X_test['FrameworkWorkedWith'].fillna('
input_layer = Input(shape=(32,32,3))
x = Convolution2D(16, kernel_size=3,activation='relu' )(input_layer)
 x = MaxPooling2D()(x)
 x = Convolution2D(32, kernel_size=3,activation='relu' )(x)
 x = Convolution2D(64, kernel_size=3,activation='relu' )(x)
 x = MaxPooling2D()(x)
 x = Dense(256,activation='relu' )(Flatten()(x))
x = Dropout(rate=0.5 )(x)
 output = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=output)
 model.compile(optimizer='adam', loss='binary_crossentropy',
metrics=['accuracy'])
 model.summary()<train_model>
def mapping(map_col, mapping):
X_train[map_col]=X_train[map_col].map(mapping)
 X_test[map_col]=X_test[map_col].map(mapping )
model.fit(X_all,y_all,epochs=10 )<load_from_csv>
Age_mapping = {'Under 18 years old':18, '18 - 24 years old':24, '25 - 34 years old':34, '35 - 44 years old':44,
'45 - 54 years old':54, '55 - 64 years old':64, '65 years or older':65}
CompanySize_mapping ={'Fewer than 10 employees':10, '10 to 19 employees':19,
'20 to 99 employees':99, '100 to 499 employees':499, '500 to 999 employees':999,
'1,000 to 4,999 employees':4999, '5,000 to 9,999 employees':9999,
'10,000 or more employees':20000}
Exercise_mapping = {'I don't typically exercise':0, '1 - 2 times per week':2,
'3 - 4 times per week':4, 'Daily or almost every day':7}
HoursComputer_mapping = {'Less than 1 hour':1, '1 - 4 hours':4, '5 - 8 hours':8, '9 - 12 hours':12, 'Over 12 hours':15}
HoursOutside_mapping = {'Less than 30 minutes':0.5, '30 - 59 minutes':1, '1 - 2 hours':2, '3 - 4 hours':4, 'Over 4 hours':6}
LastNewJob_mapping = {'I've never had a job':0, 'Less than a year ago':1, 'Between 1 and 2 years ago':2,
'Between 2 and 4 years ago':4, 'More than 4 years ago': 5}
SkipMeals_mapping = {'Never':0, '1 - 2 times per week':2, '3 - 4 times per week':4, 'Daily or almost every day':7}
StackOverflowJobsRecommend_mapping = {'0(Not Likely)':0, '1':1, '2':2, '3':3, '4':4, '5':5, '6':6, '7':7, '8':8, '9':9,
'10(Very Likely)':10}
YearsCoding_mapping ={'0-2 years':2, '3-5 years':5, '6-8 years': 8, '9-11 years':11, '12-14 years':14, '15-17 years':17,
'18-20 years':20, '21-23 years':23, '24-26 years':26,
'27-29 years':29, '30 or more years':25}
YearsCodingProf_mapping = YearsCoding_mapping
tpath = '.. /input/test/test/'
submit = pd.read_csv('.. /input/sample_submission.csv')
 t_X = np.zeros(shape=(len(submit),32,32,3))
for i,v in submit.iterrows() :
t_X[i] =(mpimg.imread(f'{tpath}/{v[0]}')) /255<predict_on_test>
mapping('Age', Age_mapping)
 mapping('CompanySize', CompanySize_mapping)
 mapping('Exercise', Exercise_mapping)
 mapping('HoursComputer', HoursComputer_mapping)
 mapping('HoursOutside', HoursOutside_mapping)
 mapping('LastNewJob', LastNewJob_mapping)
 mapping('SkipMeals', SkipMeals_mapping)
 mapping('StackOverflowJobsRecommend', StackOverflowJobsRecommend_mapping)
 mapping('YearsCoding', YearsCoding_mapping)
 mapping('YearsCodingProf', YearsCodingProf_mapping )
t_y = model.predict(t_X )<prepare_output>
count_col = ['DevType', 'CommunicationTools', 'FrameworkWorkedWith']
for i in count_col:
X_train[i + '_count'] = X_train[i].apply(lambda x: len(re.split('\s*;\s*', x)))
 X_test[i + '_count'] = X_test[i].apply(lambda x: len(re.split('\s*;\s*', x)) )
submit['has_cactus'] = t_y
submit.head()<save_to_csv>
X_train['Github'] = X_train['CommunicationTools'].str.contains('Github')
 X_test['Github'] = X_test['CommunicationTools'].str.contains('Github')
 X_train['Full-stack'] = X_train['DevType'].str.contains('Full-stack')
 X_test['Full-stack'] = X_test['DevType'].str.contains('Full-stack')
 X_train['DataScientist'] = X_train['DevType'].str.contains('Data scientist')
 X_test['DataScientist'] = X_test['DevType'].str.contains('Data scientist' )
submit.to_csv('170419_perm.csv',index=False )<set_options>
del X_train['Country']
del X_test['Country']
del X_train['DevType']
del X_test['DevType']
del X_train['CommunicationTools']
del X_test['CommunicationTools']
del X_train['FrameworkWorkedWith']
del X_test['FrameworkWorkedWith']
del X_train['MilitaryUS']
del X_test['MilitaryUS']
del X_train['SurveyTooLong']
del X_test['SurveyTooLong']
del X_train['SurveyEasy']
del X_test['SurveyEasy']
%matplotlib inline<load_from_csv>
X_train['AssessJob_null'] = X_train['AssessJob1'].isnull().astype(int)
 X_test['AssessJob_null'] = X_test['AssessJob1'].isnull().astype(int)
 X_train['AssessBenefits_null'] = X_train['AssessBenefits1'].isnull().astype(int)
 X_test['AssessBenefits_null'] = X_train['AssessBenefits1'].isnull().astype(int)
 X_train['JobContactPriorities_null'] = X_train['JobContactPriorities1'].isnull().astype(int)
 X_test['JobContactPriorities_null'] = X_test['JobContactPriorities1'].isnull().astype(int)
 X_train['JobEmailPriorities_null'] = X_train['JobEmailPriorities1'].isnull().astype(int)
 X_test['JobEmailPriorities_null'] = X_train['JobEmailPriorities1'].isnull().astype(int)
 X_train['questionaare_missing_count'] = X_train['AssessJob_null'] + X_train['AssessBenefits_null'] + X_train['JobContactPriorities_null'] + X_train['JobEmailPriorities_null']
X_test['questionaare_missing_count'] = X_test['AssessJob_null'] + X_test['AssessBenefits_null'] + X_test['JobContactPriorities_null'] + X_test['JobEmailPriorities_null']
del X_train['AssessJob_null']
del X_test['AssessJob_null']
del X_train['AssessBenefits_null']
del X_test['AssessBenefits_null']
del X_train['JobContactPriorities_null']
del X_test['JobContactPriorities_null']
del X_train['JobEmailPriorities_null']
del X_test['JobEmailPriorities_null']
input_path = '.. /input/'
train_path = input_path + 'train/train/'
test_path = input_path + 'test/test/'
train_df = pd.read_csv(input_path + 'train.csv')
 sample = pd.read_csv(input_path + 'sample_submission.csv')
 train_id = train_df['id']
labels = train_df['has_cactus']
test_id = sample['id']<split>
col_yj_transform = ['GDP($ per capita)']
for col in col_yj_transform:
pt = PowerTransformer(method='yeo-johnson')
 reshape_train = X_train[col].values.reshape(-1,1)
 reshape_test = X_test[col].values.reshape(-1,1)
 pt.fit(reshape_train)
 X_train[col] = pt.transform(reshape_train)
 X_test[col] = pt.transform(reshape_test )
x_train, x_val, y_train, y_val = train_test_split(train_id, labels, test_size=0.2 )<data_type_conversions>
cats = []
for col in X_train.columns:
if(X_train[col].dtype == 'object'):
cats.append(col)
 print(col, X_train[col].nunique() )
def fetch_images(ids, filepath):
arr = []
for img_id in ids:
img = plt.imread(filepath + img_id)
 arr.append(img)
 arr = np.array(arr ).astype('float32')
 arr = arr / 255
return arr<categorify>
ordinal_col = cats
ordinal_encoder = OrdinalEncoder(cols = ordinal_col)
 X_train = ordinal_encoder.fit_transform(X_train)
 X_test = ordinal_encoder.transform(X_test )
x_train = fetch_images(ids=x_train, filepath=train_path)
 x_val = fetch_images(ids=x_val, filepath=train_path)
 test = fetch_images(ids=test_id, filepath=test_path)
 img_dim = x_train.shape[1:]<import_modules>
X_train.fillna(-9999, inplace = True)
 X_test.fillna(-9999, inplace = True )
from keras.models import Model
from keras.layers import Input, Dense, Flatten, Dropout, LeakyReLU, Activation
from keras.layers.normalization import BatchNormalization
from keras.applications.densenet import DenseNet201<define_variables>
for col in X_train.columns:
print(X_train[col].value_counts())
 print('____' )
batch_size = 64
epochs = 30
steps = x_train.shape[0] // batch_size<choose_model_class>
%%time
num_split = 5
num_iter = 3
stop_round = 50
scores = []
y_pred_cva = np.zeros(len(X_test))
scores = []
for h in range(num_iter):
kf = KFold(n_splits=num_split, random_state=h, shuffle=True)
 for i,(train_ix, test_ix)in tqdm(enumerate(kf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
clf = LGBMRegressor(n_estimators=9999, random_state=71, colsample_bytree=0.9,
learning_rate=0.05, min_child_samples=20,
min_child_weight=0.001, min_split_gain=0.0, num_leaves=15)
 clf.fit(X_train_, y_train_, early_stopping_rounds=stop_round, eval_metric='rmse', eval_set=[(X_val, y_val)])
 y_pred = clf.predict(X_val)
 score = mean_squared_error(y_val, y_pred)**0.5
scores.append(score)
 y_pred_cva += clf.predict(X_test)
 print(clf.predict(X_test))
print(np.mean(scores))
print(scores)
 y_pred_cva /=(num_split * num_iter )
inputs = Input(shape=img_dim)
 densenet201 = DenseNet201(weights='imagenet', include_top=False )(inputs)
 flat1 = Flatten()(densenet201)
 dense1 = Dense(units=256, use_bias=True )(flat1)
 batchnorm1 = BatchNormalization()(dense1)
 act1 = Activation(activation='relu' )(batchnorm1)
 drop1 = Dropout(rate=0.5 )(act1)
 out = Dense(units=1, activation='sigmoid' )(drop1)
 model = Model(inputs=inputs, outputs=out)
 model.compile(optimizer='adam', loss='binary_crossentropy' )<train_model>
y_pred_exp = np.exp(y_pred_cva)- 1
y_pred_exp
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=3, verbose=2, mode='max')
 img_aug = ImageDataGenerator(rotation_range=20, vertical_flip=True, horizontal_flip=True)
 img_aug.fit(x_train )<train_model>
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission.ConvertedSalary = y_pred_exp
<predict_on_test><EOS> 
submission.to_csv('submission.csv')
 submission