Kaggle competition: Exam for Students20200129
test_df.has_cactus = preds.numpy() [:, 0]<save_to_csv>
plt.style.use('ggplot')
 %matplotlib inline
os.getcwd()
pd.set_option('display.max_columns', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
test_df.to_csv('submission.csv', index=False )<set_options>
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 %matplotlib inline
%matplotlib inline
train_on_gpu = True 
<install_modules>
sample_submission = pd.read_csv(".. /input/exam-for-students20200129/sample_submission.csv", index_col=0)
 df_test = pd.read_csv(".. /input/exam-for-students20200129/test.csv")
 df_train = pd.read_csv(".. /input/exam-for-students20200129/train.csv" )
!pip install albumentations > /dev/null 2>&1
!pip install pretrainedmodels > /dev/null 2>&1
!pip install kekas > /dev/null 2>&1
!pip install adabound > /dev/null 2>&1<import_modules>
df_info = pd.read_csv(".. /input/exam-for-students20200129/country_info.csv" )
import albumentations
from albumentations import torch as AT
import pretrainedmodels
import adabound
from kekas import Keker, DataOwner, DataKek
from kekas.transformations import Transformer, to_torch, normalize
from kekas.metrics import accuracy
from kekas.modules import Flatten, AdaptiveConcatPool2d
from kekas.callbacks import Callback, Callbacks, DebuggerCallback
from kekas.utils import DotDict<load_from_csv>
df_train = df_train.merge(df_info, on=['Country'], how='left')
 df_test = df_test.merge(df_info, on=['Country'], how='left' )
labels = pd.read_csv('.. /input/train.csv')
 fig = plt.figure(figsize=(25, 8))
train_imgs = os.listdir(".. /input/train/train")
 for idx, img in enumerate(np.random.choice(train_imgs, 20)) :
ax = fig.add_subplot(4, 20//4, idx+1, xticks=[], yticks=[])
 im = Image.open(".. /input/train/train/" + img)
 plt.imshow(im)
 lab = labels.loc[labels['id'] == img, 'has_cactus'].values[0]
ax.set_title(f'Label: {lab}' )<create_dataframe>
df_train['ConvertedSalary'] = df_train['ConvertedSalary'].apply(np.log1p)
test_img = os.listdir('.. /input/test/test')
 test_df = pd.DataFrame(test_img, columns=['id'])
 test_df['has_cactus'] = -1
test_df['data_type'] = 'test'
labels['has_cactus'] = labels['has_cactus'].astype(int)
 labels['data_type'] = 'train'
labels.head()<count_values>
df_train['SalaryType_flg'] = df_train.SalaryType.isnull()
df_test['SalaryType_flg'] = df_test.SalaryType.isnull()
df_train['Country_flg'] = df_train.Country.isnull()
df_test['Country_flg'] = df_test.Country.isnull()
df_train['Employment_flg'] = df_train.Employment.isnull()
df_test['Employment_flg'] = df_test.Employment.isnull()
df_train['Currency_flg'] = df_train.Currency.isnull()
df_test['Currency_flg'] = df_test.Currency.isnull()
df_train['YearsCodingProf'] = df_train.YearsCodingProf.isnull()
df_test['YearsCodingProf'] = df_test.YearsCodingProf.isnull()
labels.loc[labels['data_type'] == 'train', 'has_cactus'].value_counts()<split>
y_train=df_train.ConvertedSalary
X_train=df_train.drop(['ConvertedSalary'], axis=1)
 X_test=df_test
train, valid = train_test_split(labels, stratify=labels.has_cactus, test_size=0.2 )<load_from_csv>
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )
def reader_fn(i, row):
image = cv2.imread(f".. /input/{row['data_type']}/{row['data_type']}/{row['id']}")[:,:,::-1]
label = torch.Tensor([row["has_cactus"]])
 return {"image": image, "label": label}<concatenate>
target = 'ConvertedSalary'
col='SalaryType'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'SalaryType_te'})
 X_test = X_test.rename(columns={0: 'SalaryType_te'} )
def augs(p=0.5):
return albumentations.Compose([
albumentations.HorizontalFlip() ,
albumentations.VerticalFlip() ,
albumentations.RandomBrightness() ,
], p=p )<categorify>
target = 'ConvertedSalary'
col='Country'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'Country_te'})
 X_test = X_test.rename(columns={0: 'Country_te'} )
def get_transforms(dataset_key, size, p):
PRE_TFMS = Transformer(dataset_key, lambda x: cv2.resize(x,(size, size)))
 AUGS = Transformer(dataset_key, lambda x: augs()(image=x)["image"])
 NRM_TFMS = transforms.Compose([
Transformer(dataset_key, to_torch()),
Transformer(dataset_key, normalize())
 ])
 train_tfms = transforms.Compose([PRE_TFMS, AUGS, NRM_TFMS])
 val_tfms = transforms.Compose([PRE_TFMS, NRM_TFMS])
 return train_tfms, val_tfms<feature_engineering>
target = 'ConvertedSalary'
col='Employment'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'Employment_te'})
 X_test = X_test.rename(columns={0: 'Employment_te'} )
train_tfms, val_tfms = get_transforms("image", 32, 0.5 )<load_pretrained>
train_dk = DataKek(df=train, reader_fn=reader_fn, transforms=train_tfms)
 val_dk = DataKek(df=valid, reader_fn=reader_fn, transforms=val_tfms)
 batch_size = 64
workers = 0
train_dl = DataLoader(train_dk, batch_size=batch_size, num_workers=workers, shuffle=True, drop_last=True)
 val_dl = DataLoader(val_dk, batch_size=batch_size, num_workers=workers, shuffle=False )<load_pretrained>
test_dk = DataKek(df=test_df, reader_fn=reader_fn, transforms=val_tfms)
 test_dl = DataLoader(test_dk, batch_size=batch_size, num_workers=workers, shuffle=False )<init_hyperparams>
class Net(nn.Module):
def __init__(
 self,
num_classes: int,
p: float = 0.2,
pooling_size: int = 2,
last_conv_size: int = 1664,
arch: str = "densenet169",
pretrained: str = "imagenet")-> None:

 super().__init__()
net = pretrainedmodels.__dict__[arch](pretrained=pretrained)
 modules = list(net.children())[:-1]
modules += [nn.Sequential(
 Flatten() ,
nn.BatchNorm1d(1664),
nn.Dropout(p),
nn.Linear(1664, num_classes)
)]
self.net = nn.Sequential(*modules)
 def forward(self, x):
logits = self.net(x)
 return logits<create_dataframe>
dataowner = DataOwner(train_dl, val_dl, None)
 model = Net(num_classes=1)
 criterion = nn.BCEWithLogitsLoss()<categorify>
def step_fn(model: torch.nn.Module,
batch: torch.Tensor)-> torch.Tensor:

 inp = batch["image"]
return model(inp )<compute_test_metric>
def bce_accuracy(target: torch.Tensor,
preds: torch.Tensor,
thresh: bool = 0.5)-> float:
target = target.cpu().detach().numpy()
preds =(torch.sigmoid(preds ).cpu().detach().numpy() > thresh ).astype(int)
 return accuracy_score(target, preds)
 def roc_auc(target: torch.Tensor,
preds: torch.Tensor)-> float:
target = target.cpu().detach().numpy()
preds = torch.sigmoid(preds ).cpu().detach().numpy()
return roc_auc_score(target, preds )<choose_model_class>
encoder =OrdinalEncoder(cols=cats)
 X_train[cats] = encoder.fit_transform(X_train[cats])
 X_test[cats] = encoder.transform(X_test[cats] )
keker = Keker(model=model,
dataowner=dataowner,
criterion=criterion,
step_fn=step_fn,
target_key="label",
metrics={"acc": bce_accuracy, 'auc': roc_auc},
opt=torch.optim.SGD,
opt_params={"momentum": 0.99} )<load_pretrained>
X_train.fillna(-9999,inplace=True)
 X_test.fillna(-9999,inplace=True )
keker.unfreeze(model_attr="net")
 layer_num = -1
keker.freeze_to(layer_num, model_attr="net" )<find_best_params>
keker.kek_one_cycle(max_lr=1e-2,
cycle_len=4,
momentum_range=(0.95, 0.85),
div_factor=25,
increase_fraction=0.3,
logdir='train_logs')
 keker.plot_kek('train_logs' )<find_best_params>
keker.kek_one_cycle(max_lr=1e-3,
cycle_len=4,
momentum_range=(0.95, 0.85),
div_factor=25,
increase_fraction=0.2,
logdir='train_logs1')
 keker.plot_kek('train_logs1' )<predict_on_test>
sample_submission = pd.read_csv(".. /input/exam-for-students20200129/sample_submission.csv", index_col=0)
 df_test = pd.read_csv(".. /input/exam-for-students20200129/test.csv")
 df_train = pd.read_csv(".. /input/exam-for-students20200129/train.csv" )
preds = keker.predict_loader(loader=test_dl )<categorify>
df_train['SalaryType_flg'] = df_train.SalaryType.isnull()
df_test['SalaryType_flg'] = df_test.SalaryType.isnull()
df_train['Country_flg'] = df_train.Country.isnull()
df_test['Country_flg'] = df_test.Country.isnull()
df_train['Employment_flg'] = df_train.Employment.isnull()
df_test['Employment_flg'] = df_test.Employment.isnull()
df_train['Currency_flg'] = df_train.Currency.isnull()
df_test['Currency_flg'] = df_test.Currency.isnull()
df_train['YearsCodingProf'] = df_train.YearsCodingProf.isnull()
df_test['YearsCodingProf'] = df_test.YearsCodingProf.isnull()
<save_to_csv>
df_train['ConvertedSalary'] = df_train['ConvertedSalary'].apply(np.log1p)
test_preds = pd.DataFrame({'imgs': test_df.id.values, 'preds': preds.reshape(-1,)})
 test_preds.columns = ['id', 'has_cactus']
test_preds.to_csv('sub.csv', index=False)
 test_preds.head()<set_options>
y_train=df_train.ConvertedSalary
X_train=df_train.drop(['ConvertedSalary'], axis=1)
 X_test=df_test
%matplotlib inline
print(os.listdir(".. /input/")) 
<load_pretrained>
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )
def loadImagesData(glob_path):
images = []
names = []
for img_path in glob.glob(glob_path):
names.append(os.path.basename(img_path))
img = cv2.imread(img_path, cv2.IMREAD_COLOR)
 images.append(img)
 return(images,names)
 trainData = {}
namesData = {}
for label in os.listdir('.. /input/train/'): 
(images,names)= loadImagesData(f".. /input/train/{label}/*.jpg")
 print(f".. /input/train/{label}/*.jpg")
 trainData[label] = images
namesData[label] = names
print("train labels:", ",".join(trainData.keys()))
print(len(trainData['train']))
plt.figure(figsize=(4,2))
columns = 4
for i in range(0,8):
plt.subplot(8 / columns + 1, columns, i + 1)
 plt.imshow(trainData['train'][i])
 plt.show()<prepare_x_and_y>
target = 'ConvertedSalary'
col='SalaryType'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'SalaryType_te'})
 X_test = X_test.rename(columns={0: 'SalaryType_te'} )
train_meta = pd.read_csv('.. /input/train.csv')
 print(train_meta.shape)
 print(train_meta.has_cactus.value_counts())
 lookupY = {}
for i in range(0,len(train_meta)) :
row = train_meta.iloc[i,:]
lookupY[row.id] = row.has_cactus
train_meta.head()<count_values>
target = 'ConvertedSalary'
col='Country'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'Country_te'})
 X_test = X_test.rename(columns={0: 'Country_te'} )
trainList = []
maxCount = 4364
counts = {'0':0,'1':0}
for(i,image)in enumerate(trainData['train']):
label = lookupY[namesData['train'][i]]
counts[str(label)] = 1 + counts[str(label)]
if counts[str(label)] < maxCount:
trainList.append({
'label': label,
'data': image
})
 random.shuffle(trainList)
 train_df = pd.DataFrame(trainList)
 gc.collect()
print(train_df.shape)
 print(train_df.label.value_counts())
 train_df.head()<data_type_conversions>
target = 'ConvertedSalary'
col='Employment'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'Employment_te'})
 X_test = X_test.rename(columns={0: 'Employment_te'} )
data_stack = np.stack(train_df['data'].values)
 dfloats = data_stack.astype(np.float)
 all_x = np.multiply(dfloats, 1.0 / 255.0)
 print(all_x.shape)
 print(type(all_x))
all_x[0,0,0,0]<data_type_conversions>
target = 'ConvertedSalary'
col='Currency'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'Currency_te'})
 X_test = X_test.rename(columns={0: 'Currency_te'} )
all_y = np.array(train_df.label ).astype(np.float)
 all_y[0:5]<split>
target = 'ConvertedSalary'
col='MilitaryUS'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
enc_test = X_test[col].map(summary)
 enc_test = enc_test.rename(columns={'purpose': 'purpose_te'})
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train=pd.concat([X_train, enc_train], axis=1)
 X_test=pd.concat([X_test, enc_test], axis=1)
 X_train = X_train.rename(columns={0: 'MilitaryUS_te'})
 X_test = X_test.rename(columns={0: 'MilitaryUS_te'} )
train_x,test_x,train_y,test_y=train_test_split(all_x,all_y,test_size=0.2,random_state=7)
 print(train_x.shape,test_x.shape )<train_model>
encoder =OrdinalEncoder(cols=cats)
 X_train[cats] = encoder.fit_transform(X_train[cats])
 X_test[cats] = encoder.transform(X_test[cats] )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
rotation_range=60,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True)
 datagen.fit(train_x )<choose_model_class>
X_train.fillna(-9999,inplace=True)
 X_test.fillna(-9999,inplace=True )
num_filters = 8
input_shape = train_x.shape[1:]
output_shape = 1
m = Sequential()
def tdsNet(m):
m.add(Conv2D(32, kernel_size=3, activation='relu', input_shape=input_shape))
m.add(Conv2D(16, kernel_size=3, activation='relu'))
m.add(Flatten())
 m.add(Dropout(0.5))
m.add(Dense(units = output_shape, activation='sigmoid'))
tdsNet(m)
 m.compile(optimizer = 'nadam',
loss = 'binary_crossentropy',
metrics = ['accuracy'])
 m.summary()<train_model>
%%time
X_train,X_val,y_train,y_val=train_test_split(X_train,y_train,random_state=71)
 train_dataset=lgb.Dataset(X_train,y_train)
 valid_dataset=lgb.Dataset(X_val,y_val,reference=train_dataset)
 params={"objective":"regression",
"metric":"rmse"
}
model2=lgb_tuner.train(params,
train_set=train_dataset,
valid_sets=[valid_dataset],
num_boost_round=300,
early_stopping_rounds=50 
)
 y_pred_tuned = model2.predict(X_val)
 tuned_metric = mean_squared_error(y_val, y_pred_tuned)**0.5
print('tuned model metric: ', tuned_metric)
batch_size = 32
history = m.fit_generator(datagen.flow(train_x, train_y,
batch_size=batch_size),
steps_per_epoch=(train_x.shape[0] // batch_size),
epochs = 4,
validation_data=(test_x, test_y),
workers=4 )<choose_model_class>
y_pred2= np.expm1(model2.predict(X_test))
num_filters = 8
input_shape = train_x.shape[1:]
output_shape = 1
m = Sequential()
def cnnNet(m):
m.add(Conv2D(32, kernel_size=3, input_shape=input_shape))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(Conv2D(32, kernel_size=3))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(Dropout(0.25))
m.add(Conv2D(64, kernel_size=3))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(Dropout(0.25))
m.add(Conv2D(64, kernel_size=3))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(MaxPooling2D(2,2))
m.add(Dropout(0.25))
m.add(Conv2D(128, kernel_size=3))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(Dropout(0.25))
m.add(Conv2D(128, kernel_size=3))
m.add(BatchNormalization())
 m.add(Activation("relu"))
m.add(MaxPooling2D(2,2))
m.add(Dropout(0.25))
m.add(Flatten())
 m.add(Dense(64, activation='relu'))
m.add(BatchNormalization())
 m.add(Dropout(0.5))
m.add(Dense(units = output_shape, activation='sigmoid'))

 cnnNet(m)
 m.compile(optimizer = 'nadam',
loss = 'binary_crossentropy',
metrics = ['accuracy'])
 m.summary()<train_model>
sample_submission = pd.read_csv(".. /input/exam-for-students20200129/sample_submission.csv", index_col=0)
 df_test = pd.read_csv(".. /input/exam-for-students20200129/test.csv")
 df_train = pd.read_csv(".. /input/exam-for-students20200129/train.csv")
 df_train['SalaryType_flg'] = df_train.SalaryType.isnull()
df_test['SalaryType_flg'] = df_test.SalaryType.isnull()
df_train['Country_flg'] = df_train.Country.isnull()
df_test['Country_flg'] = df_test.Country.isnull()
df_train['Employment_flg'] = df_train.Employment.isnull()
df_test['Employment_flg'] = df_test.Employment.isnull()
df_train['Currency_flg'] = df_train.Currency.isnull()
df_test['Currency_flg'] = df_test.Currency.isnull()
df_train['YearsCodingProf'] = df_train.YearsCodingProf.isnull()
df_test['YearsCodingProf'] = df_test.YearsCodingProf.isnull()
df_train['ConvertedSalary'] = df_train['ConvertedSalary'].apply(np.log1p)
 y_train=df_train.ConvertedSalary
X_train=df_train.drop(['ConvertedSalary'], axis=1)
 X_test=df_test
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique())
 encoder =OrdinalEncoder(cols=cats)
 X_train[cats] = encoder.fit_transform(X_train[cats])
 X_test[cats] = encoder.transform(X_test[cats])
 X_train.fillna(-9999,inplace=True)
 X_test.fillna(-9999,inplace=True)
batch_size = 64
history = m.fit_generator(datagen.flow(train_x, train_y,
batch_size=batch_size),
steps_per_epoch=(train_x.shape[0] // batch_size),
epochs = 95,
validation_data=(test_x, test_y),
workers=4 )<split>
%%time
X_train,X_val,y_train,y_val=train_test_split(X_train,y_train,random_state=71)
 train_dataset=lgb.Dataset(X_train,y_train)
 valid_dataset=lgb.Dataset(X_val,y_val,reference=train_dataset)
 params={"objective":"regression",
"metric":"rmse"
}
model3=lgb_tuner.train(params,
train_set=train_dataset,
valid_sets=[valid_dataset],
num_boost_round=300,
early_stopping_rounds=50 
)
 y_pred_tuned = model3.predict(X_val)
 tuned_metric = mean_squared_error(y_val, y_pred_tuned)**0.5
print('tuned model metric: ', tuned_metric)
trainList = []
for(i,image)in enumerate(trainData['train']):
label = lookupY[namesData['train'][i]]
trainList.append({
'label': label,
'data': image
})
 random.shuffle(trainList)
 train_df = pd.DataFrame(trainList)
 gc.collect()
data_stack = np.stack(train_df['data'].values)
 dfloats = data_stack.astype(np.float)
 all_x = np.multiply(dfloats, 1.0 / 255.0)
 all_x.shape
all_y = np.array(train_df.label ).astype(np.float)
 train_x,test_x,train_y,test_y=train_test_split(all_x,all_y,test_size=0.2,random_state=7)
 print(train_x.shape,test_x.shape )<train_model>
y_pred3 = np.expm1(model3.predict(X_test))
batch_size = 128
history = m.fit_generator(datagen.flow(train_x, train_y,
batch_size=batch_size),
steps_per_epoch=(train_x.shape[0] // batch_size),
epochs = 95,
validation_data=(test_x, test_y),
workers=4 )<save_to_csv>
y_pred_x=0.5*y_pred2+0.5*y_pred3
( test_images, test_names)= loadImagesData(f".. /input/test/test/*.jpg")
 data_stack = np.stack(test_images)
 dfloats = data_stack.astype(np.float32)
 unknown_x = np.multiply(dfloats, 1.0 / 255.0)
 predicted = np.ravel(m.predict(unknown_x))
submission_df = pd.DataFrame({'id':test_names,'has_cactus':predicted})
 submission_df.to_csv('submission.csv', index=False)
 len(submission_df )<import_modules>
sample_submission = pd.read_csv(".. /input/exam-for-students20200129/sample_submission.csv", index_col=0)
 sample_submission.ConvertedSalary = y_pred_x
now = datetime.datetime.now()
filename = './output/log_' + now.strftime('%Y%m%d_%H%M%S')+ '.csv'
sample_submission.to_csv(now.strftime('%Y%m%d_%H%M%S')+ '_sample_submission.csv' )
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch<define_variables>