Kaggle competition: Pycon Korea 2018 - Tutorial
<SOS> metric: logloss Kaggle data source: pycon-korea-2018-tutorial<create_dataframe>
warnings.filterwarnings('ignore')
 print(os.listdir(".. /input"))
train_val = 0.8
train = data.sample(frac=train_val, random_state=RANDOM_STATE)
 val = data.drop(train.index ).reset_index(drop=True)
 train = train.reset_index(drop=True)
 train_input_ids, train_attention_masks, train_labels = prepare_features(
 train['text'], train['target'])
 val_input_ids, val_attention_masks, val_labels = prepare_features(
 val['text'], val['target'])
 test_input_ids, test_attention_masks = prepare_features(
 test['text'])
 training_set = TensorDataset(train_input_ids, train_attention_masks, train_labels)
 validation_set = TensorDataset(val_input_ids, val_attention_masks, val_labels)
 test_set = TensorDataset(test_input_ids, test_attention_masks)
 loading_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'drop_last': False,
'num_workers': 1}
loading_params_no_shuffle = {'batch_size': BATCH_SIZE,
'shuffle': False,
'drop_last': False,
'num_workers': 1}
training_loader = DataLoader(training_set, **loading_params)
 validation_loader = DataLoader(validation_set, **loading_params)
 training_loader_no_shuffle = DataLoader(training_set, **loading_params_no_shuffle)
 validation_loader_no_shuffle = DataLoader(validation_set, **loading_params_no_shuffle)
 testing_loader = DataLoader(test_set, **loading_params_no_shuffle )<choose_model_class>
baseline_tree_score = 0.23092278864723115
baseline_neuralnetwork_score = 0.5480561937041435
optimizer = AdamW(model.parameters() ,
lr = LEARNING_RATE,
eps = EPSILON 
)
 total_steps = len(training_loader)* MAX_EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<data_type_conversions>
train = pd.read_csv('.. /input/kaggletutorial/covertype_train.csv')
 test = pd.read_csv('.. /input/kaggletutorial/covertype_test.csv' )
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<train_model>
train_index = train.shape[0]
for epoch in tqdm_notebook(range(MAX_EPOCHS)) :
t0 = time.time()
total_train_loss = 0
model.train()
print("EPOCH -- {} / {}".format(epoch, MAX_EPOCHS))
for step, batch in enumerate(training_loader):
if step % 30 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {} of {}.Elapsed: {:}'.format(step, len(training_loader), elapsed))
input_ids = batch[0].to(device ).to(torch.int64)
 input_masks = batch[1].to(device ).to(torch.int64)
 labels = batch[2].to(device ).to(torch.int64)
 model.zero_grad()
loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=input_masks,
labels=labels)
 logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(training_loader)
 training_time = format_time(time.time() - t0)
 print('')
 print(' Average training loss: {0:.4f}'.format(avg_train_loss))
print(' Training epoch took: {:}'.format(training_time))
print('Running Validation')
 model.eval()
val_predictions = []
val_labels = []
for batch in validation_loader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 val_predictions.append(logits.detach().cpu().numpy())
 val_labels.append(b_labels.to('cpu' ).numpy())
 val_predictions = np.array([item for sublist in val_predictions for item in sublist])
 val_labels = np.array([item for sublist in val_labels for item in sublist])
 val_predictions_flat = np.argmax(val_predictions, axis=1)
 print(val_predictions_flat.shape)
 val_accuracy = accuracy_score(val_labels, val_predictions_flat)
 val_recall = recall_score(val_labels, val_predictions_flat)
 val_precision = precision_score(val_labels, val_predictions_flat)
 val_f1 = f1_score(val_labels, val_predictions_flat)
 print('\tAccuracy: {0:.4f}
\tRecall: {1:.4f}
\tPrecision: {2:.4f}
\tF1_score: {3:.4f}
\t'.
 format(val_accuracy, val_recall, val_precision, val_f1))<predict_on_test>
lgbm_param = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.06,
"num_leaves": 16,
"max_depth": 6,
"colsample_bytree": 0.7,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}
model.eval()
roberta_train_preds = []
for batch in training_loader_no_shuffle:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 roberta_train_preds.append(logits.detach().cpu().numpy())
 roberta_train_preds = np.array([item for sublist in roberta_train_preds for item in sublist])
 roberta_val_preds = []
for batch in validation_loader_no_shuffle:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 roberta_val_preds.append(logits.detach().cpu().numpy())
 roberta_val_preds = np.array([item for sublist in roberta_val_preds for item in sublist])
 roberta_test_preds = []
for batch in testing_loader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 with torch.no_grad() :
logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)[0]
roberta_test_preds.append(logits.detach().cpu().numpy())
 roberta_test_preds = np.array([item for sublist in roberta_test_preds for item in sublist])
 roberta_train_preds_binary = np.argmax(roberta_train_preds, axis=1)
 roberta_val_preds_binary = np.argmax(roberta_val_preds, axis=1)
 roberta_test_preds_binary = np.argmax(roberta_test_preds, axis=1)
 roberta_train_preds = pd.DataFrame(roberta_train_preds, columns=["roberta_fake","roberta_real"])
 roberta_val_preds = pd.DataFrame(roberta_val_preds, columns=["roberta_fake","roberta_real"])
 roberta_test_preds = pd.DataFrame(roberta_test_preds, columns=["roberta_fake","roberta_real"] )<prepare_x_and_y>
def keras_model(input_dims):
model = Sequential()
model.add(Dense(input_dims, input_dim=input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//2))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
 return model
def keras_history_plot(history):
plt.plot(history.history['loss'], 'y', label='train loss')
 plt.plot(history.history['val_loss'], 'r', label='val loss')
 plt.xlabel('epoch')
 plt.ylabel('loss')
 plt.legend(loc='upper right')
 plt.show()
train_stacking = train.loc[:, ["target"]]
val_stacking = val.loc[:, ["target"]]
test_stacking = test.loc[:, []]
train_stacking["svm_predicted"] = svm_train_preds
train_stacking = pd.concat([train_stacking, roberta_train_preds], axis=1)
 X_train_stacking = train_stacking.drop(["target"], axis=1)
 y_train_stacking = train_stacking["target"]
val_stacking["svm_predicted"] = svm_val_preds
val_stacking = pd.concat([val_stacking, roberta_val_preds], axis=1)
 X_val_stacking = val_stacking.drop(["target"], axis=1)
 y_val_stacking = val_stacking["target"]
test_stacking["svm_predicted"] = svm_test_preds
test_stacking = pd.concat([test_stacking, roberta_test_preds], axis=1)
 X_test_stacking = test_stacking<predict_on_test>
def baseline_tree_cv(train):
train_df = train.copy()
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
NFOLD = 5
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_iteration = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
evals_result_dict = {}
dtrain = lgbm.Dataset(train_x, label=train_y)
 dvalid = lgbm.Dataset(valid_x, label=valid_y)
 clf = lgbm.train(lgbm_param, train_set=dtrain, num_boost_round=3000, valid_sets=[dtrain, dvalid],
early_stopping_rounds=200, evals_result=evals_result_dict, verbose_eval=500)
 predict = clf.predict(valid_x)
 cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_iteration = max(best_iteration, clf.best_iteration)
 print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
lgbm.plot_metric(evals_result_dict)
 plt.show()
print("Best Iteration", best_iteration)
 print("Total LogLoss", total_score / NFOLD)
 print("Baseline model Score Diff", total_score / NFOLD - baseline_tree_score)
 del train_df
return best_iteration
def baseline_keras_cv(train):
train_df = train.copy()
y_value = train_df['Cover_Type']
del train_df['Cover_Type'], train_df['ID']
model = keras_model(train_df.shape[1])
 callbacks = [
EarlyStopping(
 patience=10,
verbose=10)
 ]
NFOLD = 5
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_epoch = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
history = model.fit(train_x.values, train_y.values, nb_epoch=30, batch_size = 64, validation_data=(valid_x.values, valid_y.values),
verbose=1, callbacks=callbacks)
 keras_history_plot(history)
 predict = model.predict(valid_x.values)
 null_count = np.sum(pd.isnull(predict))
if null_count > 0:
print("Null Prediction Error: ", null_count)
 predict[pd.isnull(predict)] = predict[~pd.isnull(predict)].mean()
cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_epoch = max(best_epoch, np.max(history.epoch))
print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
print("Best Epoch: ", best_epoch)
 print("Total LogLoss", total_score/NFOLD)
 print("Baseline model Score Diff", total_score/NFOLD - baseline_neuralnetwork_score )
svm_clf_stacking = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
 svm_clf_stacking.fit(X_train_stacking, y_train_stacking)
 pred = svm_clf_stacking.predict(X_val_stacking)
 print(f1_score(y_val_stacking,pred))<train_model>
def outlier_binary(frame, col, outlier_range):
outlier_feature = col + '_Outlier'
frame[outlier_feature] = 0
frame.loc[frame[col] > outlier_range, outlier_feature] = 1
return frame
def outlier_divide_ratio(frame, col, outlier_range):
outlier_index = frame[col] >= outlier_range
outlier_median = frame.loc[outlier_index, col].median()
normal_median = frame.loc[frame[col] < outlier_range, col].median()
outlier_ratio = outlier_median / normal_median
frame.loc[outlier_index, col] = frame.loc[outlier_index, col]/outlier_ratio
return frame
def frequency_encoding(frame, col):
freq_encoding = frame.groupby([col] ).size() /frame.shape[0]
freq_encoding = freq_encoding.reset_index().rename(columns={0:'{}_Frequncy'.format(col)})
 return frame.merge(freq_encoding, on=col, how='left')
 def binning_category_combine_feature(frame, col1, col2, col1_quantile, col2_quantile):
print(col1, ' ', col2, 'Bining Combine')
 col1_quantile = np.arange(0,1.1,col1_quantile)
 col2_quantile = np.arange(0,1.1,col2_quantile)
 col1_label = '{}_quantile_label'.format(col1)
 frame[col1_label] = pd.qcut(frame[col1], q=col1_quantile, labels = ['{}_quantile_{:.1f}'.format(col1, col)for col in col1_quantile][1:])
 col2_label = '{}_quantile_label'.format(col2)
 frame[col2_label] = pd.qcut(frame[col2], q=col2_quantile, labels = ['{}_quantile_{:.1f}'.format(col2, col)for col in col2_quantile][1:])
 combine_label = 'Binnig_{}_{}_Combine'.format(col1, col2)
 frame[combine_label] = frame[[col1_label, col2_label]].apply(lambda row: row[col1_label] +'_'+ row[col2_label] ,axis=1)
 for col in [col1_label, col2_label, combine_label]:
frame[col] = frame[col].factorize() [0]
gc.collect()
return frame, [col1_label, col2_label, combine_label]
lr_clf_stacking = LogisticRegression(fit_intercept=True,
random_state=RANDOM_STATE,
solver = 'liblinear',
penalty = 'l1',
max_iter = 200)
 lr_clf_stacking.fit(X_train_stacking, y_train_stacking)
 lr_preds = lr_clf_stacking.predict(X_train_stacking)
 print(accuracy_score(y_train_stacking,lr_preds))<train_model>
def tree_data_preprocessing(train, test):
train_index = train.shape[0]
all_data = pd.concat([train, test])
 del all_data['oil_Type']
all_column_set = set(all_data.columns)
 category_feature = []
for col in all_data.loc[:, all_data.dtypes=='object'].columns:
all_data[col] = all_data[col].factorize() [0]
category_feature.append(col)
 numerical_feature = list(all_column_set - set(category_feature)- set(['Cover_Type','ID']))
all_data['Elevation'] = np.log1p(all_data['Elevation'])
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = frequency_encoding(all_data, 'Soil_Type')
 all_data = frequency_encoding(all_data, 'Wilderness_Area')
 aspect_train = all_data.loc[all_data['Aspect'].notnull() ]
aspect_test = all_data.loc[all_data['Aspect'].isnull() ]
del aspect_train["Cover_Type"], aspect_train['ID']
del aspect_test["Cover_Type"], aspect_test['ID']
numerical_feature_woaspect = numerical_feature[:]
numerical_feature_woaspect.remove('Aspect')
 sc = StandardScaler()
aspect_train[numerical_feature_woaspect] = sc.fit_transform(aspect_train[numerical_feature_woaspect])
 aspect_test[numerical_feature_woaspect] = sc.transform(aspect_test[numerical_feature_woaspect])
 y_value = aspect_train['Aspect']
del aspect_train['Aspect'], aspect_test['Aspect']
knn = KNeighborsRegressor(n_neighbors=7)
 knn.fit(aspect_train,y_value)
 predict = knn.predict(aspect_test)
 sns.distplot(predict)
 sns.distplot(all_data['Aspect'].dropna())
 plt.title('KNN Aspect Null Imputation')
 plt.show()
all_data.loc[all_data['Aspect'].isnull() ,'Aspect'] = predict
all_data['Horizontal_Distance_To_Hydrology'] = all_data['Horizontal_Distance_To_Hydrology']/1000
all_data['HF1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Fire_Points'])
 all_data['HF4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Fire_Points']
all_data['HR1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Roadways']
all_data['HR2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Roadways']
all_data['HR3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['HR4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Roadways']
all_data['HH1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Vertical_Distance_To_Hydrology']
all_data['HH2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Vertical_Distance_To_Hydrology']
all_data['HH3'] = np.log1p(abs(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Vertical_Distance_To_Hydrology']))
all_data['HH4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Vertical_Distance_To_Hydrology']
all_data['FR1'] = all_data['Horizontal_Distance_To_Fire_Points'] + all_data['Horizontal_Distance_To_Roadways']
all_data['FR2'] = all_data['Horizontal_Distance_To_Fire_Points'] - all_data['Horizontal_Distance_To_Roadways']
all_data['FR3'] = np.log1p(all_data['Horizontal_Distance_To_Fire_Points'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['FR4'] = all_data['Horizontal_Distance_To_Fire_Points'] / all_data['Horizontal_Distance_To_Roadways']
all_data['Direct_Distance_Hydrology'] =(all_data['Horizontal_Distance_To_Hydrology']**2+all_data['Vertical_Distance_To_Hydrology']**2)**0.5
all_data.loc[np.isinf(all_data['HF4']),'HF4'] = 0
all_data.loc[np.isinf(all_data['HR4']),'HR4'] = 0
all_data.loc[np.isinf(all_data['HH4']),'HH4'] = 0
all_data.loc[np.isinf(all_data['FR4']),'FR4'] = 0
all_data[['HF4','HH4']] = all_data[['HF4','HH4']].fillna(0)
 all_data, new_col = binning_category_combine_feature(all_data, 'Elevation', 'Aspect', 0.1, 0.1)
 for col in new_col:
all_data = frequency_encoding(all_data, col)
 train_df = all_data.iloc[:train_index]
test_df = all_data.iloc[train_index:]
soil_mean_encoding = train_df.groupby(['Soil_Type'])['Cover_Type'].agg({'Soil_Type_Mean':'mean',
'Soil_Type_Std':'std',
'Soil_Type_Size':'size',
'Soil_Type_Sum':'sum'} ).reset_index()
train_df = train_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 test_df = test_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 wildness_mean_encoding = train_df.groupby(['Wilderness_Area'])['Cover_Type'].agg({'Wilderness_Area_Mean':'mean',
'Wilderness_Area_Std':'std',
'Wilderness_Area_Size':'size',
'Wilderness_Area_Sum':'sum'} ).reset_index()
train_df = train_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 test_df = test_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 del all_data, predict, aspect_train, aspect_test
gc.collect()
return train_df, test_df
rf_clf = RandomForestClassifier(n_estimators = 100, max_depth=2, random_state = RANDOM_STATE)
 rf_clf.fit(X_train_stacking, y_train_stacking)
 rf_preds = rf_clf.predict(X_train_stacking)
 print(accuracy_score(y_train_stacking, rf_preds))<predict_on_test>
def nn_data_preprocessing(train, test):
train_index = train.shape[0]
all_data = pd.concat([train, test])
 del all_data['oil_Type']
all_column_set = set(all_data.columns)
 category_feature = []
for col in all_data.loc[:, all_data.dtypes=='object'].columns:
all_data[col] = all_data[col].factorize() [0]
category_feature.append(col)
 numerical_feature = list(all_column_set - set(category_feature)- set(['Cover_Type','ID']))
all_data['Elevation'] = np.log1p(all_data['Elevation'])
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = frequency_encoding(all_data, 'Soil_Type')
 all_data = frequency_encoding(all_data, 'Wilderness_Area')
 aspect_train = all_data.loc[all_data['Aspect'].notnull() ]
aspect_test = all_data.loc[all_data['Aspect'].isnull() ]
del aspect_train["Cover_Type"], aspect_train['ID']
del aspect_test["Cover_Type"], aspect_test['ID']
numerical_feature_woaspect = numerical_feature[:]
numerical_feature_woaspect.remove('Aspect')
 sc = StandardScaler()
aspect_train[numerical_feature_woaspect] = sc.fit_transform(aspect_train[numerical_feature_woaspect])
 aspect_test[numerical_feature_woaspect] = sc.transform(aspect_test[numerical_feature_woaspect])
 y_value = aspect_train['Aspect']
del aspect_train['Aspect'], aspect_test['Aspect']
knn = KNeighborsRegressor(n_neighbors=7)
 knn.fit(aspect_train,y_value)
 predict = knn.predict(aspect_test)
 sns.distplot(predict)
 sns.distplot(all_data['Aspect'].dropna())
 plt.title('KNN Aspect Null Imputation')
 plt.show()
all_data.loc[all_data['Aspect'].isnull() ,'Aspect'] = predict
all_data['Horizontal_Distance_To_Hydrology'] = all_data['Horizontal_Distance_To_Hydrology']/1000
all_data['HF1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Fire_Points'])
 all_data['HF4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Fire_Points']
all_data['HR1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Roadways']
all_data['HR2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Roadways']
all_data['HR3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['HR4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Roadways']
all_data['HH1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Vertical_Distance_To_Hydrology']
all_data['HH2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Vertical_Distance_To_Hydrology']
all_data['HH3'] = np.log1p(abs(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Vertical_Distance_To_Hydrology']))
all_data['HH4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Vertical_Distance_To_Hydrology']
all_data['FR1'] = all_data['Horizontal_Distance_To_Fire_Points'] + all_data['Horizontal_Distance_To_Roadways']
all_data['FR2'] = all_data['Horizontal_Distance_To_Fire_Points'] - all_data['Horizontal_Distance_To_Roadways']
all_data['FR3'] = np.log1p(all_data['Horizontal_Distance_To_Fire_Points'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['FR4'] = all_data['Horizontal_Distance_To_Fire_Points'] / all_data['Horizontal_Distance_To_Roadways']
all_data['Direct_Distance_Hydrology'] =(all_data['Horizontal_Distance_To_Hydrology']**2+all_data['Vertical_Distance_To_Hydrology']**2)**0.5
all_data.loc[np.isinf(all_data['HF4']),'HF4'] = 0
all_data.loc[np.isinf(all_data['HR4']),'HR4'] = 0
all_data.loc[np.isinf(all_data['HH4']),'HH4'] = 0
all_data.loc[np.isinf(all_data['FR4']),'FR4'] = 0
all_data[['HF4','HH4']] = all_data[['HF4','HH4']].fillna(0)
 all_data, new_col = binning_category_combine_feature(all_data, 'Elevation', 'Aspect', 0.1, 0.1)
 for col in new_col:
all_data = frequency_encoding(all_data, col)
 all_data.drop(columns=new_col,axis=1,inplace=True)
 before_one_hot = set(all_data.columns)
 for col in category_feature:
all_data = pd.concat([all_data,pd.get_dummies(all_data[col],prefix=col)],axis=1)
 one_hot_feature = set(all_data.columns)- before_one_hot
train_df = all_data.iloc[:train_index]
test_df = all_data.iloc[train_index:]
soil_mean_encoding = train_df.groupby(['Soil_Type'])['Cover_Type'].agg({'Soil_Type_Mean':'mean',
'Soil_Type_Std':'std',
'Soil_Type_Size':'size',
'Soil_Type_Sum':'sum'} ).reset_index()
train_df = train_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 test_df = test_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 wildness_mean_encoding = train_df.groupby(['Wilderness_Area'])['Cover_Type'].agg({'Wilderness_Area_Mean':'mean',
'Wilderness_Area_Std':'std',
'Wilderness_Area_Size':'size',
'Wilderness_Area_Sum':'sum'} ).reset_index()
train_df = train_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 test_df = test_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 train_df.drop(columns=category_feature, axis=1, inplace=True)
 test_df.drop(columns=category_feature, axis=1, inplace=True)
 scale_feature = list(set(train_df.columns)-one_hot_feature-set(['Cover_Type','ID']))
sc = StandardScaler()
train_df[scale_feature] = sc.fit_transform(train_df[scale_feature])
 test_df[scale_feature] = sc.transform(test_df[scale_feature])
 return train_df, test_df
test["target"] = lr_clf_stacking.predict(X_test_stacking )<save_to_csv>
org_train_df, org_test_df = tree_data_preprocessing(train, test )
test[['id', 'target']].to_csv("submission_stacking.csv", index=False )<import_modules>
train_df = org_train_df.copy()
test_df = org_test_df.copy()
import pandas as pd
import torch
import transformers
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
from transformers import DistilBertModel, DistilBertTokenizer
from sklearn import metrics
from tqdm import tqdm
import numpy as np
import re<define_variables>
lgbm_param = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 24,
"max_depth": 6,
"colsample_bytree": 0.65,
"subsample": 0.7,
"reg_alpha": 0.1,
"reg_lambda": 0.2,
"nthread":8
}
device = 'cuda' if cuda.is_available() else 'cpu'<load_from_csv>
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
del test_df["Cover_Type"], test_df["ID"]
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")[["text","target"]]<categorify>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_numbers(text):
text = ''.join([i for i in text if not i.isdigit() ])
 return text
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_username(text):
url = re.compile(r'@[A-Za-z0-9_]+')
 return url.sub(r'',text )<string_transform>
dtrain = lgbm.Dataset(train_df, label=y_value)
 clf = lgbm.train(lgbm_param, train_set=dtrain, num_boost_round=5000)
 predict = clf.predict(test_df )
def pre_process_text(text):
text = remove_URL(text)
 text = remove_numbers(text)
 text = remove_html(text)
 text = remove_username(text)
 return " ".join(text.split() )<load_pretrained>
submission = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission['Cover_Type'] = predict
submission.to_csv('lgbm_last.csv', index=False )
MAX_LEN = 160
BATCH_SIZE = 16
EPOCHS = 1
LEARNING_RATE = 1e-05
BERT_PATH = '/kaggle/input/distillbert-huggingface-model/'
MODEL_PATH = "distilbert-base-uncased-pytorch_model.bin"
tokenizer = DistilBertTokenizer.from_pretrained(
 BERT_PATH,
do_lower_case=True
)<categorify>
org_train_df, org_test_df = nn_data_preprocessing(train, test )
class tweet_Dataset(Dataset):
def __init__(self, dataframe, tokenizer, max_len):
self.data = dataframe
self.tokenizer = tokenizer
self.max_len = max_len
def __getitem__(self, index):
tweet = str(self.data.text[index])
 tweet = pre_process_text(tweet)
 inputs = self.tokenizer.encode_plus(
 tweet,
None,
add_special_tokens=True,
max_length=self.max_len,
pad_to_max_length=True,
return_token_type_ids=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.tensor(ids, dtype=torch.long),
'mask': torch.tensor(mask, dtype=torch.long),
'targets': torch.tensor(self.data.target[index], dtype=torch.float)
 }
def __len__(self):
return len(self.data )<load_pretrained>
train_df = org_train_df.copy()
test_df = org_test_df.copy()
train_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
valid_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
train_dl = DataLoader(training_set, **train_params)
 valid_dl = DataLoader(validation_set, **valid_params )<define_search_model>
def keras_model(input_dims):
model = Sequential()
model.add(Dense(input_dims, input_dim=input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//2))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//5))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
 return model
class DistillBERTClass(torch.nn.Module):
def __init__(self):
super().__init__()
self.distill_bert = transformers.DistilBertModel.from_pretrained(BERT_PATH)
 self.drop = torch.nn.Dropout(0.3)
 self.out = torch.nn.Linear(768, 1)
 def forward(self, ids, mask):
distilbert_output = self.distill_bert(ids, mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0]
output_1 = self.drop(pooled_output)
 output = self.out(output_1)
 return output<train_model>
y_value = train_df['Cover_Type']
del train_df['Cover_Type'], train_df['ID']
del test_df['Cover_Type'], test_df['ID']
model = keras_model(train_df.shape[1])
 callbacks = [
EarlyStopping(
 patience=10,
verbose=10)
 ]
model = DistillBERTClass()
model.to(device )<compute_test_metric>
def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets)
 optimizer = torch.optim.Adam(params = model.parameters() , lr=LEARNING_RATE )<categorify>
history = model.fit(train_df.values, y_value.values, nb_epoch=30, batch_size = 64, verbose=1)
 predict = model.predict(test_df.values )
def eval_fn(data_loader, model):
model.eval()
fin_targets = []
fin_outputs = []
with torch.no_grad() :
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)) :
ids = d["ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(ids=ids, mask=mask)
 fin_targets.extend(targets.cpu().detach().numpy().tolist())
 fin_outputs.extend(torch.sigmoid(outputs ).cpu().detach().numpy().tolist())
 fin_outputs = np.array(fin_outputs)>= 0.5
f1 = metrics.f1_score(fin_targets, fin_outputs)
 return f1<train_model>
submission_nn = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_nn['Cover_Type'] = predict
submission_nn.to_csv('nn_last.csv', index=False )
def fit(num_epochs, model, loss_fn, opt, train_dl, valid_dl):
for epoch in range(num_epochs):
model.train()
for _,data in enumerate(train_dl, 0):
ids = data['ids'].to(device, dtype = torch.long)
 mask = data['mask'].to(device, dtype = torch.long)
 targets = data['targets'].to(device, dtype = torch.float)
 outputs = model(ids, mask ).squeeze()
loss = loss_fn(outputs, targets)
 loss.backward()
opt.step()
opt.zero_grad()
valid_acc = eval_fn(valid_dl, model)
 print('Epoch [{}/{}], Train Loss: {:.4f} and Validation f1 {:.4f}'.format(epoch+1, num_epochs, loss.item() ,valid_acc))<train_model>
def calculate_correlation(base_df, target_df):
source = base_df.copy()
source = source.merge(target_df,on='ID')
 corr_df = source.corr()
corr = corr_df.ix['Cover_Type_x']['Cover_Type_y']
del corr_df, source
return corr
fit(7, model, loss_fn, optimizer, train_dl,valid_dl )<string_transform>
source = submission.copy()
source = source.merge(submission_nn,on='ID')
 source
def sentence_prediction(sentence):
max_len = MAX_LEN
tweet = str(sentence)
 tweet = " ".join(tweet.split())
 inputs = tokenizer.encode_plus(
 tweet,
None,
add_special_tokens=True,
max_length=max_len,
pad_to_max_length=True, 
)
 ids = inputs["input_ids"]
mask = inputs["attention_mask"]
ids = torch.tensor(ids, dtype=torch.long ).unsqueeze(0)
 mask = torch.tensor(mask, dtype=torch.long ).unsqueeze(0)
 ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs = model(ids=ids, mask=mask)
 outputs = torch.sigmoid(outputs ).cpu().detach().numpy()
return outputs[0][0] > 0.5<prepare_output>
calculate_correlation(submission, submission_nn )
sentence_prediction("We are experiencing slight tremors in London right now" )<prepare_output>
class SklearnWrapper(object):
def __init__(self, clf, params=None, **kwargs):
seed = kwargs.get('seed', 0)
 params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train, x_cross=None, y_cross=None):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict_proba(x)[:,1]
sentence_prediction("Lol this movie is an absolute disaster!" )<load_from_csv>
class LgbmWrapper(object):
def __init__(self, params=None, **kwargs):
seed = kwargs.get('seed', 0)
 num_rounds = kwargs.get('num_rounds', 1000)
 early_stopping = kwargs.get('ealry_stopping', 100)
 eval_function = kwargs.get('eval_function', None)
 verbose_eval = kwargs.get('verbose_eval', 100)
 self.param = params
self.param['seed'] = seed
self.num_rounds = num_rounds
self.early_stopping = early_stopping
self.eval_function = eval_function
self.verbose_eval = verbose_eval
def train(self, x_train, y_train, x_cross=None, y_cross=None):
need_cross_validation = True
if x_cross is None:
need_cross_validation = False
if isinstance(y_train, pd.DataFrame)is True:
y_train = y_train[y_train.columns[0]]
if need_cross_validation is True:
y_cross = y_cross[y_cross.columns[0]]
if need_cross_validation is True:
dtrain = lgbm.Dataset(x_train, label=y_train, silent=True)
 dvalid = lgbm.Dataset(x_cross, label=y_cross, silent=True)
 self.clf = lgbm.train(self.param, train_set=dtrain, num_boost_round=self.num_rounds, valid_sets=dvalid,
feval=self.eval_function, early_stopping_rounds=self.early_stopping,
verbose_eval=self.verbose_eval)
 else:
dtrain = lgbm.Dataset(x_train, label=y_train, silent= True)
 self.clf = lgbm.train(self.param, dtrain, self.num_rounds)
 def predict(self, x):
return self.clf.predict(x, num_iteration=self.clf.best_iteration)
 def get_params(self):
return self.param
df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")[["id","text"]]<feature_engineering>
def get_oof(clf, x_train, y_train, x_test, eval_func, **kwargs):
nfolds = kwargs.get('NFOLDS', 5)
 kfold_shuffle = kwargs.get('kfold_shuffle', True)
 kfold_random_state = kwargs.get('kfold_random_sate', 0)
 ntrain = x_train.shape[0]
ntest = x_test.shape[0]
kf = StratifiedKFold(n_splits= nfolds, shuffle=kfold_shuffle, random_state=kfold_random_state)
 oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( nfolds, ntest))
cv_sum = 0
try:
if clf.clf is not None:
print(clf.clf)
 except:
print(clf)
 print(clf.get_params())
 for i,(train_index, cross_index)in enumerate(kf.split(x_train, y_train)) :
x_tr, x_cr = None, None
y_tr, y_cr = None, None
if isinstance(x_tr, pd.DataFrame):
x_tr, x_cr = x_train.iloc[train_index], x_train.iloc[cross_index]
y_tr, y_cr = y_train.iloc[train_index], y_train.iloc[cross_index]
else:
x_tr, x_cr = x_train[train_index], x_train[cross_index]
y_tr, y_cr = y_train[train_index], y_train[cross_index]
clf.train(x_tr, y_tr, x_cr, y_cr)
 oof_train[cross_index] = clf.predict(x_cr)
 cv_score = eval_func(y_cr, oof_train[cross_index])
 print('Fold %d / ' %(i+1), 'CV-Score: %.6f' % cv_score)
 cv_sum = cv_sum + cv_score
score = cv_sum / nfolds
print("Average CV-Score: ", score)
 clf.train(x_train, y_train)
 oof_test = clf.predict(x_test)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
df_test['target'] = df_test['text'].apply(sentence_prediction )<data_type_conversions>
lgbm_param1 = {
'boosting_type': 'dart',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 31,
"max_depth": 7,
"colsample_bytree": 0.8,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8,
'drop_rate':0.1,
'skip_drop':0.5,
'max_drop':50,
'top_rate':0.1,
'other_rate':0.1
}
lgbm_param2 = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 10,
"max_depth": 4,
"colsample_bytree": 0.5,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}
lgbm_param3 = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 24,
"max_depth": 6,
"colsample_bytree": 0.5,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}
rf_params = {
'criterion':'gini', 'max_leaf_nodes':24, 'n_estimators':200, 'min_impurity_split':0.0000001,
'max_features':0.4, 'max_depth':6, 'min_samples_leaf':20, 'min_samples_split':2,
'min_weight_fraction_leaf':0.0, 'bootstrap':True,
'random_state':1, 'verbose':False
}
et_parmas = {
'criterion':'gini', 'max_leaf_nodes':31, 'n_estimators':200, 'min_impurity_split':0.0000001,
'max_features':0.6, 'max_depth':10, 'min_samples_leaf':20, 'min_samples_split':2,
'min_weight_fraction_leaf':0.0, 'bootstrap':True,
'random_state':1, 'verbose':False
}
df_test['target'] = df_test['target'].astype(int )<save_to_csv>
org_train_df, org_test_df = tree_data_preprocessing(train, test)
 train_df = org_train_df.copy()
test_df = org_test_df.copy()
df_test.to_csv("submission.csv", columns = ['id','target'], index=False )<set_options>
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
del test_df["Cover_Type"], test_df["ID"]
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline
warnings.filterwarnings('ignore')
 dir_19_name = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 dir_15_name = os.path.join('.. ', 'input/diabetic-retinopathy-resized/')
 sys.path.append(os.path.abspath('.. /input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master')) 
<load_pretrained>
et_model = SklearnWrapper(clf = ExtraTreesClassifier, params=et_parmas)
 rf_model = SklearnWrapper(clf = RandomForestClassifier, params=rf_params )
md_ef = EfficientNet.from_name('efficientnet-b5',override_params={'num_classes':1})
 !mkdir models
!cp '.. /input/kaggle-public/abcdef.pth' 'models'<load_from_csv>
et_train, et_test = get_oof(et_model, train_df.values, y_value, test_df.values, log_loss, NFOLDS=3)
 rf_train, rf_test = get_oof(rf_model, train_df.values, y_value, test_df.values, log_loss, NFOLDS=3 )
def get_df(dir_15_name,dir_19_name):
valid_dir = os.path.join(dir_19_name,'train_images/')
 valid_df = pd.read_csv(os.path.join(dir_19_name,'train.csv'))
valid_df['path'] = valid_df['id_code'].map(lambda x: os.path.join(valid_dir,'{}.png'.format(x)))
 valid_df['is_valid'] = [True] * len(valid_df)
 valid_df = valid_df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv(os.path.join(dir_19_name,'sample_submission.csv'))
train_dir = os.path.join(dir_15_name,'resized_train_cropped/resized_train_cropped/')
 train_df = pd.read_csv(os.path.join(dir_15_name,'trainLabels_cropped.csv'))
train_df['path'] = train_df['image'].map(lambda x: os.path.join(train_dir,'{}.jpeg'.format(x)))
 train_df['diagnosis'] = train_df['level']
train_df['id_code'] = train_df['image']
train_df['is_valid'] = [False] * len(train_df)
 train_df1 = train_df[train_df['diagnosis'] == 0]
train_df2 = train_df[train_df['diagnosis'] != 0]
train_df1 = train_df1.sample(frac=1 ).reset_index(drop=True)
 train_df1 = train_df1[:5000]
train_df = pd.concat([train_df1,train_df2],axis=0,ignore_index=True)
 train_df = train_df.drop(columns = ['Unnamed: 0.1','Unnamed: 0','level','image'])
 train_df = train_df.sample(frac=1 ).reset_index(drop=True)
 return train_df, valid_df, test_df
train_df, valid_df, test_df = get_df(dir_15_name,dir_19_name)
 train_df1 = train_df[:4000]
train_df2 = train_df[4000:8000]
train_df3 = train_df[8000:12000]
train_df4 = train_df[12000:]
valid_df1 = valid_df[:1000]
valid_df2 = valid_df[1000:2000]
valid_df3 = valid_df[2000:3000]
valid_df4 = valid_df[3000:]<concatenate>
x_train_second_layer = np.concatenate(( rf_train, et_train), axis=1)
 x_test_second_layer = np.concatenate(( rf_test, et_test), axis=1 )
res1 = pd.concat([train_df1,valid_df1],axis=0,ignore_index=True)
 res2 = pd.concat([train_df2,valid_df2],axis=0,ignore_index=True)
 res3 = pd.concat([train_df3,valid_df3],axis=0,ignore_index=True)
 res4 = pd.concat([train_df4,valid_df4],axis=0,ignore_index=True )<feature_engineering>
x_train = pd.DataFrame(x_train_second_layer)
 x_test = pd.DataFrame(x_test_second_layer )
batch_size = 64
image_size = 224
transforms = get_transforms(do_flip=True,flip_vert=True,max_rotate=360, max_zoom=1.3, max_warp=0.3, max_lighting=0.4,p_affine = 0.7 )<define_variables>
lgbm_meta_params = {
'boosting':'gbdt', 'num_leaves':28, 'learning_rate':0.03, 'min_sum_hessian_in_leaf':0.1,
'max_depth':7, 'feature_fraction':0.6, 'min_data_in_leaf':30, 'poission_max_delta_step':0.7,
'bagging_fraction':0.8, 'min_gain_to_split':0,
'objective':'binary', 'seed':1,'metric': 'binary_logloss'
}
NFOLD = 3
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_iteration = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(x_train, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
evals_result_dict = {}
dtrain = lgbm.Dataset(train_x, label=train_y)
 dvalid = lgbm.Dataset(valid_x, label=valid_y)
 clf = lgbm.train(lgbm_meta_params, train_set=dtrain, num_boost_round=5000, valid_sets=[dtrain, dvalid],
early_stopping_rounds=200, evals_result=evals_result_dict, verbose_eval=500)
 predict = clf.predict(valid_x)
 cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_iteration = max(best_iteration, clf.best_iteration)
 print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
lgbm.plot_metric(evals_result_dict)
 plt.show()
print("Best Iteration", best_iteration)
 print("Total LogLoss", total_score / NFOLD)
 print("Baseline model Score Diff", total_score / NFOLD - baseline_tree_score )
preds = ['','','','','','','','','','']
i = 0
for sz in [224]:
test =(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
data =(ImageList.from_df(df=valid_df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.add_test(test)
.transform(transforms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.load('abcdef');
preds[i],y = learn.get_preds(DatasetType.Test)
 i = i + 1
preds_A =(preds[0] + preds[1] + preds[2] + preds[3] + preds[4] + preds[5] + preds[6] + preds[7] + preds[8] + preds[9])/10<load_pretrained>
dtrain = lgbm.Dataset(x_train, label=y_value)
 clf = lgbm.train(lgbm_meta_params, train_set=dtrain, num_boost_round=5000)
 predict_stacking = clf.predict(x_test )
!cp '.. /input/aptosdataset/19model2.pth' 'models'
vision.data.open_image = _load_format
data =(ImageList.from_df(df=res1,path='./',cols='path')
.split_from_df() 
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(transforms,size=image_size,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='valid_loss', name='best_loss')],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('19model2')
 learn.fit_one_cycle(5,1e-3)
 learn.save('15model2')
 learn.unfreeze()<categorify>
submission_stacking = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_stacking['Cover_Type'] = predict_stacking
submission_stacking.to_csv('submission_stacking.csv', index=False )
data =(ImageList.from_df(df=res4,path='./',cols='path')
.split_from_df() 
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(transforms,size=456,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=16,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='valid_loss', name='15_final')],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('15_kappa_3')
 learn.fit_one_cycle(5,5e-5 )<define_search_space>
submission_et = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_et['Cover_Type'] = et_test
submission_et.to_csv('submission_et.csv', index=False )
preds = ['','','','','','','','','','']
i = 0
opt = OptimizedRounder()
for sz in [264,232,240,248,256,264,232,240,248,256]:
test =(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
data =(ImageList.from_df(df=valid_df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.add_test(test)
.transform(transforms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [quadratic_kappa],
model_dir="models" ).to_fp16()
learn.load('19_kappa');
preds[i],y = learn.get_preds(DatasetType.Test)
 i = i + 1
preds_B =(preds[0] + preds[1] + preds[2] + preds[3] + preds[4] + preds[5] + preds[6] + preds[7] + preds[8] + preds[9])/10<save_to_csv>
submission_rf = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_rf['Cover_Type'] = rf_test
submission_rf.to_csv('submission_rf.csv', index=False )