Kaggle competition: iWildCam 2020 - FGVC7
<SOS> metric: CategorizationAccuracy Kaggle data source: iwildcam-2020-fgvc7<feature_engineering>
seed(10)
 %matplotlib inline
df_train['duration'] = df_train['issue_d_year'] - df_train['earliest_cr_line_year']
df_test['duration'] = df_test['issue_d_year'] - df_test['earliest_cr_line_year']<feature_engineering>
!pip install git+https://github.com/qubvel/efficientnet -q
df_train.loc[df_train['duration'] > 100, 'duration'] = -9999
df_test.loc[df_test['duration'] > 100, 'duration'] = -9999<create_dataframe>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('tpu:',tpu)
 except:
print('no tpu.....')
 strategy=None
tpu=None
if tpu:
GCS_DS_PATH = KaggleDatasets().get_gcs_path('iwildcam-2020-fgvc7')
 tf_records_path= KaggleDatasets().get_gcs_path('iwildcam2020-64-tf-records')
 tf_records_path2=KaggleDatasets().get_gcs_path('iwildcam2020-64-tf-records')
 test_tf_records_path=KaggleDatasets().get_gcs_path('iwildcam2020-64-tf-records')
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
EPOCHS = 8
img_size = 64
df_train_emp_len = df_train['emp_length'].fillna(-9999)
 df_test_emp_len = df_test['emp_length'].fillna(-9999)
 df_train_emp_len_next = pd.DataFrame(df_train_emp_len)
 df_test_emp_len_next = pd.DataFrame(df_test_emp_len)
 df_train_emp_len_next = df_train_emp_len_next.rename(columns = {'emp_length':'emp_length_num'})
 df_test_emp_len_next = df_test_emp_len_next.rename(columns = {'emp_length':'emp_length_num'})
 df_train_emp_len_next['emp_length_num'] = df_train_emp_len_next['emp_length_num'].replace({'< 1 year':0,'1 year':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5,'6 years':6,'7 years': 7,'8 years': 8,'9 years': 9,'10+ years': 10})
 df_test_emp_len_next['emp_length_num'] = df_test_emp_len_next['emp_length_num'].replace({'< 1 year':0,'1 year':1,'2 years':2,'3 years':3,'4 years':4,'5 years':5,'6 years':6,'7 years': 7,'8 years': 8,'9 years': 9,'10+ years': 10})
 df_train_emp_len_next['emp_length_num'] = df_train_emp_len_next['emp_length_num'].astype(int)
 df_test_emp_len_next['emp_length_num'] = df_test_emp_len_next['emp_length_num'].astype(int)
 df_train = pd.merge(df_train_emp_len_next, df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test_emp_len_next, df_test,left_index=True, right_index=True)
<data_type_conversions>
TRAINING_FILENAMES = tf.io.gfile.glob(tf_records_path + '/*train.rec')
 TRAINING_FILENAMES.extend(tf.io.gfile.glob(tf_records_path2 + '/*zero.rec'))
TRAINING_FILENAMES
df_train = pd.merge(df_train['zip_code'].str[:3], df_train,left_index=True, right_index=True)
 df_train['zip_code_x'] = df_train['zip_code_x'].astype(int)
 df_test = pd.merge(df_test['zip_code'].str[:3], df_test,left_index=True, right_index=True)
 df_test['zip_code_x'] = df_test['zip_code_x'].astype(int)
<data_type_conversions>
TEST_FILENAMES=tf.io.gfile.glob(test_tf_records_path + '/*test.rec' )
df_train = pd.merge(df_train['zip_code_y'].str[:1], df_train,left_index=True, right_index=True)
 df_test = pd.merge(df_test['zip_code_y'].str[:1], df_test,left_index=True, right_index=True)
<load_from_csv>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [img_size,img_size, 3])
 return image
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
gdp_df = pd.read_csv('.. /input/homework-for-students4plus/US_GDP_by_State.csv', index_col=0)
 desc_df = pd.read_csv('.. /input/homework-for-students4plus/description.csv', index_col=0)
 zip_df = pd.read_csv('.. /input/homework-for-students4plus/free-zipcode-database.csv', index_col=0)
 spi_df = pd.read_csv('.. /input/homework-for-students4plus/spi.csv', index_col=0)
 state_df = pd.read_csv('.. /input/homework-for-students4plus/statelatlong.csv', index_col=0)
<load_from_csv>
train_dataset=get_training_dataset()
X_test = df_test
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis=1)
 gc.collect()<define_variables>
sub_df = pd.read_csv('.. /input/iwildcam-2020-fgvc7/sample_submission.csv')
 sub_df.head()
cat = []
num = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cat.append(col)
 else:
num.append(col)
<feature_engineering>
test_dataset=get_test_dataset()
X_train['loan_amnt'] = X_train['loan_amnt'].apply(np.log1p)
 X_test['loan_amnt'] = X_test['loan_amnt'].apply(np.log1p)
<split>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(img_size, img_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(216, activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary()
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 for col in cat:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train 
<train_model>
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
 best_checkpoint='model.h5'
checkpoint = ModelCheckpoint(
 best_checkpoint,
monitor='val_accuracy',
verbose=1,
save_weights_only=True,
mode='auto'
)
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )<import_modules>
%%time
STEPS_PER_EPOCH=(143736+3709)// BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
verbose=2,
callbacks=[early,checkpoint],
steps_per_epoch=STEPS_PER_EPOCH,
)
import random<choose_model_class>
gc.collect()
clf_arry = []
skf_arry = []
for i in range(0,25):
rand_int =random.randint(1, 10000)
 clf_random = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.9,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.005, min_split_gain=0.0,
n_estimators=99999, n_jobs=-1, num_leaves=15, objective=None,
random_state= rand_int, reg_alpha=0.0, reg_lambda=0.0, silent=True,
subsample=1.0, subsample_for_bin=200000, subsample_freq=0)
 skf_random = StratifiedKFold(n_splits=5, random_state= rand_int, shuffle=True)
 print(clf_random)
 print(skf_random)
 clf_arry.append(clf_random)
 skf_arry.append(skf_random)
<split>
sam_sub_df=sub_df.copy()
sam_sub_df["file_name"] = sam_sub_df["Id"].map(lambda str : str + ".jpg" )
scores = []
y_pred_test = np.zeros(len(X_test))
for y in range(0,len(clf_arry)) :
clf = clf_arry[y]
skf = skf_arry[y]
print(clf)
 print(skf)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
print('⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️')
 print(y)
 print(i)
 print('⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️⭐️')
 X_train_, y_train_ = X_train.iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train.iloc[test_ix], y_train.iloc[test_ix]
clf.fit(X_train_, y_train_, early_stopping_rounds=200, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(y_val, y_pred))
y_pred_test += clf.predict_proba(X_test)[:,1] 
<compute_test_metric>
%%time
model.load_weights(best_checkpoint)
 predict=model.predict(test_dataset, verbose=1 ).astype(float )
print(len(clf_arry))
y_pred = y_pred_test /(len(clf_arry)* 5)
 print('----------------------')
 print('各スコア：')
 x = 0
for i in scores:
x += i
print(i)
 print('----------------------')
 print(x)
 print('Ave.CV score：')
 print(x/(len(clf_arry)* 5))<create_dataframe>
predicted_class_indices=np.argmax(predict,axis=1 )
imp = DataFrame(clf.booster_.feature_importance(importance_type='gain'), index = X_train.columns, columns=['importance'] ).sort_values(['importance'], ascending=False)
 imp<save_to_csv>
with open('.. /input/iwildcam2020-classes-dict/cid_invert_dict.pkl', mode='rb')as fin:
cid_invert_dict=pickle.load(fin )
submission = pd.read_csv('/kaggle/input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )<set_options>
def transform(x):
return cid_invert_dict[str(x)]
<load_from_csv><EOS> 
sam_sub_df["Category"] = predicted_class_indices
sam_sub_df["Category"]=sam_sub_df["Category"].apply(transform)
 sam_sub_df = sam_sub_df.loc[:,["Id", "Category"]]
sam_sub_df.to_csv("submission.csv",index=False)
 sam_sub_df.head()