Kaggle competition: iWildCam 2019 - FGVC6
<SOS> metric: MacroFScore Kaggle data source: iwildcam-2019-fgvc6<categorify>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f)
<feature_engineering>
from fastai import *
from fastai.vision import *
import pandas as pd
from fastai.utils.mem import *
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
<prepare_x_and_y>
path = Path('/kaggle/input/iwildcam-2019-fgvc6')
 debug =1
if debug:
train_pct=0.04
else:
train_pct=0.5
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape)
<compute_train_metric>
train_df = pd.read_csv(path/'train.csv')
 train_df = pd.concat([train_df['id'],train_df['category_id']],axis=1,keys=['id','category_id'])
 train_df.head()
cv_scores = []
kf = model_selection.KFold(n_splits=7, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break 
<save_to_csv>
test_df = pd.read_csv(path/'test.csv')
 test_df = pd.DataFrame(test_df['id'])
 test_df['predicted'] = 0
test_df.head()
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False)
<set_options>
free = gpu_mem_get_free_no_cache()
if free > 8200: bs=64
else: bs=32
print(f"using bs={bs}, have {free}MB of GPU RAM free")
 tfms = get_transforms(max_rotate=20, max_zoom=1.3, max_lighting=0.4, max_warp=0.4,
p_affine=1., p_lighting=1.)
%matplotlib inline 
<define_variables>
data = get_data(224, bs, 'zeros' )
horizontal_flip = False
rotation_augmentation = True
brightness_augmentation = True
shift_augmentation = True
random_noise_augmentation = True
include_unclean_data = True
sample_image_index = 20
rotation_angles = [12]
pixel_shifts = [12]
NUM_EPOCHS = 80
BATCH_SIZE = 64<load_pretrained>
gc.collect()
wd=1e-1
learn = cnn_learner(data, models.resnet34, metrics=error_rate, bn_final=True, wd=wd)
 learn.model_dir= '/kaggle/working/'
print("Contents of input/facial-keypoints-detection directory: ")
 !ls.. /input/facial-keypoints-detection/
print("
Extracting.zip dataset files to working directory...")
 !unzip -u.. /input/facial-keypoints-detection/test.zip
!unzip -u.. /input/facial-keypoints-detection/training.zip
print("
Current working directory:")
 !pwd
print("
Contents of working directory:")
 !ls<load_from_csv>
data = get_data(352,bs)
 learn.data = data
%%time
train_file = 'training.csv'
test_file = 'test.csv'
idlookup_file = '.. /input/facial-keypoints-detection/IdLookupTable.csv'
train_data = pd.read_csv(train_file)
 test_data = pd.read_csv(test_file)
 idlookup_data = pd.read_csv(idlookup_file )<count_values>
learn.fit_one_cycle(2, max_lr=slice(1e-6,1e-4))
learn.save('352' )
print("Length of train data: {}".format(len(train_data)))
 print("Number of Images with missing pixel values: {}".format(len(train_data)- int(train_data.Image.apply(lambda x: len(x.split())).value_counts().values)) )<count_missing_values>
lr = 1e-3
learn.fit_one_cycle(8, slice(lr/100, lr))
train_data.isnull().sum()<categorify>
learn.save('stage-2-sz32' )
%%time
def load_images(image_data):
images = []
for idx, sample in image_data.iterrows() :
image = np.array(sample['Image'].split(' '), dtype=int)
 image = np.reshape(image,(96,96,1))
images.append(image)
 images = np.array(images)/255.
 return images
def load_keypoints(keypoint_data):
keypoint_data = keypoint_data.drop('Image',axis = 1)
 keypoint_features = []
for idx, sample_keypoints in keypoint_data.iterrows() :
keypoint_features.append(sample_keypoints)
 keypoint_features = np.array(keypoint_features, dtype = 'float')
 return keypoint_features
clean_train_images = load_images(clean_train_data)
 print("Shape of clean_train_images: {}".format(np.shape(clean_train_images)))
 clean_train_keypoints = load_keypoints(clean_train_data)
 print("Shape of clean_train_keypoints: {}".format(np.shape(clean_train_keypoints)))
 test_images = load_images(test_data)
 print("Shape of test_images: {}".format(np.shape(test_images)))
 train_images = clean_train_images
train_keypoints = clean_train_keypoints
fig, axis = plt.subplots()
plot_sample(clean_train_images[sample_image_index], clean_train_keypoints[sample_image_index], axis, "Sample image & keypoints")
 if include_unclean_data:
unclean_train_images = load_images(unclean_train_data)
 print("Shape of unclean_train_images: {}".format(np.shape(unclean_train_images)))
 unclean_train_keypoints = load_keypoints(unclean_train_data)
 print("Shape of unclean_train_keypoints: {}
".format(np.shape(unclean_train_keypoints)))
 train_images = np.concatenate(( train_images, unclean_train_images))
train_keypoints = np.concatenate(( train_keypoints, unclean_train_keypoints))<concatenate>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
def left_right_flip(images, keypoints):
flipped_keypoints = []
flipped_images = np.flip(images, axis=2)
 for idx, sample_keypoints in enumerate(keypoints):
flipped_keypoints.append([96.-coor if idx%2==0 else coor for idx,coor in enumerate(sample_keypoints)])
 return flipped_images, flipped_keypoints
if horizontal_flip:
flipped_train_images, flipped_train_keypoints = left_right_flip(clean_train_images, clean_train_keypoints)
 print("Shape of flipped_train_images: {}".format(np.shape(flipped_train_images)))
 print("Shape of flipped_train_keypoints: {}".format(np.shape(flipped_train_keypoints)))
 train_images = np.concatenate(( train_images, flipped_train_images))
train_keypoints = np.concatenate(( train_keypoints, flipped_train_keypoints))
fig, axis = plt.subplots()
plot_sample(flipped_train_images[sample_image_index], flipped_train_keypoints[sample_image_index], axis, "Horizontally Flipped" )<normalization>
test_preds = learn.get_preds(DatasetType.Test)
 test_df['predicted'] = test_preds[0].argmax(dim=1 )
<categorify><EOS> 
csv_path ='/kaggle/working/submission.csv'
test_df.to_csv(csv_path, index=False )