Kaggle competition: iWildCam 2020 - FGVC7
<SOS> metric: CategorizationAccuracy Kaggle data source: iwildcam-2020-fgvc7<train_model>
%matplotlib inline
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<set_options>
test_images = os.listdir(".. /input/iwildcam2020-256/256_images/test/images/")
 train_images = os.listdir(".. /input/iwildcam2020-256/256_images/train/images/" )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
with open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_train_annotations.json')as json_file:
train_data = json.load(json_file )
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
df_train = pd.DataFrame({'id': [item['id'] for item in train_data['annotations']],
'category_id': [item['category_id'] for item in train_data['annotations']],
'image_id': [item['image_id'] for item in train_data['annotations']],
'location': [item['location'] for item in train_data['images']],
'file_name': [item['file_name'] for item in train_data['images']]})
 df_train.head()
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<count_duplicates>
df_train = df_train[df_train['file_name'].isin(train_images)]
train[train.geo_region.isna() ].Country_Region.unique()
train = train.fillna('
test = test.fillna('
train[train.duplicated(['Date', 'Location'])]
train.count()<save_to_csv>
cat_images = dict()
cat_count = dict()
annotations = train_data['annotations']
_images = train_data['images']
for i, annotation in enumerate(annotations):
_img = annotation['image_id']
cat = annotation['category_id']
imgs = cat_images.get(cat, None)
 if imgs is None:
cat_images[cat] = [{'image_id': _img, 'category': cat}]
else:
cat_images[cat].append({'image_id': _img, 'category': cat})
 count = cat_count.get(cat, 0)
 if count == 0:
cat_count[cat] = 1
else:
cat_count[cat] += 1
n_train = dict()
n_val = dict()
for cat, count in cat_count.items() :
_train = math.floor(count * 0.70)
 if _train < 1:
_train = 1
_val = count - _train
n_train[cat] = _train
n_val[cat] = _val
train_images = []
val_images = []
for cat in cat_images.keys() :
random.shuffle(cat_images[cat])
 train_images += cat_images[cat][:n_train[cat]]
val_images += cat_images[cat][n_train[cat]:]
val_img_dt = pd.DataFrame(val_images)
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()<merge>
df_train['is_valid'] = np.where(df_train.image_id.isin(val_img_dt['image_id']), True, False )
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<feature_engineering>
df_train.groupby('is_valid' ).size()
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '<merge>
df_train.drop(df_train.loc[df_train['file_name']=='87022118-21bc-11ea-a13a-137349068a90.jpg'].index, inplace=True)
 df_train.drop(df_train.loc[df_train['file_name']=='8792549a-21bc-11ea-a13a-137349068a90.jpg'].index, inplace=True )
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<save_to_csv>
df_train.category_id.unique().shape
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )<save_to_csv>
with open(r'/kaggle/input/iwildcam-2020-fgvc7/iwildcam2020_test_information.json')as f:
test_data = json.load(f )
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<feature_engineering>
df_test = pd.DataFrame.from_records(test_data['images'])
 df_test.head()
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()<save_to_csv>
train, test = [ImageList.from_df(df, path='.. /input/iwildcam2020-256/256_images/', cols='file_name', folder=folder, suffix='')
 for df, folder in zip([df_train, df_test], ['train/images', 'test/images'])]
trfm = get_transforms(max_rotate=20, max_zoom=1.3, max_lighting=0.4, max_warp=0.4,
p_affine=1., p_lighting=1.)
src =(train.use_partial_data(1)
.split_from_df(col='is_valid')
.label_from_df(cols='category_id')
.add_test(test))
data =(src.transform(trfm, size = 128, padding_mode = 'reflection')
.databunch(path=Path('.'), bs = 256 ).normalize(imagenet_stats))
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
org_classes = pd.DataFrame({"org_category": data.classes})
 org_classes['Category'] = org_classes.index
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )<feature_engineering>
data.show_batch()
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()<save_to_csv>
learn = cnn_learner(data, base_arch=models.resnet50, metrics=accuracy ).mixup()
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
learn.fit_one_cycle(10, slice(0.01))
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
learn.fit_one_cycle(10, slice(1e-5, 1e-4))
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
preds,y = learn.TTA(ds_type=DatasetType.Test )
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
pred_csv = pd.DataFrame(preds.numpy())
 pred_csv['Id'] = learn.data.test_ds.items
pred_csv.to_csv("outout_preds.csv", index = False )
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<feature_engineering>
submission = pd.read_csv('.. /input/iwildcam-2020-fgvc7/sample_submission.csv')
 id_list = list(submission.Id)
 pred_list = list(np.argmax(preds.numpy() , axis=1))
pred_dict = dict(( key, value.item())for(key, value)in zip(learn.data.test_ds.items,pred_list))
pred_ordered = [pred_dict['.. /input/iwildcam2020-256/256_images/test/images/' + id + '.jpg'] for id in id_list]
submission_with_idx = pd.DataFrame({'Id':id_list,'Category':pred_ordered})
 submission_fixed_labels = pd.merge(submission_with_idx, org_classes, on = 'Category', how='left')
 submission_fixed_labels = submission_fixed_labels.drop(['Category'], axis = 1)
 submission_fixed_labels.rename(columns={'org_category': 'Category'}, inplace=True)
 submission_fixed_labels.to_csv("submission.csv".format(Category),index = False)
 print("Done" )