Kaggle competition: Cdiscount’s Image Classification Challenge
<SOS> metric: CategorizationAccuracy Kaggle data source: cdiscount-image-classification-challenge<load_from_csv>
import numpy as np
import pandas as pd
import io
import bson
import matplotlib.pyplot as plt
from skimage.data import imread
from tqdm import tqdm_notebook
train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 tt = pd.concat([train, test], sort=False)
 tt = train.merge(test, on=['Province_State','Country_Region','Date'], how='outer')
 def name_place(x):
try:
x_new = x['Country_Region'] + "_" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
tt['Place'] = tt.apply(lambda x: name_place(x), axis=1)
 tt['Date'] = pd.to_datetime(tt['Date'])
 tt['doy'] = tt['Date'].dt.dayofyear
tt['dow'] = tt['Date'].dt.dayofweek
tt['hasProvidence'] = ~tt['Province_State'].isna()
country_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 tt = tt.merge(country_meta, how='left')
 country_date_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_date_metadata.csv')
 tt['HasFatality'] = tt.groupby('Place')['Fatalities'].transform(lambda x: x.max() > 0)
 tt['HasCases'] = tt.groupby('Place')['ConfirmedCases'].transform(lambda x: x.max() > 0)
 first_case_date = tt.query('ConfirmedCases >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_case_date = tt.query('ConfirmedCases >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_case_date = tt.query('ConfirmedCases >= 100' ).groupby('Place')['Date'].min().to_dict()
first_fatal_date = tt.query('Fatalities >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_fatal_date = tt.query('Fatalities >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_fatal_date = tt.query('Fatalities >= 100' ).groupby('Place')['Date'].min().to_dict()
tt['First_Case_Date'] = tt['Place'].map(first_case_date)
 tt['Ten_Case_Date'] = tt['Place'].map(ten_case_date)
 tt['Hundred_Case_Date'] = tt['Place'].map(hundred_case_date)
 tt['First_Fatal_Date'] = tt['Place'].map(first_fatal_date)
 tt['Ten_Fatal_Date'] = tt['Place'].map(ten_fatal_date)
 tt['Hundred_Fatal_Date'] = tt['Place'].map(hundred_fatal_date)
 tt['Days_Since_First_Case'] =(tt['Date'] - tt['First_Case_Date'] ).dt.days
tt['Days_Since_Ten_Cases'] =(tt['Date'] - tt['Ten_Case_Date'] ).dt.days
tt['Days_Since_Hundred_Cases'] =(tt['Date'] - tt['Hundred_Case_Date'] ).dt.days
tt['Days_Since_First_Fatal'] =(tt['Date'] - tt['First_Fatal_Date'] ).dt.days
tt['Days_Since_Ten_Fatal'] =(tt['Date'] - tt['Ten_Fatal_Date'] ).dt.days
tt['Days_Since_Hundred_Fatal'] =(tt['Date'] - tt['Hundred_Fatal_Date'] ).dt.days
smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 smoking = smoking.rename(columns={'Smoking prevalence, total(ages 15+ )(% of adults)': 'Smoking_Rate'})
 smoking_dict = smoking.groupby('Entity')['Year'].max().to_dict()
smoking['LastYear'] = smoking['Entity'].map(smoking_dict)
 smoking = smoking.query('Year == LastYear' ).reset_index()
smoking['Entity'] = smoking['Entity'].str.replace('United States', 'US')
 tt = tt.merge(smoking[['Entity','Smoking_Rate']],
left_on='Country_Region',
right_on='Entity',
how='left',
validate='m:1')\ 
.drop('Entity', axis=1)
 country_info = pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 tt = tt.merge(country_info, left_on=['Country_Region','Province_State'],
right_on=['country','region'],
how='left',
validate='m:1')
 us_state_info = pd.read_html('https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population')[0] \
[['State','Population estimate, July 1, 2019[2]']] \ 
.rename(columns={'Population estimate, July 1, 2019[2]' : 'Population'})
 tt = tt.merge(us_state_info[['State','Population']],
left_on='Province_State',
right_on='State',
how='left')
 tt['pop'] = pd.to_numeric(tt['pop'].str.replace(',',''))
tt['pop'] = tt['pop'].fillna(tt['Population'])
 tt['pop'] = pd.to_numeric(tt['pop'])
 tt['pop_diff'] = tt['pop'] - tt['Population']
tt['Population_final'] = tt['Population']
tt.loc[~tt['hasProvidence'], 'Population_final'] = tt.loc[~tt['hasProvidence']]['pop']
tt['Confirmed_Cases_Diff'] = tt.groupby('Place')['ConfirmedCases'].diff()
tt['Fatailities_Diff'] = tt.groupby('Place')['Fatalities'].diff()
max_date = tt.dropna(subset=['ConfirmedCases'])['Date'].max()
tt['gdp2019'] = pd.to_numeric(tt['gdp2019'].str.replace(',',''))<define_variables>
categories = pd.read_csv('.. /input/category_names.csv', index_col='category_id' )
pop_dict = {'Angola': int(29.78 * 10**6),
'Australia_Australian Capital Territory': 423_800,
'Australia_New South Wales': int(7.544 * 10**6),
'Australia_Northern Territory': 244_300,
'Australia_Queensland' : int(5.071 * 10**6),
'Australia_South Australia' : int(1.677 * 10**6),
'Australia_Tasmania': 515_000,
'Australia_Victoria': int(6.359 * 10**6),
'Australia_Western Australia': int(2.589 * 10**6),
'Brazil': int(209.3 * 10**6),
'Canada_Alberta' : int(4.371 * 10**6),
'Canada_British Columbia' : int(5.071 * 10**6),
'Canada_Manitoba' : int(1.369 * 10**6),
'Canada_New Brunswick' : 776_827,
'Canada_Newfoundland and Labrador' : 521_542,
'Canada_Nova Scotia' : 971_395,
'Canada_Ontario' : int(14.57 * 10**6),
'Canada_Prince Edward Island' : 156_947,
'Canada_Quebec' : int(8.485 * 10**6),
'Canada_Saskatchewan': int(1.174 * 10**6),
'China_Anhui': int(62 * 10**6),
'China_Beijing': int(21.54 * 10**6),
'China_Chongqing': int(30.48 * 10**6),
'China_Fujian' : int(38.56 * 10**6),
'China_Gansu' : int(25.58 * 10**6),
'China_Guangdong' : int(113.46 * 10**6),
'China_Guangxi' : int(48.38 * 10**6),
'China_Guizhou' : int(34.75 * 10**6),
'China_Hainan' : int(9.258 * 10**6),
'China_Hebei' : int(74.7 * 10**6),
'China_Heilongjiang' : int(38.31 * 10**6),
'China_Henan' : int(94 * 10**6),
'China_Hong Kong' : int(7.392 * 10**6),
'China_Hubei' : int(58.5 * 10**6),
'China_Hunan' : int(67.37 * 10**6),
'China_Inner Mongolia' : int(24.71 * 10**6),
'China_Jiangsu' : int(80.4 * 10**6),
'China_Jiangxi' : int(45.2 * 10**6),
'China_Jilin' : int(27.3 * 10**6),
'China_Liaoning' : int(43.9 * 10**6),
'China_Macau' : 622_567,
'China_Ningxia' : int(6.301 * 10**6),
'China_Qinghai' : int(5.627 * 10**6),
'China_Shaanxi' : int(37.33 * 10**6),
'China_Shandong' : int(92.48 * 10**6),
'China_Shanghai' : int(24.28 * 10**6),
'China_Shanxi' : int(36.5 * 10**6),
'China_Sichuan' : int(81.1 * 10**6),
'China_Tianjin' : int(15 * 10**6),
'China_Tibet' : int(3.18 * 10**6),
'China_Xinjiang' : int(21.81 * 10**6),
'China_Yunnan' : int(45.97 * 10**6),
'China_Zhejiang' : int(57.37 * 10**6),
'Denmark_Faroe Islands' : 51_783,
'Denmark_Greenland' : 56_171,
'France_French Guiana' : 290_691,
'France_French Polynesia' : 283_007,
'France_Guadeloupe' : 395_700,
'France_Martinique' : 376_480,
'France_Mayotte' : 270_372,
'France_New Caledonia' : 99_926,
'France_Reunion' : 859_959,
'France_Saint Barthelemy' : 9_131,
'France_St Martin' : 32_125,
'Netherlands_Aruba' : 105_264,
'Netherlands_Curacao' : 161_014,
'Netherlands_Sint Maarten' : 41_109,
'Papua New Guinea' : int(8.251 * 10**6),
'US_Guam' : 164_229,
'US_Virgin Islands' : 107_268,
'United Kingdom_Bermuda' : 65_441,
'United Kingdom_Cayman Islands' : 61_559,
'United Kingdom_Channel Islands' : 170_499,
'United Kingdom_Gibraltar' : 34_571,
'United Kingdom_Isle of Man' : 84_287,
'United Kingdom_Montserrat' : 4_922
}
tt['Population_final'] = tt['Population_final'].fillna(tt['Place'].map(pop_dict))<feature_engineering>
df_dict = {
'category': prod_category,
'num_imgs': prod_num_imgs
}
df = pd.DataFrame(df_dict, index=prod_id)
 del df_dict
tt.loc[tt['Place'] == 'Diamond Princess', 'Population final'] = 2_670<feature_engineering>
cat_counts = df.category.value_counts().to_frame()
cat_counts = cat_counts / cat_counts["category"].sum()
print(cat_counts.head())
 print(pd.unique(df.category))
tt['ConfirmedCases_Log'] = tt['ConfirmedCases'].apply(np.log1p)
 tt['Fatalities_Log'] = tt['Fatalities'].apply(np.log1p )<feature_engineering>
cat_counts.sort_values(by="category",inplace=True)
 bot_5_categories = cat_counts.head()
top_5_categories = cat_counts.tail()
print(bot_5_categories)
 print(top_5_categories )
<remove_duplicates><EOS> 
samp_sub_df = pd.read_csv(".. /input/sample_submission.csv")
 samp_sub_df["category_id"] = np.random.choice(cat_counts.index,size=len(samp_sub_df),p=cat_counts["category"].values)
 samp_sub_df.to_csv("baised_rand_submission.csv", index=False )