Kaggle competition: Humpback Whale Identification Challenge
<SOS> metric: MAP@{K} Kaggle data source: whale-categorization-playground<train_on_grid>
trainData = pd.read_csv(".. /input/whale-categorization-playground/train.csv")
 trainData.sample(5)
model = XGBRegressor()
model1 = gridSearchCV(model, x, y1, param_grid, 10, 'neg_mean_squared_error')
 model2 = gridSearchCV(model, x, y2, param_grid, 10, 'neg_mean_squared_error' )<predict_on_test>
Image(filename=".. /input/whale-categorization-playground/train/"+random.choice(trainData['Image']))
y1_pred = model1.predict(x_test)
 y1_pred = y1_pred.round()<predict_on_test>
def prepareImages(data, m, dataset):
print("Preparing images")
 X_train = np.zeros(( m, 100, 100, 3))
count = 0
for fig in data['Image']:
img = image.load_img(".. /input/whale-categorization-playground/"+dataset+"/"+fig, target_size=(100, 100, 3))
x = image.img_to_array(img)
 x = preprocess_input(x)
 X_train[count] = x
if(count%500 == 0):
print("Processing image: ", count+1, ", ", fig)
 count += 1
count = 0
print("Finished!")
 return X_train
y2_pred = model2.predict(x_test)
 y2_pred = y2_pred.round()<load_from_csv>
def prepareY(Y):
values = array(Y)
 print(values.shape)
 label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
 print(integer_encoded)
 onehot_encoder = OneHotEncoder(sparse=False)
 integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
 onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
 print(onehot_encoded)
 y = onehot_encoded
print(y.shape)
 return y, label_encoder
Sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Sample_submission.columns
submission_xg = Sample_submission[["ForecastId"]]<prepare_output>
mod = Sequential()
mod.add(Conv2D(32,(7, 7), strides =(1, 1), name = 'conv0', input_shape =(100, 100, 3)))
 mod.add(BatchNormalization(axis = 3, name = 'bn0'))
mod.add(Activation('relu'))
mod.add(MaxPooling2D(( 2, 2), name='max_pool'))
mod.add(Conv2D(64,(3, 3), strides =(1,1), name="conv1"))
mod.add(Activation('relu'))
mod.add(AveragePooling2D(( 3, 3), name='avg_pool'))
mod.add(Flatten())
 mod.add(Dense(500, activation="relu", name='rl'))
mod.add(Dropout(0.8))
mod.add(Dense(4251, activation='softmax', name='sm'))
print(mod.output_shape)
 mod.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['accuracy'] )
cases=pd.DataFrame(y1_pred)
 cases.columns=["ConfirmedCases"]
fatal=pd.DataFrame(y2_pred)
 fatal.columns=["Fatalities"]
Final_submission = pd.concat([cases,fatal,submission_xg],axis=1)
 Final_submission.head(20 )<import_modules>
X = prepareImages(trainData, 9850, "train")
 X /= 255
print("Shape X-train: ", X.shape )
print("Read in libraries")
<load_from_csv>
Y = trainData['Id']
print("Shape Y-train: ", Y.shape)
 y, label_encoder = prepareY(Y )
print("read in train file")
 df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv",
usecols=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities'])
<drop_column>
history = mod.fit(X, y, epochs=100, batch_size=100, verbose=1)
 gc.collect()
print("fill blanks and add region for counting")
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province_State']+df['Country_Region']
df.drop('Province_State',axis=1,inplace=True)
 df.drop('Country_Region',axis=1,inplace=True)
<load_from_csv>
countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )<choose_model_class>
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
<data_type_conversions>
df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1)
 df_submit=df_submit.fillna(method='pad' ).astype(int )<merge>
df_submit.rename(columns={0: 'ForecastId', 1: 'ConfirmedCases',2: 'Fatalities',}, inplace=True)
 complete_test= pd.merge(test, df_submit, how="left", on="ForecastId" )<save_to_csv>
test = os.listdir(".. /input/whale-categorization-playground/test/")
 print(len(test))
col = ['Image']
testData1 = pd.DataFrame(test[0:3899], columns=col)
 testData2 = pd.DataFrame(test[3900:7799], columns=col)
 testData3 = pd.DataFrame(test[7800:11699], columns=col)
 testData4 = pd.DataFrame(test[11700:15609], columns=col)
 testData = pd.DataFrame(test, columns=col )
df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 complete_test.to_csv('complete_test.csv',index=False)
<load_from_csv>
gc.collect()
X = prepareImages(testData1, 3900, "test")
 X /= 255
path = '/kaggle/input/covid19-global-forecasting-week-2/'
pd.set_option('display.max_rows', None)
 train_df = pd.read_csv(path+'train.csv')
 test_df = pd.read_csv(path+'test.csv')
 submission_df = pd.read_csv(path+'submission.csv' )<data_type_conversions>
predictions1 = mod.predict(np.array(X), verbose=1)
 gc.collect()
test_df['Date']= pd.to_datetime(test_df['Date'])
 train_df['Date']= pd.to_datetime(train_df['Date'] )<create_dataframe>
X = prepareImages(testData2, 3900, "test")
 X /= 255
predictions2 = mod.predict(np.array(X), verbose=1)
 gc.collect()
null_vals = pd.DataFrame(round(100 *(train_df.isnull().sum() / len(train_df.index)) , 3))
null_vals.columns = ['% null values']
null_vals<create_dataframe>
X = prepareImages(testData3, 3900, "test")
 X /= 255
predictions3 = mod.predict(np.array(X), verbose=1)
 gc.collect()
null_vals = pd.DataFrame(round(100 *(test_df.isnull().sum() / len(test_df.index)) , 3))
null_vals.columns = ['% null values']
null_vals<rename_columns>
X = prepareImages(testData4, 3910, "test")
 X /= 255
predictions4 = mod.predict(np.array(X), verbose=1)
 gc.collect()
train_df['Province_State'].fillna('NO_STATE', inplace=True)
 test_df['Province_State'].fillna('NO_STATE', inplace=True )<groupby>
predictions = np.concatenate(( predictions1, predictions2), axis=0)
 predictions = np.concatenate(( predictions, predictions3), axis=0)
 predictions = np.concatenate(( predictions, predictions4), axis=0)
 gc.collect()
print(predictions.shape)
 print(predictions )
train_grouped_df = train_df.groupby(['Country_Region', 'Province_State'] )<define_variables>
print(predictions.shape)
 copy_pred = np.copy(predictions)
 idx = np.argmax(copy_pred, axis=1)
 copy_pred[:,idx] = 0
idx2 = np.argmax(copy_pred, axis=1)
 copy_pred[:, idx2] = 0
idx3 = np.argmax(copy_pred, axis=1)
 copy_pred[:, idx3] = 0
idx4 = np.argmax(copy_pred, axis=1)
 copy_pred[:, idx4] = 0
idx5 = np.argmax(copy_pred, axis=1 )
cluster0 = ['Botswana', 'Fiji', 'Gabon', 'Libya', 'Seychelles', 'Suriname', 'Mongolia', 'El Salvador', 'Barbados', 'Jamaica', 'Paraguay', 'Trinidad and Tobago','Montenegro', 'Georgia', 'Mauritius', 'Venezuela', 'Belarus', 'Malta', 'Oman', 'Albania', 'Cyprus', 'Kuwait', 'Jordan', 'Azerbaijan','Kazakhstan', 'Uruguay', 'Costa Rica', 'Bulgaria', 'Tunisia', 'Bosnia and Herzegovina', 'Latvia', 'Lebanon', 'Hungary', 'Armenia','Lithuania', 'Bahrain', 'Ukraine', 'Algeria', 'United Arab Emirates', 'Qatar', 'Estonia', 'Slovenia', 'Serbia', 'Colombia', 'Argentina','Dominican Republic', 'Peru', 'Mexico', 'Panama', 'Greece', 'South Africa', 'Indonesia', 'Saudi Arabia', 'Thailand', 'Ecuador', 'Poland','Romania', 'Malaysia', 'Brazil', 'Israel', 'Portugal', 'Turkey']
cluster0_confirmed_cap = 1500
cluster0_fatality_cap = 750
cluster1 = ['Italy', 'Spain']
cluster1_confirmed_cap = 50
cluster1_fatality_cap = 25
cluster2 = ['Iceland', 'Luxembourg', 'Switzerland']
cluster2_confirmed_cap = 500
cluster2_fatality_cap = 250
cluster3 = ['Angola', 'Bangladesh', 'Belize', 'Benin', 'Bhutan', 'Burkina Faso', 'Cambodia', 'Cameroon', 'Chad', 'Egypt', 'Ethiopia', 'Ghana', 'Guatemala', 'Guinea', 'Guyana', 'Haiti', 'Honduras', 'Iraq', 'Kenya', 'Liberia', 'Madagascar', 'Mali', 'Mauritania', 'Morocco', 'Mozambique', 'Namibia', 'Nepal','Nicaragua', 'Niger', 'Nigeria', 'Pakistan', 'Papua New Guinea', 'Philippines', 'Rwanda', 'Senegal', 'Sri Lanka', 'Sudan', 'Togo', 'Uganda','Zambia', 'Zimbabwe']
cluster3_confirmed_cap = 300
cluster3_fatality_cap = 150
cluster4 = ['China']
cluster4_confirmed_cap = 1.5
cluster4_fatality_cap = 1.35
cluster5 = ['Hong Kong', 'Singapore']
cluster5_confirmed_cap = 40
cluster5_fatality_cap = 28
cluster6 = ['United States']
cluster6_confirmed_cap = 200
cluster6_fatality_cap = 140
cluster6 = ['United States']
cluster6_confirmed_cap = 500
cluster6_fatality_cap = 350
cluster7 = ['Australia', 'Austria', 'Belgium', 'Canada', 'Chile', 'Denmark', 'Finland', 'Ireland', 'Japan', 'Netherlands', 'New Zealand', 'Norway', 'South Korea','Sweden', 'United Kingdom']
cluster7_confirmed_cap = 100
cluster7_fatality_cap = 70
cluster8 = ['France', 'Germany', 'Iran']
cluster8_confirmed_cap = 50
cluster8_fatality_cap = 35
cluster9 = ['India']
cluster9_confirmed_cap = 30
cluster9_fatality_cap = 21 
<prepare_x_and_y>
results = []
print(idx[0:10])
 print(idx2[0:10])
 print(idx3[0:10])
 print(idx4[0:10])
 print(idx5[0:10])
 threshold = 0.05
for i in range(0, predictions.shape[0]):
each = np.zeros(( 4251, 1))
each2 = np.zeros(( 4251, 1))
each3 = np.zeros(( 4251, 1))
each4 = np.zeros(( 4251, 1))
each5 = np.zeros(( 4251, 1))
if(( predictions[i, idx5[i]] > threshold)) :
each5[idx5[i]] = 1
each4[idx4[i]] = 1
each3[idx3[i]] = 1
each2[idx2[i]] = 1
each[idx[i]] = 1
tags = [label_encoder.inverse_transform([argmax(each)])[0], label_encoder.inverse_transform([argmax(each2)])[0], label_encoder.inverse_transform([argmax(each3)])[0], label_encoder.inverse_transform([argmax(each4)])[0], label_encoder.inverse_transform([argmax(each5)])[0]]
else:
if(( predictions[i, idx4[i]] > threshold)) :
print(predictions[i, idx4[i]])
 each4[idx4[i]] = 1
each3[idx3[i]] = 1
each2[idx2[i]] = 1
each[idx[i]] = 1
tags = [label_encoder.inverse_transform([argmax(each)])[0], label_encoder.inverse_transform([argmax(each2)])[0], label_encoder.inverse_transform([argmax(each3)])[0], label_encoder.inverse_transform([argmax(each4)])[0]]
else:
if(( predictions[i, idx3[i]] > threshold)) :
each3[idx3[i]] = 1
each2[idx2[i]] = 1
each[idx[i]] = 1
tags = [label_encoder.inverse_transform([argmax(each)])[0], label_encoder.inverse_transform([argmax(each2)])[0], label_encoder.inverse_transform([argmax(each3)])[0]]
else:
if(( predictions[i, idx2[i]] > threshold)) :
each2[idx2[i]] = 1
each[idx[i]] = 1
tags = [label_encoder.inverse_transform([argmax(each)])[0], label_encoder.inverse_transform([argmax(each2)])[0]]
else:
each[idx[i]] = 1
tags = label_encoder.inverse_transform([argmax(each)])[0]
results.append(tags )
<data_type_conversions><EOS> 
myfile = open('output.csv','w')
 column= ['Image', 'Id']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 for i in range(0, testData.shape[0]):
pred = ""
if(len(results[i])==5):
if(results[i][4]!=results[i][0]):
pred = results[i][0] + " " + results[i][1] + " " + results[i][2] + " " + results[i][3] + " " + results[i][4]
else:
pred = results[i][0] + " " + results[i][1] + " " + results[i][2] + " " + results[i][3]
else:
if(len(results[i])==4):
pred = results[i][0] + " " + results[i][1] + " " + results[i][2] + " " + results[i][3]
else:
if(len(results[i])==3):
pred = results[i][0] + " " + results[i][1] + " " + results[i][2]
else:
if(len(results[i])==2):
pred = results[i][0] + " " + results[i][1]
else:
pred = results[i]
result = [testData['Image'][i], pred]
wrtr.writerow(result)
 myfile.close()