Kaggle competition: Open Images 2019 - Instance Segmentation
<SOS> metric: OpenImagesObjDetectionSegmentationAP Kaggle data source: open-images-2019-instance-segmentation<load_from_csv>
import numpy as np
import pandas as pd
import os
import sys
from tqdm import tqdm
from pathlib import Path
import tensorflow as tf
import skimage.io
import matplotlib.pyplot as plt
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state 
<data_type_conversions>
!git clone https://www.github.com/matterport/Mask_RCNN.git
os.chdir('Mask_RCNN')
 !rm -rf.git
!rm -rf images assets
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head() 
<categorify>
DATA_DIR = Path('/kaggle/input')
 ROOT_DIR = Path('/kaggle/working' )
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
df_train.head()
df_train.loc[df_train.Country == 'Afghanistan', :]
df_test.tail() 
<categorify>
sys.path.append(ROOT_DIR/'Mask_RCNN' )
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
<save_to_csv>
!pip install pycocotools
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )<import_modules>
from mrcnn.config import Config
from mrcnn import utils
import mrcnn.model as modellib
from mrcnn import visualize
from mrcnn.model import log
print("Read in libraries")
<load_from_csv>
!wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state 
<data_type_conversions>
COCO_MODEL_PATH = 'mask_rcnn_coco.h5'
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head() 
<categorify>
class InferenceConfig(coco.CocoConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
IMAGE_MIN_DIM = 256
IMAGE_MAX_DIM = 256
config = InferenceConfig()
config.display()
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
df_train.head()
df_train.loc[df_train.Country == 'Afghanistan', :]
df_test.tail() 
<categorify>
model = modellib.MaskRCNN(mode="inference", config=config, model_dir=ROOT_DIR)
 model.load_weights(COCO_MODEL_PATH, by_name=True )
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
<save_to_csv>
class_names = ['BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane',
'bus', 'train', 'truck', 'boat', 'traffic light',
'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird',
'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear',
'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball',
'kite', 'baseball bat', 'baseball glove', 'skateboard',
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup',
'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza',
'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed',
'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote',
'keyboard', 'cell phone', 'microwave', 'oven', 'toaster',
'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors',
'teddy bear', 'hair drier', 'toothbrush']
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )<import_modules>
IMAGE_DIR = "/kaggle/input/test/"
idx = pd.IndexSlice<load_from_csv>
class_lookup_df = pd.read_csv("./challenge-2019-classes-description-segmentable.csv", header=None)
 empty_submission_df = pd.read_csv("input/sample_empty_submission.csv" )
df = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', index_col=0 )<filter>
class_lookup_df.columns = ["encoded_label","label"]
class_lookup_df['label'] = class_lookup_df['label'].str.lower()
class_lookup_df.head()
df['Country_Region'].replace('Taiwan*', 'Taiwan', inplace=True)
 df = df[df['Country_Region'] != 'Diamond Princess']<load_from_csv>
def encode_binary_mask(mask: np.ndarray)-> t.Text:

 if mask.dtype != np.bool:
raise ValueError("encode_binary_mask expects a binary mask, received dtype == %s" % mask.dtype)
 mask = np.squeeze(mask)
 if len(mask.shape)!= 2:
raise ValueError("encode_binary_mask expects a 2d mask, received shape == %s" % mask.shape)
 mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)
 mask_to_encode = mask_to_encode.astype(np.uint8)
 mask_to_encode = np.asfortranarray(mask_to_encode)
 encoded_mask = coco_mask.encode(mask_to_encode)[0]["counts"]
binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)
 base64_str = base64.b64encode(binary_str)
 return base64_str
df_codes =(pd.read_csv('.. /input/my-covid19-dataset/iso-country-codes.csv')
.rename(columns={'Country Name': 'Country_Region', 'Alpha-3 code': 'Country Code'})
.loc[:,['Country_Region', 'Country Code']])
 df =(pd.merge(df, df_codes, how='left', on='Country_Region'))<feature_engineering>
ImageID_list = []
ImageWidth_list = []
ImageHeight_list = []
PredictionString_list = []
for num, row in tqdm(empty_submission_df.iterrows() , total=len(empty_submission_df)) :
filename = row["ImageID"] + ".jpg"
image = skimage.io.imread(os.path.join(IMAGE_DIR, filename))
results = model.detect([image])
 r = results[0]
height = image.shape[0]
width = image.shape[1]
PredictionString = ""
for i in range(len(r["class_ids"])) :
class_id = r["class_ids"][i]
roi = r["rois"][i]
mask = r["masks"][:,:,i]
confidence = r["scores"][i]
encoded_mask = encode_binary_mask(mask)
 labelname = class_names[r['class_ids'][i]]
if class_lookup_df[class_lookup_df["label"] == labelname].shape[0] == 0:
continue
encoded_label = class_lookup_df[class_lookup_df["label"] == labelname]["encoded_label"].item()
PredictionString += encoded_label
PredictionString += " "
PredictionString += str(confidence)
 PredictionString += " "
PredictionString += encoded_mask.decode()
PredictionString += " "
ImageID_list.append(row["ImageID"])
 ImageWidth_list.append(width)
 ImageHeight_list.append(height)
 PredictionString_list.append(PredictionString )
def location(state, country):
if type(state)==str and not state==country:
return country + ' - ' + state
else:
return country
df['Date'] = df['Date'].apply(lambda x:(dt.datetime.strptime(x, '%Y-%m-%d')))
 df['Location'] = df[['Province_State', 'Country_Region']].apply(lambda row: location(row[0],row[1]), axis=1)
 t_start = df['Date'].unique() [0]
t_end = df['Date'].unique() [-1]
print('Number of Locations: ' + str(len(df['Location'].unique())))
 print('Dates: from ' + np.datetime_as_string(t_start, unit='D')+ ' to ' +
np.datetime_as_string(t_end, unit='D')+ '
' )<filter>
results=pd.DataFrame({"ImageID":ImageID_list,
"ImageWidth":ImageWidth_list,
"ImageHeight":ImageHeight_list,
"PredictionString":PredictionString_list
} )
<load_from_csv><EOS> 
results.to_csv("submission.csv", index=False )