Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<compute_test_metric>
TRAIN_ABLE_FALSE=True
if TRAIN_ABLE_FALSE:
os.environ['CUDA_VISIBLE_DEVICES'] = "1"
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )<train_model>
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='.. /input/understanding_cloud_organization/train_images',
batch_size=32, dim=(1400, 2100), n_channels=3, reshape=None, gamma=None,
augment=False, n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.reshape = reshape
self.gamma = gamma
self.n_channels = n_channels
self.augment = augment
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
np.random.seed(self.random_state)
 def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 if self.augment:
X, y = self.__augment_batch(X, y)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
if self.reshape is None:
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
else:
X = np.empty(( self.batch_size, *self.reshape, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_rgb(img_path)
 if self.reshape is not None:
img = np_resize(img, self.reshape)
 if self.gamma is not None:
img = adjust_gamma(img, gamma=self.gamma)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
if self.reshape is None:
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 else:
y = np.empty(( self.batch_size, *self.reshape, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
if self.reshape is not None:
masks = build_masks(rles, input_shape=self.dim, reshape=self.reshape)
 else:
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img
def __random_transform(self, img, masks):
composition = albu.Compose([
albu.HorizontalFlip() ,
albu.VerticalFlip() ,
albu.ShiftScaleRotate(rotate_limit=30, shift_limit=0.1)
 ])
 composed = composition(image=img, mask=masks)
 aug_img = composed['image']
aug_masks = composed['mask']
return aug_img, aug_masks
def __augment_batch(self, img_batch, masks_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ], masks_batch[i, ] = self.__random_transform(
 img_batch[i, ], masks_batch[i, ])
 return img_batch, masks_batch<split>
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()
BATCH_SIZE = 32
train_idx, val_idx = train_test_split(
 mask_count_df.index, random_state=2019, test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
reshape=(320, 480),
gamma=0.8,
augment=True,
n_channels=3,
n_classes=4 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
reshape=(320, 480),
gamma=0.8,
augment=False,
n_channels=3,
n_classes=4
)<choose_model_class>
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train
opt = AdamAccumulate(lr=0.002, accum_iters=8)
 model = sm.Unet(
 'resnet18',
classes=4,
input_shape=(320, 480, 3),
activation='sigmoid' 
)
 model.compile(optimizer=opt, loss=bce_dice_loss, metrics=[dice_coef])
 model.summary()<load_pretrained>
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, static_feats, deploy=deploy)
 return basetable
model.load_weights('.. /input/resunet18lb598/model.h5' )<categorify>
%time train_basetable = create_features(train, False )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 350, 525), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<normalization>
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )
model = tta_segmentation(model, h_flip=True, h_shift=(-10, 10), merge='mean' )<define_variables>
scaler = StandardScaler()
X = scaler.fit_transform(X )
best_threshold = 0.45
best_size = 15000
threshold = best_threshold
min_size = best_size
test_df = []
encoded_pixels = []
TEST_BATCH_SIZE = 500
for i in range(0, test_imgs.shape[0], TEST_BATCH_SIZE):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + TEST_BATCH_SIZE)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
dim=(350, 525),
reshape=(320, 480),
n_channels=3,
gamma=0.8,
base_path='.. /input/understanding_cloud_organization/test_images',
target_df=sub_df,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model.predict_generator(
 test_generator,
workers=1,
verbose=1 
)
 for j, idx in enumerate(batch_idx):
filename = test_imgs['ImageId'].iloc[idx]
image_df = sub_df[sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ]
for k in range(pred_masks.shape[-1]):
pred_mask = pred_masks[...,k].astype('float32')
 if pred_mask.shape !=(350, 525):
pred_mask = cv2.resize(pred_mask, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 pred_mask, num_predict = post_process(pred_mask, threshold, min_size)
 if num_predict == 0:
encoded_pixels.append('')
 else:
r = mask2rle(pred_mask)
 encoded_pixels.append(r)
 sub_df['EncodedPixels'] = encoded_pixels<categorify>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )
def rle_decode(mask_rle, shape=(1400, 2100)) :
s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape, order='F' )<save_to_csv>
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s
sub_df.to_csv('submission.csv', columns=['Image_Label', 'EncodedPixels'], index=False)
 sub_df.head(10 )<load_from_csv>
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps
NUMBER_DAYS = 100
NUMBER_FAMILIES = 5000
MAX_BEST_CHOICE = 5
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv')
 submission = pd.read_csv('/kaggle/input/c-stochastic-product-search-65ns/submission.csv')
 assigned_days = submission['assigned_day'].values
columns = data.columns[1:11]
DESIRED = data[columns].values
COST_PER_FAMILY = [0,50,50,100,200,200,300,300,400,500]
COST_PER_FAMILY_MEMBER = [0, 0, 9, 9, 9, 18, 18, 36, 36,235]
N_PEOPLE = data['n_people'].astype(int ).values
def get_daily_occupancy(assigned_days):
daily_occupancy = np.zeros(100, np.int32)
 for i, r in enumerate(assigned_days):
daily_occupancy[r-1] += N_PEOPLE[i]
return daily_occupancy
def cost_function(prediction):
N_DAYS = 100
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
penalty = 0
days = list(range(N_DAYS,0,-1))
tmp = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 family_size_dict = tmp[['n_people']].to_dict() ['n_people']
cols = [f'choice_{i}' for i in range(10)]
choice_dict = tmp[cols].to_dict()
daily_occupancy = {k:0 for k in days}
for f, d in enumerate(prediction):
n = family_size_dict[f]
choice_0 = choice_dict['choice_0'][f]
choice_1 = choice_dict['choice_1'][f]
choice_2 = choice_dict['choice_2'][f]
choice_3 = choice_dict['choice_3'][f]
choice_4 = choice_dict['choice_4'][f]
choice_5 = choice_dict['choice_5'][f]
choice_6 = choice_dict['choice_6'][f]
choice_7 = choice_dict['choice_7'][f]
choice_8 = choice_dict['choice_8'][f]
choice_9 = choice_dict['choice_9'][f]
daily_occupancy[d] += n
if d == choice_0:
penalty += 0
elif d == choice_1:
penalty += 50
elif d == choice_2:
penalty += 50 + 9 * n
elif d == choice_3:
penalty += 100 + 9 * n
elif d == choice_4:
penalty += 200 + 9 * n
elif d == choice_5:
penalty += 200 + 18 * n
elif d == choice_6:
penalty += 300 + 18 * n
elif d == choice_7:
penalty += 300 + 36 * n
elif d == choice_8:
penalty += 400 + 36 * n
elif d == choice_9:
penalty += 500 + 36 * n + 199 * n
else:
penalty += 500 + 36 * n + 398 * n
for _, v in daily_occupancy.items() :
if v > MAX_OCCUPANCY or v < MIN_OCCUPANCY:
penalty += 100000000
accounting_cost = max(0,(daily_occupancy[days[0]]-125.0)/ 400.0 * daily_occupancy[days[0]]**(0.5))
yesterday_count = daily_occupancy[days[0]]
for day in days[1:]:
today_count = daily_occupancy[day]
diff = abs(today_count - yesterday_count)
 accounting_cost += max(0,(today_count-125.0)/ 400.0 * today_count**(0.5 + diff / 50.0))
yesterday_count = today_count
return penalty, accounting_cost, penalty + accounting_cost<categorify>
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred
%%time
for num_members in range(2, 9):
daily_occupancy = get_daily_occupancy(assigned_days)
 fids = np.where(N_PEOPLE == num_members)[0]
PCOSTM = {}
for fid in range(NUMBER_FAMILIES):
if fid in fids:
for i in range(MAX_BEST_CHOICE):
PCOSTM[fid, DESIRED[fid][i]-1] = COST_PER_FAMILY[i] + N_PEOPLE[fid] * COST_PER_FAMILY_MEMBER[i]
else:
daily_occupancy[assigned_days[fid]-1] -= N_PEOPLE[fid]
offset = fids.shape[0]
solver = pywrapgraph.SimpleMinCostFlow()
for day in range(NUMBER_DAYS):
solver.SetNodeSupply(offset+day, int(daily_occupancy[day]//num_members))
for i in range(offset):
fid = fids[i]
solver.SetNodeSupply(i, -1)
 for j in range(MAX_BEST_CHOICE):
day = DESIRED[fid][j]-1
solver.AddArcWithCapacityAndUnitCost(int(offset+day), i, 1, int(PCOSTM[fid, day]))
solver.SolveMaxFlowWithMinCost()
for i in range(solver.NumArcs()):
if solver.Flow(i)> 0:
assigned_days[fids[solver.Head(i)]] = solver.Tail(i)- offset + 1
print(cost_function(assigned_days))<save_to_csv>
print("mean crps is %f"%np.mean(crps_csv))
submission['assigned_day'] = assigned_days
submission.to_csv('submission.csv', index=False )<load_from_csv>
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()