Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<compute_test_metric>
%matplotlib inline
plt.style.use('ggplot')
 plt.rcParams['figure.figsize'] =(12,8)
 font = {'family': 'Verdana',
'weight': 'normal'}
plt.rc('font', **font )
def get_angular_status(angle, x, x_a):
if angle:
result = [x, x_a]
else:
result = x
return result 
<train_model>
data_train = pd.read_csv(".. /input/train.csv")
 data_test = pd.read_csv(".. /input/test.csv")
 data_store = pd.read_csv(".. /input/store.csv" )
def gen_model_weights(angle, lr, decay, channels, relu, batch_size, epochs, path_name, data, only_load=False, verbose=2):
X_train, X_angle_train, y_train, X_val, X_angles_val, y_val = data
X_train, X_angle_train, y_train = shuffle(X_train, X_angle_train, y_train, random_state=np.random.randint(1, 123))
model, partial_model = get_model_notebook(angle, lr, decay, channels, relu)
 if only_load:
model.load_weights(path_name)
 return model, partial_model
model = train_model(model, batch_size, epochs, path_name,
get_angular_status(angle, X_train, X_angle_train), y_train, verbose=verbose)
 if verbose > 0:
loss_val, acc_val = model.evaluate(get_angular_status(angle, X_val, X_angles_val), y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = model.evaluate(get_angular_status(angle, X_train, X_angle_train), y_train,
verbose=0, batch_size=batch_size)
 print('Val/Train Loss:', str(loss_val)+ '/' + str(loss_train), \
'Val/Train Acc:', str(acc_val)+ '/' + str(acc_train))
return model, partial_model<split>
data_train.StateHoliday = data_train.StateHoliday.replace(0,'0')
 data_test.StateHoliday = data_test.StateHoliday.replace(0,'0')
 data_train.DayOfWeek.value_counts()
def train_models(dataset, lr, batch_size, max_epoch, verbose=2, return_model=False):
X_angles, y_train, X_b, X_images = dataset
angle_b = True
angle_images = True
X_angles, X_angles_val,\
y_train, y_val,\
X_b, X_b_val,\
X_images, X_images_val = train_test_split(X_angles, y_train, X_b, X_images, random_state=687, train_size=0.9)
 if train_b:
if verbose > 0:
print('Training bandwidth network')
 data_b1 =(X_b, X_angles, y_train, X_b_val, X_angles_val, y_val)
 model_b, model_b_cut = gen_model_weights(angle_b, lr, 0, 3, 'relu', batch_size, max_epoch, 'model_b',
data_b1, only_load=load_b, verbose=verbose)
 if train_img:
if verbose > 0:
print('Training image network')
 data_images =(X_images, X_angles, y_train, X_b_val, X_angles_val, y_val)
 model_images, model_images_cut = gen_model_weights(angle_images, lr, 0, 3, 'relu', batch_size, max_epoch, 'model_img',
data_images, only_load=load_img, verbose=verbose)
 if train_total:
common_model = combined_model(model_b_cut, model_images_cut, lr, 0)
 common_x_train = [X_b, X_images, X_angles]
common_y_train = y_train
common_x_val = [X_b_val, X_images_val, X_angles_val]
common_y_val = y_val
if verbose > 0:
print('Training common network')
 common_model = train_model(common_model, batch_size, max_epoch, 'common_check', common_x_train,
common_y_train, verbose=verbose, val_split=0.2)
 loss_val, acc_val = common_model.evaluate(common_x_val, common_y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = common_model.evaluate(common_x_train, common_y_train,
verbose=0, batch_size=batch_size)
 if verbose > 0:
print('Loss:', loss_val, 'Acc:', acc_val)
 if return_model:
return common_model
else:
return(loss_train, acc_train),(loss_val, acc_val )<train_model>
data_train['Year'] = data_train['Date'].apply(lambda x: int(x[:4]))
data_train['Month'] = data_train['Date'].apply(lambda x: int(x[5:7]))
data_train.head()
common_model = train_models(( X_angles, y_train, X_b, X_images), 5e-04, 32, 50, 1, return_model=True )<predict_on_test>
data_train['StateHoliday'] = data_train['StateHoliday'].replace(0, '0')
 data_train["HolidayBin"] = data_train['StateHoliday'].map({"0": 0, "a": 1, "b": 1, "c": 1})
 data_train.StateHoliday.unique()
if predict_submission:
print('Reading test dataset')
 test = pd.read_json(".. /input/test.json")
 test.inc_angle = test.inc_angle.replace('na', 0)
 test.inc_angle = test.inc_angle.astype(float ).fillna(0.0)
 y_fin, X_angle_fin, X_fin_b, X_fin_img = create_dataset(test, False)
 print('X shape:', X_fin_img.shape)
 print('X angle shape:', X_angle_fin.shape)
 print('Predicting')
 prediction = common_model.predict([X_fin_b, X_fin_img, X_angle_fin], verbose=1, batch_size=32)
 print('Submitting')
 submission = pd.DataFrame({'id': test["id"], 'is_iceberg': prediction.reshape(( prediction.shape[0])) })
 submission.to_csv("./submission.csv", index=False)
 print('Done' )<data_type_conversions>
average_customers_per_month = data_train.groupby('Month')['Customers'].mean()
average_sales_per_month = data_train.groupby('Month')['Sales'].mean()
train = pd.read_json(".. /input/train.json")
 test = pd.read_json(".. /input/test.json")
 train.inc_angle = train.inc_angle.replace('na', 0)
 train.inc_angle = train.inc_angle.astype(float ).fillna(0.0)
 print("done!" )<prepare_x_and_y>
total_customers_for_store = data_train.groupby('Store')['Sales', 'Customers'].sum()
x_band1 = np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in train["band_1"]])
 x_band2 = np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in train["band_2"]])
 X_train = np.concatenate([x_band1[:, :, :, np.newaxis]
, x_band2[:, :, :, np.newaxis]
,(( x_band1+x_band1)/2)[:, :, :, np.newaxis]], axis=-1)
 X_angle_train = np.array(train.inc_angle)
 y_train = np.array(train["is_iceberg"])
 x_band1 = np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in test["band_1"]])
 x_band2 = np.array([np.array(band ).astype(np.float32 ).reshape(75, 75)for band in test["band_2"]])
 X_test = np.concatenate([x_band1[:, :, :, np.newaxis]
, x_band2[:, :, :, np.newaxis]
,(( x_band1+x_band1)/2)[:, :, :, np.newaxis]], axis=-1)
 X_angle_test = np.array(test.inc_angle)
 X_train, X_valid, X_angle_train, X_angle_valid, y_train, y_valid = train_test_split(X_train
, X_angle_train, y_train, random_state=123, train_size=0.75 )<choose_model_class>
data_total_customers_for_store = pd.DataFrame({'Sales': total_customers_for_store['Sales'],
'Customers': total_customers_for_store['Customers']},
index = total_customers_for_store.index )
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def get_model() :
bn_model = 0
p_activation = "elu"
input_1 = Input(shape=(75, 75, 3), name="X_1")
 input_2 = Input(shape=[1], name="angle")
 img_1 = Conv2D(16, kernel_size =(3,3), activation=p_activation )(( BatchNormalization(momentum=bn_model))(input_1))
img_1 = Conv2D(16, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = MaxPooling2D(( 2,2))(img_1)
 img_1 = Dropout(0.2 )(img_1)
 img_1 = Conv2D(32, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = Conv2D(32, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = MaxPooling2D(( 2,2))(img_1)
 img_1 = Dropout(0.2 )(img_1)
 img_1 = Conv2D(64, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = Conv2D(64, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = MaxPooling2D(( 2,2))(img_1)
 img_1 = Dropout(0.2 )(img_1)
 img_1 = Conv2D(128, kernel_size =(3,3), activation=p_activation )(img_1)
 img_1 = MaxPooling2D(( 2,2))(img_1)
 img_1 = Dropout(0.2 )(img_1)
 img_1 = GlobalMaxPooling2D()(img_1)
 img_2 = Conv2D(128, kernel_size =(3,3), activation=p_activation )(( BatchNormalization(momentum=bn_model))(input_1))
img_2 = MaxPooling2D(( 2,2))(img_2)
 img_2 = Dropout(0.2 )(img_2)
 img_2 = GlobalMaxPooling2D()(img_2)
 img_concat =(Concatenate()([img_1, img_2, BatchNormalization(momentum=bn_model )(input_2)]))
dense_ayer = Dropout(0.5 )(BatchNormalization(momentum=bn_model )(Dense(256, activation=p_activation )(img_concat)))
 dense_ayer = Dropout(0.5 )(BatchNormalization(momentum=bn_model )(Dense(64, activation=p_activation )(dense_ayer)))
 output = Dense(1, activation="sigmoid" )(dense_ayer)
 model = Model([input_1,input_2], output)
 optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model
model = get_model()
model.summary()<train_model>
data_total_customers_for_store = data_total_customers_for_store.reset_index()
file_path = ".model_weights.hdf5"
callbacks = get_callbacks(filepath=file_path, patience=5)
 model = get_model()
model.fit([X_train, X_angle_train], y_train, epochs=25
, validation_data=([X_valid, X_angle_valid], y_valid)
 , batch_size=32
, callbacks=callbacks )<train_model>
average_sales_customers = data_train.groupby('Store')['Sales', 'Customers'].mean()
model.load_weights(filepath=file_path)
 print("Train evaluate:")
 print(model.evaluate([X_train, X_angle_train], y_train, verbose=1, batch_size=200))
print("
print("watch list evaluate:")
 print(model.evaluate([X_valid, X_angle_valid], y_valid, verbose=1, batch_size=200))<predict_on_test>
data_average_sales_customers = pd.DataFrame({'Sales': average_sales_customers['Sales'],
'Customers': average_sales_customers['Customers']},
index = average_sales_customers.index)
 data_average_sales_customers = data_average_sales_customers.reset_index()
data_stores_average = data_average_sales_customers.join(data_store.set_index('Store'), on='Store')
 data_stores_average.head()
prediction = model.predict([X_test, X_angle_test], verbose=1, batch_size=200 )<prepare_output>
data_average_sales_customers = pd.DataFrame({'Sales': average_sales_customers['Sales'],
'Customers': average_sales_customers['Customers']},
index = average_sales_customers.index )
submission = pd.DataFrame({'id': test["id"], 'is_iceberg': prediction.reshape(( prediction.shape[0])) })
 submission.head(10 )<save_to_csv>
data_average_sales_customers = data_average_sales_customers.reset_index()
submission.to_csv("./submission.csv", index=False )<set_options>
data_stores_average = data_average_sales_customers.join(data_store.set_index('Store'), on='Store' )
env = make("halite", debug=True)
 env.render()<set_options>
data_stores_new = data_total_customers_for_store.join(data_store.set_index('Store'), on='Store' )
%%writefile submission.py

 CONFIG_MAX_SHIPS=20
all_actions=[ShipAction.NORTH, ShipAction.EAST,ShipAction.SOUTH,ShipAction.WEST]
all_dirs=[Point(0,1), Point(1,0), Point(0,-1), Point(-1,0)]
start=None
num_shipyard_targets=4
size=None
ship_target={}
me=None
did_init=False
quiet=False
C=None
class Obj:
pass
turn=Obj()
turns_optimal=np.array(
 [[0, 2, 3, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8],
[0, 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7],
[0, 0, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7],
[0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6],
[0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6],
[0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
 def print_enemy_ships(board):
print('
Enemy Ships')
 for ship in board.ships.values() :
if ship.player_id != me.id:
print('{:6} {} halite {}'.format(ship.id,ship.position,ship.halite))
def print_actions(board):
print('
Ship Actions')
 for ship in me.ships:
print('{:6} {} {} halite {}'.format(ship.id,ship.position,ship.next_action,ship.halite))
print('Shipyard Actions')
 for sy in me.shipyards:
print('{:6} {} {}'.format(sy.id,sy.position,sy.next_action))
def print_none(*args):
pass
def compute_max_ships(step):
if step < 200:
return CONFIG_MAX_SHIPS
elif step < 300:
return CONFIG_MAX_SHIPS-2
elif step < 350:
return CONFIG_MAX_SHIPS-4
else:
return CONFIG_MAX_SHIPS-5
def set_turn_data(board):
turn.num_ships=len(me.ships)
 turn.max_ships=compute_max_ships(board.step)
 turn.total_halite=me.halite
turn.halite_matrix=np.reshape(board.observation['halite'],(board.configuration.size,board.configuration.size))
turn.num_shipyards=len(me.shipyards)
 turn.EP,turn.EH,turn.ES=gen_enemy_halite_matrix(board)
 turn.taken={}
turn.last_episode =(board.step ==(board.configuration.episode_steps-2))
def init(obs,config):
global size
global print
if hasattr(config,'myval')and config.myval==9 and not quiet:
pass
else:
print=print_none
pprint.pprint=print_none
size = config.size
def limit(x,a,b):
if x<a:
return a
if x>b:
return b
return x
def num_turns_to_mine(C,H,rt_travel):
if C==0:
ch=0
elif H==0:
ch=turns_optimal.shape[0]
else:
ch=int(math.log(C/H)*2.5+5.5)
 ch=limit(ch,0,turns_optimal.shape[0]-1)
 rt_travel=int(limit(rt_travel,0,turns_optimal.shape[1]-1))
return turns_optimal[ch,rt_travel]
def halite_per_turn(carrying, halite,travel,min_mine=1):
turns=num_turns_to_mine(carrying,halite,travel)
 if turns<min_mine:
turns=min_mine
mined=carrying+(1-.75**turns)*halite
return mined/(travel+turns), turns
def move(pos, action):
ret=None
if action==ShipAction.NORTH:
ret=pos+Point(0,1)
 if action==ShipAction.SOUTH:
ret=pos+Point(0,-1)
 if action==ShipAction.EAST:
ret=pos+Point(1,0)
 if action==ShipAction.WEST:
ret=pos+Point(-1,0)
 if ret is None:
ret=pos
return ret % size
def dirs_to(p1, p2, size=21):
deltaX, deltaY=p2 - p1
if abs(deltaX)>size/2:
if deltaX<0:
deltaX+=size
elif deltaX>0:
deltaX-=size
if abs(deltaY)>size/2:
if deltaY<0:
deltaY+=size
elif deltaY>0:
deltaY-=size
ret=[]
if deltaX>0:
ret.append(ShipAction.EAST)
 if deltaX<0:
ret.append(ShipAction.WEST)
 if deltaY>0:
ret.append(ShipAction.NORTH)
 if deltaY<0:
ret.append(ShipAction.SOUTH)
 if len(ret)==0:
ret=[None]
return ret,(deltaX,deltaY)
 def shipyard_actions() :
for sy in me.shipyards:
if turn.num_ships < turn.max_ships:
if turn.total_halite >= 500 and sy.position not in turn.taken:
sy.next_action = ShipyardAction.SPAWN
turn.taken[sy.position]=1
turn.num_ships+=1
turn.total_halite-=500
def gen_enemy_halite_matrix(board):
EP=np.zeros(( size,size))
EH=np.zeros(( size,size))
ES=np.zeros(( size,size))
for id,ship in board.ships.items() :
if ship.player_id != me.id:
EH[ship.position.y,ship.position.x]=ship.halite
EP[ship.position.y,ship.position.x]=1
for id, sy in board.shipyards.items() :
if sy.player_id != me.id:
ES[sy.position.y,sy.position.x]=1
return EP,EH,ES
def dist(a,b):
action,step=dirs_to(a, b, size=21)
 return abs(step[0])+ abs(step[1])
 def nearest_shipyard(pos):
mn=100
best_pos=None
for sy in me.shipyards:
d=dist(pos, sy.position)
 if d<mn:
mn=d
best_pos=sy.position
return mn,best_pos
def assign_targets(board,ships):
old_target=copy.copy(ship_target)
 ship_target.clear()
if len(ships)==0:
return
halite_min=50
pts1=[]
pts2=[]
for pt,c in board.cells.items() :
assert isinstance(pt,Point)
 if c.halite > halite_min:
pts1.append(pt)
 for sy in me.shipyards:
for i in range(num_shipyard_targets):
pts2.append(sy.position)
 C=np.zeros(( len(ships),len(pts1)+len(pts2)))
 for i,ship in enumerate(ships):
for j,pt in enumerate(pts1+pts2):
d1=dist(ship.position,pt)
 d2,shipyard_position=nearest_shipyard(pt)
 if shipyard_position is None:
d2=1
my_halite=ship.halite
if j < len(pts1):
v, mining=halite_per_turn(my_halite,board.cells[pt].halite, d1+d2)
 else:
if d1>0:
v=my_halite/d1
else:
v=0
if board.cells[pt].ship and board.cells[pt].ship.player_id != me.id:
enemy_halite=board.cells[pt].ship.halite
if enemy_halite <= my_halite:
v = -1000
else:
if d1<5:
v+= enemy_halite/(d1+1)
 C[i,j]=v
print('C is {}'.format(C.shape))
row,col=scipy.optimize.linear_sum_assignment(C, maximize=True)
 pts=pts1+pts2
for r,c in zip(row,col):
ship_target[ships[r].id]=pts[c]
print('
Ship Targets')
 print('Ship position target')
 for id,t in ship_target.items() :
st=''
ta=''
if board.ships[id].position==t:
st='MINE'
elif len(me.shipyards)>0 and t==me.shipyards[0].position:
st='SHIPYARD'
if id not in old_target or old_target[id] != ship_target[id]:
ta=' NEWTARGET'
print('{0:6} at({1[0]:2},{1[1]:2})assigned({2[0]:2},{2[1]:2})h {3:3} {4:10} {5:10}'.format(
 id, board.ships[id].position, t, board.cells[t].halite,st, ta))
return
def make_avoidance_matrix(myship_halite):
filter=np.array([[0,1,0],[1,1,1],[0,1,0]])
 bad_ship=np.logical_and(turn.EH <= myship_halite,turn.EP)
 avoid=scipy.ndimage.convolve(bad_ship, filter, mode='wrap',cval=0.0)
 avoid=np.logical_or(avoid,turn.ES)
 return avoid
def make_attack_matrix(myship_halite):
attack=np.logical_and(turn.EH > myship_halite,turn.EP)
 return attack
def get_max_halite_ship(board, avoid_danger=True):
mx=-1
the_ship=None
for ship in me.ships:
x=ship.position.x
y=ship.position.y
avoid=make_avoidance_matrix(ship.halite)
 if ship.halite>mx and(not avoid_danger or not avoid[y,x]):
mx=ship.halite
the_ship=ship
return the_ship
def remove_dups(p):
ret=[]
for x in p:
if x not in ret:
ret.append(x)
 return ret
def matrix_lookup(matrix,pos):
return matrix[pos.y,pos.x]
def ship_converts(board):
if turn.num_shipyards==0 and not turn.last_episode:
mx=get_max_halite_ship(board)
 if mx is not None:
if mx.halite + turn.total_halite > 500:
mx.next_action=ShipAction.CONVERT
turn.taken[mx.position]=1
turn.num_shipyards+=1
turn.total_halite-=500
for ship in me.ships:
if ship.next_action:
continue
avoid=make_avoidance_matrix(ship.halite)
 z=[matrix_lookup(avoid,move(ship.position,a)) for a in all_actions]
if np.all(z)and ship.halite > 500:
ship.next_action=ShipAction.CONVERT
turn.taken[ship.position]=1
turn.num_shipyards+=1
turn.total_halite-=500
print('ship id {} no escape converting'.format(ship.id))
if turn.last_episode and ship.halite > 500:
ship.next_action=ShipAction.CONVERT
turn.taken[ship.position]=1
turn.num_shipyards+=1
turn.total_halite-=500
def ship_moves(board):
ships=[ship for ship in me.ships if ship.next_action is None]
assign_targets(board,ships)
 actions={}
for ship in ships:
if ship.id in ship_target:
a,delta = dirs_to(ship.position, ship_target[ship.id],size=size)
 actions[ship.id]=a
else:
actions[ship.id]=[random.choice(all_actions)]
for ship in ships:
action=None
x=ship.position
avoid=make_avoidance_matrix(ship.halite)
 attack=make_attack_matrix(ship.halite)
 action_list=actions[ship.id]+[None]+all_actions
for a in all_actions:
m=move(x,a)
 if attack[m.y,m.x]:
print('ship id {} attacking {}'.format(ship.id,a))
action_list.insert(0,a)
 break
action_list=remove_dups(action_list)
 for a in action_list:
m=move(x,a)
 if avoid[m.y,m.x]:
print('ship id {} avoiding {}'.format(ship.id,a))
if m not in turn.taken and not avoid[m.y,m.x]:
action=a
break
ship.next_action=action
turn.taken[m]=1
def agent(obs, config):
global size
global start
global prev_board
global me
global did_init
start_step=time.time()
if start is None:
start=time.time()
if not did_init:
init(obs,config)
 did_init=True
board = Board(obs, config)
 me=board.current_player
set_turn_data(board)
 print('==== step {} sim {}'.format(board.step,board.step+1))
print('ships {} shipyards {}'.format(turn.num_ships,turn.num_shipyards))
print_enemy_ships(board)
 ship_converts(board)
 ship_moves(board)
 shipyard_actions()
print_actions(board)
 print('time this turn: {:8.3f} total elapsed {:8.3f}'.format(time.time() -start_step,time.time() -start))
return me.next_actions 
<statistical_test>
average_store_type = data_stores_new.groupby('StoreType')['Sales', 'Customers', 'CompetitionDistance'].mean()
%%writefile main.py
def random_agent_seeded(observation, configuration, seed=42):
if observation.step == 0:
random.seed(seed)
 action = random.randint(0, configuration.signs-1)
 print(f'random_agent_seeded({seed})= {action}')
 return int(action)
 def random_agent_unseeded(observation, configuration):
if observation.step == 0:
random.seed(None)
 action = random.randint(0, configuration.signs-1)
 print(f'random_agent_unseeded() = {action}')
 return int(action)
<categorify>
Data_cmp = pd.DataFrame()
Data_cmp['Customers'] = average_store_type['Sales']
Data_cmp['Sales'] = average_store_type['Customers']
Data_cmp['Comp'] = average_store_type['CompetitionDistance']
%%writefile -a main.py
mp.dps = 2048
def encode_irrational(irrational: Union[str,mp.mpf], offset=0)-> List[int]:

 if isinstance(irrational, list)and all([ 0 <= n <= 2 for n in irrational ]):
return irrational
string = re.sub('[^1-9]', '', str(irrational))
sequence = [ 
(int(c)+ int(offset)) % 3
for c in string
]
assert len(sequence)>= 1000
return sequence
class IrrationalAgent() :

 irrationals = {
name: encode_irrational(irrational)
 for name, irrational in {
'pi': mp.pi() ,
'phi': mp.phi() ,
'e': mp.e() ,
'sqrt2': mp.sqrt(2),
'sqrt5': mp.sqrt(5),
'euler': mp.euler() ,
'catalan': mp.catalan() ,
'apery': mp.apery() ,
}.items()
}
def __init__(self, name='irrational', irrational: Union[str,mp.mpf] = None, offset=0, verbose=True):
if irrational is not None and(name == 'irrational' or name in self.irrationals.keys()):
name = 'secret'
if name == 'irrational':
irrational = self.generate_secure_irrational()
if name in self.irrationals.keys() :
irrational = self.irrationals[name]
self.irrational = self.encode_irrational(irrational, offset=offset)
 self.name = name
self.offset = offset
self.verbose = verbose
self.reset()
def reset(self):

 self.history = {
"action": [],
"opponent": []
}
if self.name == 'irrational':
irrational = self.generate_secure_irrational()
self.irrational = self.encode_irrational(irrational, offset=self.offset)
 def __call__(self, obs, conf):
return self.agent(obs, conf)
 def agent(self, obs, conf):

 if obs.step == 0:
self.reset()
if obs.step > 0:
self.history['opponent'].append(obs.lastOpponentAction)
 action = self.action(obs, conf)
 self.history['action'].append(action)
 return action
def action(self, obs, conf):

 action = int(self.irrational[ obs.step % len(self.irrational)])
 action =(action + self.offset)% conf.signs
if self.verbose:
name = self.__class__.__name__ + ':' + self.name +(f'+{self.offset}' if self.offset else '')
 opponent =(self.history['opponent'] or [None])[-1]
expected =(action - 1)% 3
print(f"{obs.step:4d} | {opponent}{self.win_symbol() } > action {action} | " +
f"{name}")
 return action
@staticmethod
def generate_secure_irrational() :

 irrational = sum([
mp.sqrt(n)*(time.monotonic_ns() % 2**32)
 for n in range(2, 5 +(time.monotonic_ns() % 1024))
])
 return irrational
@classmethod
def encode_irrational(cls, irrational: Union[str,mp.mpf], offset=0)-> List[int]:
if irrational is None:
irrational = cls.generate_secure_irrational()
return encode_irrational(irrational, offset)
 def win_symbol(self):

 action =(self.history['action'] or [None])[-1]
opponent =(self.history['opponent'] or [None])[-1]
if isinstance(action, int)and isinstance(opponent, int):
if action % 3 ==(opponent + 1)% 3: return '+'
if action % 3 ==(opponent + 0)% 3: return '|'
if action % 3 ==(opponent - 1)% 3: return '-'
return ' '
irrational_instance = IrrationalAgent(name='pi', offset=0)
 def irrational_agent(obs, conf):
return irrational_instance.agent(obs, conf)
<import_modules>
average_assort = data_stores_new.groupby('Assortment')['Sales', 'Customers'].mean()
%%writefile -a main.py
class IrrationalSearchAgent(IrrationalAgent):

 def __init__(self, name='irrational', irrational=None, search: List[Union[str, mp.mpf]]=None, verbose=True):
search = {
f'irrational
for n, number in enumerate(search)
 } if search is not None else {}
self.irrationals = { **self.__class__.irrationals, **search }
super().__init__(name=name, irrational=irrational, verbose=verbose)
 def action(self, obs, conf):
expected, irrational_name = self.search_irrationals(self.history['opponent'])
 if expected is not None:
action =(expected + 1)% conf.signs
opponent =(self.history['opponent'] or [None])[-1]
if self.verbose:
print(f"{obs.step:4d} | {opponent}{self.win_symbol() } > action {expected} -> {action} | " +
f"Found Irrational: {irrational_name}")
 else:
action = super().action(obs, conf)
 return action
def search_irrationals(self, sequence: List[int])-> Tuple[int, str]:

 expected, irrational_name = None, None
if len(sequence):
for offset in [0,1,2]:
for irrational_name, irrational in self.irrationals.items() :
irrational = [(n + offset)% 3 for n in irrational[:len(sequence)+1] ]
if irrational[:len(sequence)] == sequence:
expected = irrational[len(sequence)]
break
else: continue
break
return expected, irrational_name
irrational_search_instance = IrrationalSearchAgent()
def irrational_search_agent(obs, conf):
return irrational_search_instance.agent(obs, conf)
<feature_engineering>
data_train = pd.read_csv(".. /input/train.csv")
 data_test = pd.read_csv(".. /input/test.csv")
 data_store = pd.read_csv(".. /input/store.csv" )
%%writefile -a main.py
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
if os.environ.get('GFOOTBALL_DATA_DIR', ''):
os.chdir('/kaggle_simulations/agent/')
 class RandomSeedSearch(IrrationalSearchAgent):

 methods = [ 'random' ]
cache_steps = 20
cache_seeds = int(4 * 100_000_000 / len(methods)/ cache_steps)
 cache = {
method: np.load(f'{method}.npy')
 for method in methods
if os.path.exists(f'{method}.npy')
 }
def __init__(self,
min_length=4,
max_offset=1000,
use_stats=True,
cheat=False,
no_submit=False,
verbose=True 
):

 self.use_stats = use_stats
self.cheat = cheat
self.min_length = min_length
self.max_offset = max_offset
self.no_submit = no_submit
self.conf = {'episodeSteps': 1000, 'actTimeout': 1000, 'agentTimeout': 15, 'runTimeout': 1200, 'isProduction': False, 'signs': 3}
super().__init__(verbose=verbose)
 self.print_cache_size()
def reset(self):

 if self.verbose >= 2: print(f'{self.__class__.__name__} | reset() ')
 super().reset()
self.candidate_seeds = defaultdict(lambda: defaultdict(dict))
self.repeating_seeds = defaultdict(lambda: defaultdict(int))
if self.cheat:
self.set_global_seed()
def action(self, obs, conf):
self.conf = conf
if self.no_submit and obs.step > 900:
raise Exception("Don't Submit To Competition")
 irrational, irrational_name = self.search_irrationals(self.history['opponent'])
 if irrational is not None and obs.step > self.min_length + 2:
return super().action(obs, conf)
 expected, seed, offset, method = self.search_cache(self.history['opponent'])
 if seed is None and irrational is not None:
action = super().action(obs, conf)
 elif expected is not None:
action =(expected + 1)% conf.signs
opponent =(self.history['opponent'] or [None])[-1]
if seed is None: seed = f"{'many':>12s}"
else: seed = f"{seed +(offset or 0)/1000:12.3f}"
if self.verbose:
print(
 f"{obs.step:4d} | {opponent}{self.win_symbol() } > action {expected} -> {action} | " +
f"Found RNG Seed: {method:6s} {seed} |",
self.log_repeating_seeds() 
)
 else:
action = super().action(obs, conf)
 return int(action)% conf.signs
def search_cache(self, history: List[int])\
-> Tuple[Optional[int], Optional[int], Optional[int], Optional[str]]:

 expected, seed, offset, method = None, None, None, None
for method in self.methods:
seed, expected, offset = self.search_cache_method(history, method=method)
 if expected is not None:
break
return expected, seed, offset, method
def search_cache_method(self, history: List[int], method='random')\
-> Tuple[Optional[int], Optional[int], Optional[int]]:

 time_start = time.perf_counter()
seed = None
expected = None
offset = None
if method in self.cache.keys() and len(history):
seeds, offset = self.find_candidate_seeds(history, method)
 sequence = history[offset:]
if self.use_stats and len(seeds)>= 2:
seed = None
stats = np.bincount(self.cache[method][seeds,len(sequence)])
 expected = np.argmax(stats)
 if np.count_nonzero(stats)== 1:
seed = np.min(seeds)
 elif len(seeds):
seed = np.min(seeds)
 size = len(sequence)
 if self.cache[method].shape[1] > size:
expected = self.cache[method][seed,size]
else:
expected = self.get_rng_sequence(seed, length=size + 1, method=method)[-1]
if seed is not None:
self.repeating_seeds[method][seed + offset/1000] += 1
if self.verbose >= 2:
time_taken =(time.perf_counter() - time_start)* 1000
print(f'{self.__class__.__name__} | search_cache({method}): {time_taken:.3f}ms')
 return seed, expected, offset
def find_candidate_seeds(self, history, method: str)-> Tuple[np.ndarray, int]:

 seeds_by_offset = {}
max_offset = min(self.max_offset, len(history))
for offset in range(max_offset):
sequence = history[offset:]
size = np.min([len(sequence), self.cache[method].shape[1]])
 if size < self.min_length: continue
if offset in self.candidate_seeds[method]:
candidates = self.candidate_seeds[method][offset]
if len(candidates)== 0:
seeds = candidates
else:
seeds_idx = np.where(np.all(
 self.cache[method][ candidates, :size ] == sequence[ :size ]
, axis=1)) [0]
seeds = candidates[seeds_idx]
else:
seeds = np.where(np.all(
 self.cache[method][ : , :size ] == sequence[ :size ]
, axis=1)) [0]
self.candidate_seeds[method][offset] = seeds
if len(seeds):
seeds_by_offset[offset] = seeds
if len(seeds_by_offset)== 0:
seeds = np.array([])
 offset = 0
else:
offset = sorted(seeds_by_offset.items() , key=lambda pair: len(pair[1])) [0][0]
seeds = seeds_by_offset[offset]
return seeds, offset
def get_rng_sequence(self, seed, length, method='random', use_cache=True)-> List[int]:

 sequence = []
if(use_cache
and method in self.cache.keys()
and seed < self.cache[method].shape[0]
and length < self.cache[method].shape[1] 
):
sequence = self.cache[method][seed][:length]
else:
if method == 'random':
seed_state = random.getstate()
random.seed(seed)
 sequence = [ random.randint(0,2)for _ in range(length)]
random.setstate(seed_state)
 elif method == 'np':
seed_state = np.random.get_state()
np.random.seed(seed)
 sequence = np.random.randint(0,2, length)
 np.random.set_state(seed_state)
 elif method == 'tf':
generator = tf.random.Generator.from_seed(seed)
 sequence = generator.uniform(( length,), minval=0, maxval=3, dtype=tf.dtypes.int32 ).numpy()
return list(sequence)
 def precache(self)-> List[str]:
return [ self.precache_method(method)for method in self.methods ]
def precache_method(self, method='random')-> str:

 filename = f'{method}.npy'
shape =(self.cache_seeds, self.cache_steps)
 if os.path.exists(filename)and np.load(filename ).shape == shape:
print(f'cached {filename:10s} =', shape, '=', naturalsize(os.path.getsize(filename)))
 else:
method_cache = np.array([
self.get_rng_sequence(seed=seed, length=self.cache_steps, method=method)
 for seed in range(self.cache_seeds)
 ], dtype=np.int8)
 np.save(filename, method_cache)
 print(f'wrote {filename:10s} =', method_cache.shape, '=', naturalsize(os.path.getsize(filename)))
 return filename
def log_repeating_seeds(self)-> dict:

 repeating_seeds = {}
for method in self.repeating_seeds.keys() :
repeating_seeds[method] = {
key: value
for key, value in self.repeating_seeds[method].items()
if value >= self.min_length
}
repeating_seeds[method] = dict(sorted(repeating_seeds[method].items() , key=itemgetter(1), reverse=True))
if len(repeating_seeds[method])== 0:
del repeating_seeds[method]
return repeating_seeds
def print_cache_size(self):

 if self.verbose:
filenames = [(filename + ' = ' + naturalsize(os.path.getsize(filename)))
 for filename in glob.glob('*')]
print('tar.gz =', filenames)
 print(f'{self.__class__.__name__}::cache.keys() ', list(self.cache.keys()))
for name, cache in self.cache.items() :
print(f'{self.__class__.__name__}::cache[{name}] = {cache.shape}')
 @staticmethod
def set_global_seed(seed=42):

 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 random_seed_search_instance = RandomSeedSearch()
def random_seed_search_agent(obs, conf):
return random_seed_search_instance.agent(obs, conf)
<set_options>
closed_store_data = data_test["Id"][data_test["Open"] == 0].values
data_train.StateHoliday = data_train.StateHoliday.replace(0,'0')
 data_test.StateHoliday = data_test.StateHoliday.replace(0,'0' )
%run main.py<find_best_params>
data_train['Year'] = data_train['Date'].apply(lambda x: int(x[:4]))
data_train['Month'] = data_train['Date'].apply(lambda x: int(x[5:7]))
data_train["HolidayBin"] = data_train.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1} )
%%time
RandomSeedSearch().precache()<set_options>
del data_train['Date']
del data_train['StateHoliday']
%%time
%run main.py<define_variables>
data_test['Year'] = data_test['Date'].apply(lambda x: int(x[:4]))
data_test['Month'] = data_test['Date'].apply(lambda x: int(x[5:7]))
data_test["HolidayBin"] = data_test.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1} )
%%time
random.seed(42)
 method = 'random'
size = RandomSeedSearch.cache[method].shape[1]
history = np.array([ random.randint(0,2)for _ in range(size)])
 np.where(np.all(
 RandomSeedSearch.cache[method][ : , :size ] == history[ :size ]
, axis=1)) [0]<set_options>
del data_test['Date']
del data_test['StateHoliday']
%%time
%run main.py<set_options>
data_test = data_test[data_test["Open"] != 0]
%%writefile test_RandomSeedSearch.py
@pytest.mark.parametrize("name", IrrationalSearchAgent.irrationals.keys())
 @pytest.mark.parametrize("offset", [0,1,2])
 def test_RandomSeedSearch_vs_named_irrational(name, offset):

 assert len(RandomSeedSearch.cache)
 episodeSteps = 100
results = evaluate(
 "rps",
[
IrrationalAgent(name=name, offset=offset, verbose=False),
RandomSeedSearch(verbose=False)
 ],
configuration={
"episodeSteps": episodeSteps,
"tieRewardThreshold": 1,
"actTimeout": 1000,
}, 
)
 results = np.array(results ).reshape(( -1,2))
assert len(results[ results == None ])== 0
assert(results[0][0] + episodeSteps/2.1)< results[0][1]
def test_RandomSeedSearch_vs_seeded_rng() :

 assert len(RandomSeedSearch.cache)
 episodeSteps = 100
results = evaluate(
 "rps",
[
random_agent_seeded,
RandomSeedSearch(verbose=False)
 ],
configuration={
"episodeSteps": episodeSteps,
"tieRewardThreshold": 1,
"actTimeout": 1000,
}, 
)
 assert results[0][0] < results[0][1], results
def test_RandomSeedSearch_vs_Irrational() :

 episodeSteps = 100
results = Parallel(-1 )(delayed(evaluate )(
 "rps",
[
IrrationalAgent(verbose=False),
RandomSeedSearch(verbose=False)
 ],
configuration={
"episodeSteps": episodeSteps,
"tieRewardThreshold": 1,
"actTimeout": 1000,
},
num_episodes=1 
)for _ in range(int(1000/episodeSteps)))
 results = np.array(results ).reshape(( -1,2))
assert len(results[ results == None ])== 0
totals = np.mean(results, axis=0)
 std = np.std(results, axis=0 ).round(2)
 winrate = [ np.sum(results[:,0] > results[:,1]),
np.sum(results[:,0] < results[:,1])]
print('results', results)
 print('totals', totals)
 print('std', std)
 print('winrate', winrate)
 assert np.abs(totals[0])< 0.2 * episodeSteps
assert np.abs(totals[1])< 0.2 * episodeSteps
assert np.abs(std[0])< 0.2 * episodeSteps
assert np.abs(std[1])< 0.2 * episodeSteps
def test_RandomSeedSearch_vs_unseeded_RNG() :

 episodeSteps = 100
results = Parallel(-1 )(delayed(evaluate )(
 "rps",
[
"rng/random_agent_unseeded.py",
RandomSeedSearch(verbose=False)
 ],
configuration={
"episodeSteps": episodeSteps,
"tieRewardThreshold": 1,
"actTimeout": 1000,
},
num_episodes=1 
)for _ in range(int(1000/episodeSteps)))
 results = np.array(results ).reshape(( -1,2))
assert len(results[ results == None ])== 0
totals = np.sum(results, axis=0)
 std = np.std(results, axis=0 ).round(2)
 winrate = [ np.sum(results[:,0] > results[:,1]),
np.sum(results[:,0] < results[:,1])]
print('results', results)
 print('totals', totals)
 print('std', std)
 print('winrate', winrate)
 assert winrate[0] <= winrate[1], winrate
assert totals[0] < totals[1], totals 
<install_modules>
arr_tmp = []
for i in data_test['Store']:
arr_tmp.append(float(data_store['CompetitionDistance'][data_store['Store'] == i]))
data_test['CompetitionDistance'] = arr_tmp
!pip install --upgrade -q kaggle_environments 2> /dev/null<install_modules>
arr_tmp = []
for i in data_train['Store']:
arr_tmp.append(float(data_store['CompetitionDistance'][data_store['Store'] == i]))
data_train['CompetitionDistance'] = arr_tmp
data_train['CompetitionDistance'] = data_train['CompetitionDistance'].fillna(data_train['CompetitionDistance'].mean() )
!pytest -v test_RandomSeedSearch.py<set_options>
train_stores = dict(list(data_train.groupby('Store')))
 test_stores = dict(list(data_test.groupby('Store')) )
env = make("rps", configuration={"episodeSteps": 10}, debug=True)
 env.run([ RandomSeedSearch(max_offset=1000), random_agent_seeded, ])
 env.render(mode="ipython", width=400, height=400 )<set_options>
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
env = make("rps", configuration={"episodeSteps": 20}, debug=True)
 env.run([ RandomSeedSearch(verbose=2, max_offset=1000), random_agent_unseeded, ])
 env.render(mode="ipython", width=400, height=400 )<set_options>
<set_options><EOS> 
result = pd.Series()
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ind = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 estimator = RandomForestRegressor(n_estimators=10, max_depth=13, criterion = 'mse')
 estimator.fit(X_train, Y_train)
 Y_pred = estimator.predict(X_test)
 result = result.append(pd.Series(Y_pred, index=store_ind))
result = result.append(pd.Series(0, index=closed_store_data))
result = pd.DataFrame({ "Id": result.index, "Sales": result.values})
 result.to_csv('result_new.csv', index=False )