Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_from_csv>
print('torch version:', torch.__version__)
 warnings.filterwarnings('ignore' )
def optimize_deliveries(input_file, submission_name):
_, _, num_drones, max_time, max_cargo, wh_list, order_list = read_file(input_file)
 drones = []
first_wh = wh_list[0]
for i in range(num_drones):
drones.append(Drone(id=i, position=first_wh.position, max_capacity=max_cargo))
simulation = Simulation(
 max_time=max_time, drones=drones, warehouses=wh_list, orders=order_list 
)
 simulation.run()
write_submission(simulation.commands, submission_name)
 optimize_deliveries(".. /input/hashcode-drone-delivery/busy_day.in", "submission.csv" )<categorify>
DATA_DIR = '.. /input/bengaliai-cv19'
SUBMISSION_CSV_FILE = 'submission.csv'
MYFILE_DIR = '.. /input/myfile'
def check_submission(input_file, submission_file):
commands, num_drones, max_time, max_cargo, wh_list, order_list = read_submission(
 input_file, submission_file 
)
 drone_to_position = {c.drone: c.drone.position for c in commands}
if len(drone_to_position)> num_drones:
print("Too many drones.")
 return 0
drone_to_delivery_time: Dict[Drone, int] = defaultdict(int)
 order_to_delivery_time: Dict[Order, list] = defaultdict(list)
 score = 0
for i, command in enumerate(commands):
drone = command.drone
product = command.product
quantity = command.quantity
basket = {product: quantity}
if isinstance(command.destination, Warehouse):
warehouse = command.destination
if warehouse.products.get(product, 0)< quantity:
print(f"Command {i}: {warehouse} have not enough {product}.")
 return 0
warehouse.remove_products(basket)
 drone.add_products(basket)
 if drone.product_weight > max_cargo:
print(f"Command {i}: {drone} overloaded.")
 return 0
drone_to_delivery_time[drone] += drone.distance_from(warehouse)+ 1
drone.set_position(warehouse.position)
 elif isinstance(command.destination, Order):
order = command.destination
if order.is_complete() :
print(
 f"Command {i}: the {order} is closed, nothing can be delivered there." 
)
 return 0
if drone.products.get(product, 0)< quantity:
print(f"Command {i}: {drone} have not enough {product}.")
 return 0
drone.remove_products(basket)
 if order.products.get(product, 0)< quantity:
print(f"Command {i}: Too many {product} for {order}.")
 return 0
order.remove_products(basket)
 drone_to_delivery_time[drone] += drone.distance_from(order)+ 1
drone.set_position(order.position)
 order_to_delivery_time[order].append(drone_to_delivery_time[drone])
 if order.is_complete() :
delivery_time = max(order_to_delivery_time[order])
 if delivery_time < max_time:
score += ceil(100 *(max_time - delivery_time)/ max_time)
 else:
print(f"Command {i}: Run out of time.")
 return 0
else:
print(f"Command {i}: Unknown destination {command.destination}.")
 return 0
return score
check_submission(".. /input/hashcode-drone-delivery/busy_day.in", "submission.csv" )<feature_engineering>
def do_predict(net, input):
def logit_to_probability(logit):
probability=[]
for l in logit:
p = F.softmax(l,1)
 probability.append(p)
 return probability
num_ensemble = len(net)
 for i in range(num_ensemble):
net[i].eval()
probability=[0,0,0]
for i in range(num_ensemble):
logit = net[i](input)
 prob = logit_to_probability(logit)
 probability = [p+q for p,q in zip(probability,prob)]
probability = [p/num_ensemble for p in probability]
predict = [torch.argmax(p,-1)for p in probability]
predict = [p.data.cpu().numpy() for p in predict]
predict = np.array(predict ).T
predict = predict.reshape(-1)
 return predict
def calculate_score(commands: List[Command], max_time: int)-> int:
command_to_time = get_command_times(commands)
 order_to_times = defaultdict(list)
 for c, time in command_to_time.items() :
if isinstance(c.destination, Order):
order_to_times[c.destination].append(time)
 order_to_score = {}
for o, ts in order_to_times.items() :
order_to_score[o] = ceil(100 *(max_time - max(ts)) / max_time)
 return sum(order_to_score.values())
 def get_command_times(commands: List[Command])-> Dict[Command, int]:
command_to_time = {}
drone_to_delivery_time = defaultdict(int)
 drone_to_position = {}
for c in commands:
drone = c.drone
destination = c.destination
previous_position = drone_to_position.get(drone, drone.position)
 drone_to_delivery_time[drone] +=(
 previous_position.distance_from(destination.position)+ 1 
)
 drone_to_position[drone] = destination.position
command_to_time[c] = drone_to_delivery_time[drone]
return command_to_time
def get_execution_time(commands: List[Command])-> int:
drone_to_time = defaultdict(int)
 drone_to_position = {}
for c in commands:
drone = c.drone
destination = c.destination
previous_position = drone_to_position.get(drone, drone.position)
 drone_to_time[drone] +=(
 previous_position.distance_from(destination.position)+ 1 
)
 drone_to_position[drone] = destination.position
return max(drone_to_time.values())
 class DroneRoute:
def __init__(
 self,
start_point: Union[Order, Warehouse],
end_point: Union[Order, Warehouse],
warehouses: List[Command],
orders: List[Command], 
):
self.start_point = start_point
self.end_point = end_point
self.warehouses = warehouses
self.orders = orders
def _get_time(self, orders: List[Order]):
time = 0
previous_point = self.start_point
for order in orders:
time += order.distance_from(previous_point)
 previous_point = order
time += previous_point.distance_from(self.end_point)
 return time
def optimize(self):
if len(self.orders)< 2:
return self.orders
min_time, optimal_order = None, None
for p in itertools.permutations(self.orders):
time = self._get_time([x.destination for x in p])
 if min_time is None or time < min_time:
min_time, optimal_order = time, p
return optimal_order
def swap_orders_in_routes(commands: List[Command])-> List[Command]:
all_drones = {c.drone for c in commands}
new_commands = []
for drone in all_drones:
drone_commands = [c for c in commands if c.drone == drone]
start_point, end_point, orders = None, None, []
for i, c in enumerate(drone_commands):
if isinstance(c.destination, Order):
orders.append(c)
 if i < len(drone_commands)- 1:
continue
if len(orders)== 0:
new_commands.append(c)
 continue
if start_point is None:
start_point = c
end_point = c
route = DroneRoute(
 start_point=start_point.destination,
end_point=end_point.destination,
warehouses=[],
orders=orders, 
)
 new_commands += route.optimize()
start_point = c
orders = []
if isinstance(end_point.destination, Warehouse):
new_commands.append(end_point)
 return new_commands
def time_order_swap(commands: List[Command], max_time: int)-> List[Command]:
previous_score = calculate_score(commands, max_time)
 drone_to_commands = defaultdict(list)
 for c in commands:
drone_to_commands[c.drone].append(c)
 bar = tqdm(total=int(len(commands)** 2 / 2), desc="swap orders")
 for i, j in itertools.product(range(len(commands)) , repeat=2):
if i > j:
continue
bar.update(1)
 c1, c2 = commands[i], commands[j]
if isinstance(c1.destination, Warehouse)or isinstance(
 c2.destination, Warehouse 
):
continue
if c1.product != c2.product or c1.quantity != c2.quantity:
continue
d1, d2 = c1.drone, c2.drone
t = get_execution_time(drone_to_commands[d1])+ get_execution_time(
 drone_to_commands[d2] 
)
 c1.destination, c2.destination = c2.destination, c1.destination
new_t = get_execution_time(drone_to_commands[d1])+ get_execution_time(
 drone_to_commands[d2] 
)
 if new_t >= t:
c1.destination, c2.destination = c2.destination, c1.destination
else:
score = calculate_score(commands, max_time)
 if score < previous_score:
c1.destination, c2.destination = c2.destination, c1.destination
else:
previous_score = score
bar.close()
return commands<load_from_csv>
DESENET_CHECKPOINT_FILE = [
'.. /input/hengmodels/swa_fold1_no_bn_model.pth',
]
TASK_NAME = [ 'grapheme_root', 'vowel_diacritic', 'consonant_diacritic' ]
def post_processing(input_file, submission_file):
commands, _, max_time, _, _, _ = read_submission(input_file, submission_file)
 print(calculate_score(commands, max_time))
commands = swap_orders_in_routes(commands)
 print(calculate_score(commands, max_time))
commands = time_order_swap(commands, max_time)
 print(calculate_score(commands, max_time))
write_submission(commands, submission_file)
 post_processing(".. /input/hashcode-drone-delivery/busy_day.in", "submission.csv" )<load_from_csv>
net = []
for checkpoint_file in DESENET_CHECKPOINT_FILE:
n = DenseNet().cuda()
n.load_state_dict(torch.load(checkpoint_file, map_location=lambda storage, loc: storage),strict=True)
 net.append(n )
check_submission(".. /input/hashcode-drone-delivery/busy_day.in", "submission.csv" )<import_modules>
def run_make_submission_csv() :
row_id=[]
target=[]
batch_size= 32
print('
start here !!!!')
 for i in range(4):
start_timer = timer()
df = pd.read_parquet(DATA_DIR+'/test_image_data_%d.parquet'%i, engine='pyarrow')
 print('pd.read_parquet() = %s'%(time_to_str(( timer() - start_timer),'sec')))
 start_timer = timer()
num_test = len(df)
 for b in range(0,num_test,batch_size):
if b%1000==0:
print('test_image_data_%d.parquet @%06d, %s'%(i,b,time_to_str(( timer() - start_timer),'sec')))
 B = min(num_test,b+batch_size)-b
image = df.iloc[b:b+B, range(1,32332+1)].values
image_id = df.iloc[b:b+B, 0].values
image = image.reshape(B,1,137, 236)
 image = np.tile(image,(1,3,1,1))
image = image.astype(np.float32)/255
input = torch.from_numpy(image ).float().cuda()
predict = do_predict(net, input)
 image_id = np.tile(image_id.reshape(B,1),(1,3,)) + ['_'] + TASK_NAME
image_id = image_id.reshape(-1)
 row_id.append(image_id)
 target.append(predict)
 print('')
 row_id = np.concatenate(row_id)
 target = np.concatenate(target)
 df = pd.DataFrame(zip(row_id, target), columns=['row_id', 'target'])
 df.to_csv(SUBMISSION_CSV_FILE, index=False)
<import_modules><EOS> 
run_make_submission_csv()