Kaggle competition: Santa's Uncertain Bags
<SOS> metric: SantaWeightedBins Kaggle data source: santas-uncertain-bags<save_to_csv>
%matplotlib inline
df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 complete_test.to_csv('complete_test.csv',index=False)
<load_from_csv>
def sample_horse(size=1):
return np.maximum(0, np.random.normal(5,2,size))
def sample_ball(size=1):
return np.maximum(0, 1 + np.random.normal(1,0.3,size))
def sample_bike(size=1):
return np.maximum(0, np.random.normal(20,10,size))
def sample_train(size=1):
return np.maximum(0, np.random.normal(10,5,size))
def sample_coal(size=1):
return 47 * np.random.beta(0.5,0.5,size)
 def sample_book(size=1):
return np.random.chisquare(2,size)
 def sample_doll(size=1):
return np.random.gamma(5,1,size)
 def sample_block(size=1):
return np.random.triangular(5,10,20,size)
 def sample_gloves(size=1):
dist1 = 3.0 + np.random.rand(size)
 dist2 = np.random.rand(size)
 toggle = np.random.rand(size)< 0.3
dist2[toggle] = dist1[toggle]
return dist2
samplers = {
"horse": sample_horse,
"ball": sample_ball,
"bike": sample_bike,
"train": sample_train,
"coal": sample_coal,
"book": sample_book,
"doll": sample_doll,
"blocks": sample_block,
"gloves": sample_gloves
}
def sample(gift, quantity=1, size=1):
return np.sum(samplers[gift](quantity * size ).reshape(quantity, size), axis=0)
 print(sample("horse", 2, 10))
train_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )<feature_engineering>
def bag_name(bag):
return str(list(map(lambda gift: "{}({})".format(gift, bag[gift]), sorted(bag.keys()))))
def create_bag_weight_sampler(bag):
def bag_weight_sampler(size=1):
weight = np.array([0.0]*size)
 for gift in sorted(bag.keys()):
weight += sample(gift, bag[gift], size)
 return weight
return bag_weight_sampler, bag_name(bag)
 bag = { "horse": 1, "ball": 2 }
bag_weight_sampler, name = create_bag_weight_sampler(bag)
 print("Sampling from bag {}: {}".format(name, bag_weight_sampler(3)) )
for row in range(len(train_data['Province_State'])) :
if str(train_data['Province_State'][row])== 'nan':
train_data['Province_State'][row] = train_data['Country_Region'][row]<feature_engineering>
np.random.seed(42)
 def get_gift_weight_distributions(gifts, size=10000):
def get_gift_weight_dsitribution(gift):
sampler = samplers[gift]
sample = sampler(size)
 return np.mean(sample), np.var(sample)
 distributions = np.zeros(( len(gifts), 2))
for i, gift in enumerate(gifts):
distributions[i, :] = get_gift_weight_dsitribution(gift)
 return distributions
gifts = sorted(samplers.keys())
 print("Canonical gift order: {}
".format(gifts))
gift_weight_distributions = get_gift_weight_distributions(gifts)
 print(pd.DataFrame(data=gift_weight_distributions, index=gifts, columns=["mean", "std"]))
for row in range(len(test_data['Province_State'])) :
if str(test_data['Province_State'][row])== 'nan':
test_data['Province_State'][row] = test_data['Country_Region'][row]<define_variables>
def create_candidate_bags(max_quantities):
gift_counts = []
for max_quantity in max_quantities:
gift_counts.append(np.arange(max_quantity))
return cartesian(gift_counts)
 mixed_item_candiadte_bags = create_candidate_bags(mixed_item_max_quantities)
 print("Created candiadate bags: {}".format(mixed_item_candiadte_bags.shape))
day_number = train_data[train_data['Province_State'] == 'Florida'].shape[0]
day_number<define_variables>
def get_bag_weight_distributions(candidate_bags, min_mean=30, max_mean=50):
return np.dot(candidate_bags, gift_weight_distributions)
 def filter_by_mean(bags, distributions, min_mean=30, max_mean=50):
min_mask = mean_of(distributions)> min_mean
distributions = distributions[min_mask]
bags = bags[min_mask]
max_mask = mean_of(distributions)< max_mean
distributions = distributions[max_mask]
bags = bags[max_mask]
return bags, distributions
def mean_of(distributions):
return distributions[:,0]
mixed_item_bag_weight_distributions = get_bag_weight_distributions(mixed_item_candiadte_bags)
 mixed_item_candiadte_bags, mixed_item_bag_weight_distributions = \
filter_by_mean(mixed_item_candiadte_bags, mixed_item_bag_weight_distributions)
 print("Candidate bags left: {}".format(mixed_item_candiadte_bags.shape))
country_number = train_data.shape[0]/day_number
country_number<define_variables>
def get_low_weight_item_candidate_bags() :
bags = []
distributions = []
for gift in ["ball", "book", "gloves"]:
max_quantities = get_low_weight_item_bags_max_quantities_for(gift)
 candiadte_bags = create_candidate_bags(max_quantities)
 bags.append(candiadte_bags)
 cadidate_bag_weight_distributions = get_bag_weight_distributions(candiadte_bags)
 distributions.append(cadidate_bag_weight_distributions)
 return np.vstack(bags), np.vstack(distributions)
 def get_low_weight_item_bags_max_quantities_for(gift):
max_quantities = np.ceil(50 / gift_weight_distributions[:, 0])
 gift_index = gifts.index(gift)
 for i in range(len(max_quantities)) :
if not i == gift_index:
max_quantities[i] = 5
print("Gift {}: number of different bags: {}".format(gift, np.prod(max_quantities)))
 return max_quantities
low_weight_item_candidate_bags, low_weight_item_bag_weight_distributions = \
filter_by_mean(*get_low_weight_item_candidate_bags())
 print("Total number of canidate bags: {}".format(low_weight_item_candidate_bags.shape))
state_code = 0
train_data['Province_State'][0] = 0
for instance in range(0, int(country_number)) :
for state_index in range(0, day_number):
train_data['Province_State'][state_index +(day_number * instance)] = state_code
state_code += 1<filter>
def drop_duplicate(candidate_bags, distributions):
df = pd.DataFrame(data=np.hstack(( candidate_bags, distributions)) , columns=gifts + ["mean", "std"])
 df.drop_duplicates(subset=gifts, inplace=True)
 return df[gifts].values, df[["mean", "std"]].values
candidate_bags = np.vstack([mixed_item_candiadte_bags, low_weight_item_candidate_bags])
 bag_weight_distributions = np.vstack([mixed_item_bag_weight_distributions, low_weight_item_bag_weight_distributions])
 print("Combined candiadte bags: {}".format(candidate_bags.shape))
candidate_bags, bag_weight_distributions = drop_duplicate(candidate_bags, bag_weight_distributions)
 print("Final candidate bags without duplicates: {}".format(candidate_bags.shape))
train_data[train_data['Country_Region'] == 'Turkey']<filter>
def get_bag_utility_distributions(candidate_bags):
distributions = []
size = len(candidate_bags)
 for i, candidate_bag in enumerate(candidate_bags):
if i % 7000 == 0:
sys.stdout.write("{:.4f}\r".format(float(i)/ float(size)))
 distributions.append(get_bag_utility_distribution(candidate_bag))
print("")
 return np.vstack(distributions)
 def get_bag_utility_distribution(candidate_bag):
bag = { gifts[i]: int(candidate_bag[i])for i in range(len(gifts)) if candidate_bag[i] > 0 }
sampler, name = create_bag_utility_sampler(bag)
 sample = sampler(10000)
 return np.mean(sample), np.var(sample)
 bag_utility_distributions = get_bag_utility_distributions(candidate_bags)
 print(bag_utility_distributions.shape )
train_data[train_data['Country_Region'] == 'Turkey']['Date']<data_type_conversions>
num_gifts_available = {
"horse": 1000,
"ball": 1100,
"bike": 500,
"train": 1000,
"book": 1200,
"doll": 1000,
"blocks": 1000,
"gloves": 200,
"coal": 166
}
time1 = train_data[train_data['Country_Region'] == 'Turkey']['Date'].values[-1]
time2 = '2020-04-30'
FMT = '%Y-%m-%d'
time_diff = datetime.strptime(time2, FMT)- datetime.strptime(time1, FMT )<define_variables>
def pack_linprog(bags, distributions, min_variance, max_bags=1000):
c = - distributions[:,0]
A_ub = bags.T
b_ub = np.array([num_gifts_available[gift] for gift in gifts])
 A_ub = np.vstack([A_ub, np.ones(A_ub.shape[1])])
 b_ub = np.hstack([b_ub, [max_bags]])
 if min_variance is not None:
A_ub = np.vstack([A_ub, -distributions[:,1]])
 b_ub = np.hstack([b_ub, [-min_variance]])
 result = linprog(c, A_ub=A_ub, b_ub=b_ub)
 if result["success"] == False:
return [], True
else:
return result["x"].astype('int64'), False
def pack_bags(bags, distributions, min_variance=None):
max_bags = 1000
bag_quantities, infeasible = pack_linprog(bags, distributions, min_variance=min_variance)
 while np.sum(bag_quantities)< 1000:
max_bags += 1
bag_quantities, infeasible = pack_linprog(bags, distributions, min_variance=min_variance, max_bags=max_bags)
 if max_bags > 1015:
print("WARNING: not getting 1000 bags")
 break
if infeasible:
continue
if infeasible:
print("infeasible")
 return [], [], []
chosen_bag_idx = np.where(bag_quantities)[0]
chosen_bags = bags[chosen_bag_idx]
chosen_distributions = distributions[chosen_bag_idx]
chosen_quantities = bag_quantities[chosen_bag_idx]
while np.sum(chosen_quantities)> 1000:
idx = np.random.randint(len(chosen_quantities))
chosen_quantities[idx] = max(chosen_quantities[idx]-1, 0)
 score_distribution = np.dot(chosen_quantities, chosen_distributions)
 print("{} bags - score distribution: mean = {:.2f} | var = {:.2f}" 
.format(np.sum(chosen_quantities), score_distribution[0], score_distribution[1]))
return chosen_bags, chosen_distributions, chosen_quantities
packed_bags, packed_distributions, packed_quantities \
= pack_bags(candidate_bags, bag_utility_distributions, min_variance=None )
future_day_number = time_diff.days
future_day_number<drop_column>
def evaluate_variances() :
results = {}
for i, min_variance in enumerate(np.linspace(100000, 410000, num=10)) :
bags, distributions, quantities = pack_bags(candidate_bags, bag_utility_distributions, min_variance=min_variance)
 results[min_variance] = np.dot(quantities, distributions)
 return results
scores_for_min_variance = evaluate_variances()
<create_dataframe><EOS> 
def create_submissions(bags, quantities, num_submissions=60):
def create_stock(n):
stock = { gift: list(map(lambda id: "{}_{}".format(gift, id),np.arange(num_gifts_available[gift])))for gift in gifts }
return shuffle(stock, n)
 def shuffle(stock, seed):
np.random.seed(seed)
 for gift in stock.keys() :
np.random.shuffle(stock[gift])
 return stock
def generate_submission(n):
stock = create_stock(n)
 with open("submission_{}.csv".format(n), 'w+')as submission_file:
submission_file.write('Gifts
')
 for i in range(len(bags)) :
for quantity in range(quantities[i]):
current_gifts = bags[i]
for gift_idx, gift_quantity in enumerate(current_gifts[:len(gifts)]):
gift_name = gifts[gift_idx]
for j in range(int(gift_quantity)) :
submission_file.write("{} ".format(stock[gift_name].pop()))
submission_file.write("
")
 for n in range(num_submissions):
generate_submission(n)
 create_submissions(packed_bags, packed_quantities )