Kaggle competition: House Prices - Advanced Regression Techniques
turns = 30
n_drone_per_wh = 10 * turns<define_search_space>
train_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 train_data.head()
turns = [30, 30, 30, 100, 100, 100, 40, 40, 30]
def create_data_model(wh=0):

 n_drone_per_wh = n_drones//10
df = df_assign.query("warehouse == "+str(wh)).sort_values('order')
 order_list = list(df.order)
 demands = df['total_weight'].tolist() +[0]
products = df['product_id'].tolist() +[0]
len(demands)
 X = np.array(d_order['x'])
 Y = np.array(d_order['y'])
 ref_dist = np.array([[ distance(X[i], Y[i], X[j], Y[j])for j in range(n_order)] for i in range(n_order)])
 tmp = pd.DataFrame(np.array([[ref_dist[i][j] for j in order_list] for i in order_list]))
tmp[tmp.shape[0]] = pd.Series([df_order.iloc[i]['dist_'+str(wh)] for i in order_list]+[0])
 tmp = tmp.append(pd.Series([df_order.iloc[i]['dist_'+str(wh)] for i in order_list]+[0]), ignore_index=True)
 distance_ = np.array(tmp)
 turns = [30, 30, 30, 2000, 400, 100, 100, 100, 100,100]
turns = turns[wh]
n_drone_per_wh = 10 * turns
data = {}
data['distance_matrix'] = distance_
data['demands'] = demands
data['vehicle_capacities'] = [payload]*n_drone_per_wh
data['num_vehicles'] = n_drone_per_wh
data['depot'] = tmp.shape[0]-1
data['products'] = products
data['order_list'] = order_list+[0]
data['time_matrix'] = distance_
data['vehicle_load_time'] = 1
data['vehicle_unload_time'] = 1
data['time_windows'] = [(0, T)]*len(demands)
 data['depot_capacity'] = n_drone_per_wh
return data<create_dataframe>
test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 test_data.head()
%%time
a = create_data_model(3 )<find_best_params>
train_ID = train_data['Id']
test_ID = test_data['Id']
train_data.drop("Id", axis = 1, inplace = True)
 test_data.drop("Id", axis = 1, inplace = True )
def print_solution(data, manager, routing, solution):

 dropped_nodes = 'Dropped nodes:'
for node in range(routing.Size()):
if routing.IsStart(node)or routing.IsEnd(node):
continue
if solution.Value(routing.NextVar(node)) == node:
dropped_nodes += ' {}'.format(manager.IndexToNode(node))
print(len(dropped_nodes.split(' ')))
 print(dropped_nodes)
 total_distance = 0
total_load = 0
count = 0
for vehicle_id in range(data['num_vehicles']):
index = routing.Start(vehicle_id)
 plan_output = 'Route for vehicle {}:
'.format(vehicle_id)
 route_distance = 0
route_load = 0
while not routing.IsEnd(index):
node_index = manager.IndexToNode(index)
 route_load += data['demands'][node_index]
plan_output += ' {0} Load({1})-> '.format(node_index, route_load)
 previous_index = index
index = solution.Value(routing.NextVar(index))
route_distance += routing.GetArcCostForVehicle(
 previous_index, index, vehicle_id)
 plan_output += ' {0} Load({1})
 '.format(manager.IndexToNode(index),
route_load)
 plan_output += 'Distance of the route: {}m
'.format(route_distance)
 plan_output += 'Load of the route: {}
'.format(route_load)
 if route_load > 0:
print(plan_output)
 count += 1
total_distance += route_distance
total_load += route_load
print('Total needed rounds : {} '.format(count//10))
print('Total distance of all routes: {}m'.format(total_distance))
print('Total load of all routes: {}'.format(total_load))<choose_model_class>
ntrain = train_data.shape[0]
ntest = test_data.shape[0]
y_train = train_data.SalePrice.values
all_data = pd.concat(( train_data, test_data)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
def main(wh=0, print_solution=False):

 data = create_data_model(wh)
 manager = pywrapcp.RoutingIndexManager(len(data['distance_matrix']), data['num_vehicles'], data['depot'])
 routing = pywrapcp.RoutingModel(manager)
 def distance_callback(from_index, to_index):

 from_node = manager.IndexToNode(from_index)
 to_node = manager.IndexToNode(to_index)
 return data['distance_matrix'][from_node][to_node]
transit_callback_index = routing.RegisterTransitCallback(distance_callback)
 routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)
 def demand_callback(from_index):

 from_node = manager.IndexToNode(from_index)
 return data['demands'][from_node]
demand_callback_index = routing.RegisterUnaryTransitCallback(
 demand_callback)
 routing.AddDimensionWithVehicleCapacity(
 demand_callback_index,
0,
data['vehicle_capacities'],
True,
'Capacity')
 penalty = 15000
for node in range(1, len(data['distance_matrix'])) :
routing.AddDisjunction([manager.NodeToIndex(node)], penalty)
 search_parameters = pywrapcp.DefaultRoutingSearchParameters()
search_parameters.first_solution_strategy =(routing_enums_pb2.FirstSolutionStrategy.PATH_CHEAPEST_ARC)
 search_parameters.local_search_metaheuristic =(routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH)
 search_parameters.log_search = True
search_parameters.time_limit.FromSeconds(1200)
 solution = routing.SolveWithParameters(search_parameters)
 print("Solver status: ", routing.status())
 if print_solution:
if solution:
print_solution(data, manager, routing, solution)
 return(data, manager, routing, solution )<categorify>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
def main_df(wh=0): 
(data, manager, routing, solution)= main(wh, False)
 dropped_nodes = 'Dropped nodes:'
for node in range(routing.Size()):
if routing.IsStart(node)or routing.IsEnd(node):
continue
if solution.Value(routing.NextVar(node)) == node:
dropped_nodes += ' {}'.format(manager.IndexToNode(node))
print(len(dropped_nodes.split(' ')))
 print(dropped_nodes)
 dropped_nodes = 'Dropped nodes:'
for node in range(routing.Size()):
if routing.IsStart(node)or routing.IsEnd(node):
continue
if solution.Value(routing.NextVar(node)) == node:
dropped_nodes += ' {}'.format(manager.IndexToNode(node))
print(len(dropped_nodes.split(' ')))
 print(dropped_nodes)
 total_distance = 0
total_load = 0
count = 0
df_route = pd.DataFrame(columns=['warehouse', 'route', 'order', 'product', 'load'])
 for vehicle_id in range(data['num_vehicles']):
index = routing.Start(vehicle_id)
 plan_output = 'Route for vehicle {}:
'.format(vehicle_id)
 route_distance = 0
route_load = 0
while not routing.IsEnd(index):
node_index = manager.IndexToNode(index)
 route_load += data['demands'][node_index]
prod = data['products'][node_index]
plan_output += ' {0} Load({1} - product: {2})-> '.format(data['order_list'][node_index], route_load, prod)
 tmp_data = {'warehouse': wh,
'route': vehicle_id,
'product': prod,
'order': data['order_list'][node_index],
'load': route_load }
df_route = df_route.append(tmp_data,
ignore_index=True)
 previous_index = index
index = solution.Value(routing.NextVar(index))
route_distance += routing.GetArcCostForVehicle(previous_index, index, vehicle_id)
 '.format(node_index, route_load, prod)
 plan_output += 'Distance of the route: {}m
'.format(route_distance)
 plan_output += 'Load of the route: {}
'.format(route_load)
 if route_load > 0:
print(plan_output)
 count += 1
total_distance += route_distance
total_load += route_load
print('Total needed rounds : {} '.format(count//10))
print('Total distance of all routes: {}m'.format(total_distance))
print('Total load of all routes: {}'.format(total_load))
return df_route<create_dataframe>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 train_data = train_data.drop(['PoolQC','MiscFeature', 'Alley', 'Fence'] ,axis=1)
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 all_data["GarageFinish"] = all_data["GarageFinish"].fillna("None")
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None')
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data = all_data.drop(['Utilities'], axis=1)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
%%time
warehouses = [0, 1, 2, 5, 6, 7, 8, 9 ]
tmp_list = []
df_route = pd.DataFrame()
for wh in warehouses:
print("
Warehouse : ", wh)
 tmp = main_df(wh)
 tmp_list.append(tmp )<concatenate>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
df_route = pd.concat(tmp_list)
 order = 121
tmp = df_route.query('order == '+str(order))
for prod in order_quantity_items[order]:
assert np.isin(prod, tmp['product'])
 df_route.shape<train_model>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
df_route = df_route.query('load != 0')
 print("instruction for warehouse 1 : ", df_route.query("warehouse == "+str(1)).shape[0] )<feature_engineering>
corrmat['SalePrice'].sort_values(ascending=False )
df_route['qty'] = [1]*df_route.shape[0]
df_route.groupby(['warehouse', 'route', 'product'])['qty'].sum().reset_index().query('qty > 1 and product != 0' )<define_variables>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )
%%time
def submit(df_route):
instruction = ""
time = [T]*n_drones
fleet_per_wh = n_drones//n_warehouses
instruction = []
for wh in warehouses :
print("warehouse : ", wh)
 routes = set(df_route.query('warehouse == '+str(wh)).route)
 for turn in range(len(routes)) :
instruction_d = []
instruction_l = []
drone = fleet_per_wh*wh + turn%fleet_per_wh
r = turn
route = df_route.query("route == "+str(r)+" and warehouse == "+str(wh))
prec =(int(df_wh['x'][wh]), int(df_wh['y'][wh]))
for line in range(route.shape[0]):
if line > 0:
product = route.iloc[line]['product']
qty = 1
order = route.iloc[line]['order']
instruction_l.append("{0} L {1} {2} {3}
".format(drone, wh, product, qty))
time[drone] -= 1
instruction_d.append("{0} D {1} {2} {3}
".format(drone, order, product, qty))
time[drone] -= distance(prec[0], prec[1], df_order['x'][order], df_order['y'][order])
 prec =(df_order['x'][order], df_order['y'][order])
 time[drone] -= distance(prec[0], prec[1], df_wh['x'][wh], df_wh['y'][wh])
 instruction += instruction_l + instruction_d
print(time)
 return instruction<save_to_csv>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
file1 = open("submission.csv","w")
 file1.write(str(len(instruction)) +'
')
 for line in instruction:
file1.write(line)
 file1.close()<import_modules>
train_data = all_data[:ntrain]
test_data = all_data[ntrain:]
import pandas as pd
import numpy as np
from tqdm import tqdm<feature_engineering>
from sklearn.metrics import mean_squared_error
import xgboost as xgb
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
print('Extracting data')
 with open('/kaggle/input/hashcode-drone-delivery/busy_day.in')as file:
line_list = file.read().splitlines()
ROWS, COLS, DRONES, TURNS, MAXLOAD = map(int, line_list[0].split())
 weights = line_list[2].split()
products_df = pd.DataFrame({'weight': weights})
 wh_count = int(line_list[3])
 wh_endline =(wh_count*2)+4
wh_invs = line_list[5:wh_endline+1:2]
for i, wh_inv in enumerate(wh_invs):
products_df[f'wh{i}_inv'] = wh_inv.split()
products_df = products_df.astype(int)
 wh_locs = line_list[4:wh_endline:2]
wh_rows = [wl.split() [0] for wl in wh_locs]
wh_cols = [wl.split() [1] for wl in wh_locs]
warehouse_df = pd.DataFrame(
 {'wh_row': wh_rows, 'wh_col': wh_cols} ).astype(np.uint16)
 order_locs = line_list[wh_endline+1::3]
o_rows = [ol.split() [0] for ol in order_locs]
o_cols = [ol.split() [1] for ol in order_locs]
orders_df = pd.DataFrame({'row': o_rows, 'col': o_cols})
 orders_df[orders_df.duplicated(keep=False)].sort_values('row')
 orders_df['product_count'] = line_list[wh_endline+2::3]
order_array = np.zeros(( len(orders_df), len(products_df)) , dtype=np.uint16)
 orders = line_list[wh_endline+3::3]
for i,order in enumerate(orders):
products = [int(prod)for prod in order.split() ]
for p in products:
order_array[i, p] += 1
df = pd.DataFrame(data=order_array,
columns=['p_'+ str(i)for i in range(400)],
index=orders_df.index)
 orders_df = orders_df.astype(int ).join(df)
 print('...success' )<load_from_csv>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
submission = pd.read_csv('/kaggle/input/2020hcdd-sample-submission/submission.csv')
 allcommands = submission[submission.columns[0]].values
order_completion_times = np.full(( len(orders_df)) , -1)
 missing_items = orders_df.copy()
inventory_ops = pd.DataFrame(columns=['action', 'wh', 'item', 'count', 'turn'])
 for ddd in tqdm(range(DRONES)) :
dronecommands = [iii for iii in allcommands if iii.split() [0] == str(ddd)]
currentloc = warehouse_df.loc[0].values
currenttime = 0
currentweight = 0
for cmd in dronecommands:
_, action, locidx, prod, count = cmd.split(' ')
 if action == 'W':
currenttime += locidx
continue
if action == 'L' or action == 'U':
newloc = warehouse_df.loc[int(locidx)].values
elif action == 'D':
newloc = orders_df.loc[int(locidx), ['row', 'col']].values
dist = int(np.ceil(np.sqrt(np.sum(( currentloc-newloc)**2))))
currenttime += dist
currenttime += 1
if currenttime > TURNS:
raise Exception('Maximum simulation time exceeded')
 currentloc = np.copy(newloc)
 if action == 'L':
currentweight += int(count)* products_df.loc[int(prod),'weight']
elif action == 'D' or action == 'U':
currentweight -= int(count)* products_df.loc[int(prod),'weight']
if currentweight > MAXLOAD:
raise Exception('Maximum drone load exceeded')
 if action == 'D':
if missing_items.at[int(locidx), 'p_{}'.format(prod)] >= int(count):
missing_items.at[int(locidx), 'p_{}'.format(prod)] -= int(count)
 if currenttime > order_completion_times[int(locidx)]:
order_completion_times[int(locidx)] = currenttime
else:
raise Exception('Too many items delivered')
 if action == 'L' or action == 'U':
inventory_ops = inventory_ops.append({
'action': action,
'wh': int(locidx),
'item': int(prod),
'count': int(count),
'turn': currenttime
}, ignore_index=True )<sort_values>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
model_xgb.fit(train_data, y_train)
 xgb_train_pred = model_xgb.predict(train_data)
 xgb_pred = np.expm1(model_xgb.predict(test_data))
print(rmsle(y_train, xgb_train_pred))
for wh in range(len(warehouse_df)) :
for item in tqdm(range(len(products_df))):
tmp = inventory_ops[ 
(inventory_ops['wh'] == wh)& 
(inventory_ops['item'] == item)
 ]
if not len(tmp):
continue
tmp = tmp.sort_values(by='turn')
 inv = products_df.loc[item, f'wh{wh}_inv']
if len(tmp[tmp['action'] == 'L'])<= inv:
continue
for iii in tmp.index:
if tmp.loc[iii, 'action'] == 'L':
inv -= tmp.loc[iii, 'count']
else:
inv += tmp.loc[iii, 'count']
if inv < 0:
raise Exception('Removal of unstocked product attempted' )<compute_test_metric>
scores = cross_val_score(model_xgb, train_data, y_train, scoring='neg_mean_squared_error', cv=10)
 scores
completed = np.max(missing_items.iloc[:,3:].values > 0, axis=1)<= 0
print('Orders completed:', len(np.where(completed)[0]))
print('Orders not completed:', len(np.where(completed == False)[0]))
order_scores = np.ceil(100 *(TURNS - order_completion_times[np.where(completed)[0]])/ TURNS)
 print('Score:', int(np.sum(order_scores)) )<load_pretrained>
output = pd.DataFrame({'Id': test_ID,
'SalePrice': xgb_pred})
 output.to_csv('submission.csv', index=False )