Kaggle competition: House Prices - Advanced Regression Techniques
shutil.copyfile('/kaggle/input/2020hcdd-sample-submission/submission.csv', 'submission.csv' )<install_modules>
warnings.filterwarnings("ignore")
!tar xfz.. /input/nfl-lib/pkgs.tgz 
<set_options>
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_sub=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
warnings.filterwarnings("ignore" )<load_from_csv>
corr_dict=correlation_train['SalePrice'].sort_values(ascending=False ).to_dict()
important_columns=[]
for key,value in corr_dict.items() :
if(( value>0.1)&(value<0.8)) |(value<=-0.1):
important_columns.append(key)
 important_columns
d = pd.read_csv('.. /input/nfl-impact-detection/test_player_tracking.csv')
 IS_PRIVATE = d.shape !=(19269, 12)
 IS_PRIVATE = True 
<define_variables>
train_test=pd.concat([train,test],axis=0,sort=False)
 train_test.head()
DATA_ROOT_PATH = 'test_images'<import_modules>
pd.set_option('display.max_rows', 5000)
 train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_tes_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_tes_info
import argparse
from utils.datasets import *
from utils.general import non_max_suppression
from utils.general import *
from utils import torch_utils
from utils.plots import plot_one_box<init_hyperparams>
train_test.loc[train_test['Fireplaces']==0,'FireplaceQu']='Nothing'
train_test['LotFrontage'] = train_test['LotFrontage'].fillna(train_test.groupby('1stFlrSF')['LotFrontage'].transform('mean'))
train_test['LotFrontage'].interpolate(method='linear',inplace=True)
 train_test['LotFrontage']=train_test['LotFrontage'].astype(int)
 train_test['MasVnrArea'] = train_test['MasVnrArea'].fillna(train_test.groupby('MasVnrType')['MasVnrArea'].transform('mean'))
train_test['MasVnrArea'].interpolate(method='linear',inplace=True)
 train_test['MasVnrArea']=train_test['MasVnrArea'].astype(int)
 train_test["Fence"] = train_test["Fence"].fillna("None")
 train_test["FireplaceQu"] = train_test["FireplaceQu"].fillna("None")
 train_test["Alley"] = train_test["Alley"].fillna("None")
 train_test["PoolQC"] = train_test["PoolQC"].fillna("None")
 train_test["MiscFeature"] = train_test["MiscFeature"].fillna("None")
 train_test.loc[train_test['BsmtFinSF1']==0,'BsmtFinType1']='Unf'
train_test.loc[train_test['BsmtFinSF2']==0,'BsmtQual']='TA'
train_test['YrBltRmd']=train_test['YearBuilt']+train_test['YearRemodAdd']
train_test['Total_Square_Feet'] =(train_test['BsmtFinSF1'] + train_test['BsmtFinSF2'] + train_test['1stFlrSF'] + train_test['2ndFlrSF'] + train_test['TotalBsmtSF'])
 train_test['Total_Bath'] =(train_test['FullBath'] +(0.5 * train_test['HalfBath'])+ train_test['BsmtFullBath'] +(0.5 * train_test['BsmtHalfBath']))
train_test['Total_Porch_Area'] =(train_test['OpenPorchSF'] + train_test['3SsnPorch'] + train_test['EnclosedPorch'] + train_test['ScreenPorch'] + train_test['WoodDeckSF'])
 train_test['exists_pool'] = train_test['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_garage'] = train_test['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_fireplace'] = train_test['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 train_test['exists_bsmt'] = train_test['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 train_test['old_house'] = train_test['YearBuilt'].apply(lambda x: 1 if x <1990 else 0)
 for i in train_test.columns:
if 'SalePrice' not in i:
if 'object' in str(train_test[str(i)].dtype):
train_test[str(i)]=train_test[str(i)].fillna(method='ffill' )
names = ['0','1','2']
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = 'test_images/'
device = torch_utils.select_device(opt.device)
 half = False
model = torch.load(weights, map_location=device)['model'].to(device ).float().eval()
dataset = LoadImages(source, img_size=opt.img_size)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
all_c = []
for path, img, im0s, vid_cap in tqdm(dataset):
print(im0s.shape)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
c_2 = []
if True:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
flag=False
bboxes = []
score = []
cc = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf.cpu().numpy())
 cc.append(cls.cpu().numpy())
 bboxes_2.append(bboxes)
 score_2.append(score)
 c_2.append(cc)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 all_c.append(c_2)
 return all_path,all_score,all_bboxex,all_c
if __name__ == '__main__':
class opt:
weights = ".. /input/yolov5temp1280/best(2 ).pt"
img_size = 1280
conf_thres = 0.1
iou_thres = 0.3
augment = False
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 print(opt)
 with torch.no_grad() :
res = detect()<categorify>
columns =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual',
'GarageCond', 'ExterQual', 'ExterCond','HeatingQC', 'PoolQC',
'KitchenQual', 'BsmtFinType1', 'YrSold', 'MoSold', 'BsmtFinType2',
'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond')
 for col in columns:
lbl_enc = LabelEncoder()
lbl_enc.fit(list(train_test[col].values))
train_test[col] = lbl_enc.transform(list(train_test[col].values))
%matplotlib inline
all_path,all_score,all_bboxex,all_labels = res
results =[]
results_boxes =[]
results_scores = []
results_labels = []
result_image_ids =[]
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1]
boxes = np.array(all_bboxex[row])[0]
scores = np.array(all_score[row])[0]
labels = np.array(all_labels[row])[0]
if len(boxes)==0:
continue
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes.astype(int)
 if len(boxes)>0:
result_image_ids += [image_id]*len(boxes)
 results_boxes.append(boxes)
 results_scores.append(scores)
 results_labels.append(labels)
 idx = row
idx = 0
size = 300
idx =-1
font = cv2.FONT_HERSHEY_SIMPLEX
image = image = cv2.imread(all_path[idx], cv2.IMREAD_COLOR)
 fontScale = 1
boxes = results_boxes[idx]
scores = results_scores[idx]
label = results_labels[idx]
color =(255, 0, 0)
 thickness = 2
for b,s,l in zip(boxes,scores,label):
color =(255,0,0)if int(l)==2 else(0,0,255)
 image = cv2.rectangle(image,(b[0],b[1]),(b[0]+b[2],b[1]+b[3]), color, 1)
 image = cv2.putText(image, '{:.2}'.format(s),(int(b[0]),int(b[1])) , font, fontScale, color, thickness, cv2.LINE_AA)
 plt.figure(figsize=[20,20])
 plt.imshow(image[:,:,::-1])
 plt.show()<create_dataframe>
numeric_features = train_test.dtypes[train_test.dtypes != "object"].index
skewed_features = train_test[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 print(skewed_features)
box_df = pd.DataFrame(np.concatenate(results_boxes), columns=['left', 'top', 'width', 'height'])
 score_df = pd.DataFrame({'scores':np.concatenate(results_scores),'labels':np.concatenate(results_labels), 'image_name':result_image_ids})
 test_df = pd.concat([box_df, score_df], axis=1 )<data_type_conversions>
high_skewness = skewed_features[abs(skewed_features)> 0.9]
skewed_features = high_skewness.index
test_df['gameKey'] = test_df.image_name.str.split('_' ).str[0].astype(int)
 test_df['playID'] = test_df.image_name.str.split('_' ).str[1].astype(int)
 test_df['view'] = test_df.image_name.str.split('_' ).str[2]
test_df['frame'] = test_df.image_name.str.split('_' ).str[3].str.replace('.png','' ).astype(int)
 test_df['video'] = test_df.image_name.str.rsplit('_',1 ).str[0] + '.mp4' 
<save_to_csv>
for feature in skewed_features:
train_test[feature] = boxcox1p(train_test[feature], boxcox_normmax(train_test[feature] + 1))
test_df.to_csv('off.csv',index=False )<import_modules>
train_test=pd.get_dummies(train_test,dtype='int8' )
import numpy as np
import pandas as pd
from scipy.optimize import linear_sum_assignment<define_variables>
train_test_null_info=pd.DataFrame(train_test.isnull().sum() ,columns=['Count of NaN'])
 train_test_dtype_info=pd.DataFrame(train_test.dtypes,columns=['DataTypes'])
 train_test_info=pd.concat([train_test_null_info,train_test_dtype_info],axis=1)
 train_test_info
SEED = 42
FOLDS = 5
FOLD = 2
EFF_DET = 5
Threshold = 0.45
LOOK_BACKWARD_LIMIT = 10
IOU_THRESHOLD = 0.4
IMAGE_SIZE = 512
NUM_CLASSES = 2
BATCH_SIZE = 16
FRAME_THRESHOLD = 120
keep_columns_threshold = 20<compute_test_metric>
train=train_test[0:1460]
test=train_test[1460:2919]
def iou(bbox1, bbox2):
bbox1 = [float(x)for x in bbox1]
bbox2 = [float(x)for x in bbox2] 
(x0_1, y0_1, x1_1, y1_1)= bbox1 
(x0_2, y0_2, x1_2, y1_2)= bbox2
overlap_x0 = max(x0_1, x0_2)
 overlap_y0 = max(y0_1, y0_2)
 overlap_x1 = min(x1_1, x1_2)
 overlap_y1 = min(y1_1, y1_2)
 if overlap_x1 - overlap_x0 <= 0 or overlap_y1 - overlap_y0 <= 0:
return 0
size_1 =(x1_1 - x0_1)*(y1_1 - y0_1)
 size_2 =(x1_2 - x0_2)*(y1_2 - y0_2)
 size_intersection =(overlap_x1 - overlap_x0)*(overlap_y1 - overlap_y0)
 size_union = size_1 + size_2 - size_intersection
return size_intersection / size_union
def precision_calc(gt_boxes, pred_boxes):
cost_matix = np.ones(( len(gt_boxes), len(pred_boxes)))
 for i, box1 in enumerate(gt_boxes):
for j, box2 in enumerate(pred_boxes):
dist = abs(box1[0]-box2[0])
 if dist > 4:
continue
iou_score = iou(box1[1:], box2[1:])
 if iou_score < 0.35:
continue
else:
cost_matix[i,j]=0
row_ind, col_ind = linear_sum_assignment(cost_matix)
 fn = len(gt_boxes)- row_ind.shape[0]
fp = len(pred_boxes)- col_ind.shape[0]
tp=0
for i, j in zip(row_ind, col_ind):
if cost_matix[i,j]==0:
tp+=1
else:
fp+=1
fn+=1
return tp, fp, fn<remove_duplicates>
corr_dict2=corr_new_train['SalePrice'].sort_values(ascending=False ).to_dict()
corr_dict2
def filter_duplicates_and_threshols(row):
boxes,scores,labels = row.box_predictions,row.score_list,row.label_list
filtered_boxes = []
filtered_scores = []
filtered_labels = []
for box,score,label in zip(boxes,scores,labels):
if score> Threshold:
add_box = True
for i,ref_box in enumerate(filtered_boxes):
if iou(box,ref_box)>0.5:
add_box = False
if label==2:filtered_labels[i]=2
if add_box:
filtered_boxes.append(box)
 filtered_scores.append(score)
 filtered_labels.append(label)
 row.box_predictions,row.score_list,row.label_list = filtered_boxes,filtered_scores,filtered_labels
return row
def filter_duplicates_and_low_scores(row):
boxes,scores,labels = row.box_predictions,row.score_list,row.label_list
ordered_sample_predictions = sorted(list(zip(boxes,scores,labels)) , key = lambda x:x[1],
reverse = True)
 filtered_boxes = []
filtered_scores = []
filtered_labels = []
for box,score,label in ordered_sample_predictions:
if score>Threshold:
add_box = True
for i,ref_box in enumerate(filtered_boxes):
if iou(box,ref_box)>0.5:
add_box = False
if label==2:filtered_labels[i]=2
if add_box:
filtered_boxes.append(box)
 filtered_scores.append(score)
 filtered_labels.append(label)
 row.box_predictions,row.score_list,row.label_list = filtered_boxes,filtered_scores,filtered_labels
return row<load_from_csv>
best_columns=[]
for key,value in corr_dict2.items() :
if(( value>=0.3175)&(value<0.9)) |(value<=-0.315):
best_columns.append(key)
 best_columns
video_labels = pd.read_csv('off.csv' )<feature_engineering>
train['SalePrice_Log1p'] = np.log1p(train.SalePrice)
video_labels = video_labels.sort_values(by=['video','frame'] ).reset_index(drop=True)
 video_labels = video_labels[video_labels.frame>0]
video_labels['image'] = video_labels['video'].str.replace('.mp4', '')+ '_' + video_labels['frame'].astype(str)+ '.png'
video_labels['right'] = video_labels['left'] + video_labels['width']
video_labels['bottom'] = video_labels['top'] + video_labels['height']
video_labels['label_list'] = video_labels['labels'].apply(lambda x:[x])
 video_labels['bbox'] = pd.Series(video_labels[['left','top','right','bottom']].values.tolist() , index=video_labels.index)
 video_labels['box_predictions'] = video_labels['bbox'].apply(lambda x:[x])
 video_labels['score_list'] = video_labels['scores'].apply(lambda x:[x])
 base_cols = ['video','image','gameKey','frame']
target_cols = ['box_predictions','label_list','score_list']
predictions = video_labels[base_cols+target_cols].groupby(base_cols ).sum().reset_index()
predictions = predictions.apply(filter_duplicates_and_low_scores,axis=1)
 predictions['n_impact'] = predictions.label_list.apply(lambda x:(np.array(x)==2 ).sum())
 predictions.groupby('video' ).n_impact.sum().describe()<compute_test_metric>
dbscan = DBSCAN(eps=1400, min_samples=20 ).fit(train_pca)
 core_samples_mask = np.zeros_like(dbscan.labels_, dtype=bool)
 core_samples_mask[dbscan.core_sample_indices_] = True
labels=dbscan.labels_
n_clusters_ = len(set(labels)) -(1 if -1 in labels else 0 )
def iou(bbox1, bbox2):
bbox1 = [float(x)for x in bbox1]
bbox2 = [float(x)for x in bbox2] 
(x0_1, y0_1, x1_1, y1_1)= bbox1 
(x0_2, y0_2, x1_2, y1_2)= bbox2
overlap_x0 = max(x0_1, x0_2)
 overlap_y0 = max(y0_1, y0_2)
 overlap_x1 = min(x1_1, x1_2)
 overlap_y1 = min(y1_1, y1_2)
 if overlap_x1 - overlap_x0 <= 0 or overlap_y1 - overlap_y0 <= 0:
return 0
size_1 =(x1_1 - x0_1)*(y1_1 - y0_1)
 size_2 =(x1_2 - x0_2)*(y1_2 - y0_2)
 size_intersection =(overlap_x1 - overlap_x0)*(overlap_y1 - overlap_y0)
 size_union = size_1 + size_2 - size_intersection
return size_intersection / size_union
def match_boxes(bbox_set1,bbox_set2,diagonal=True,iou_threshold=0.35):
all_pairs = {}
relation = [None for x in bbox_set2]
done1 = {}
done2 = {}
for i,bbox1 in enumerate(bbox_set1):
for j,bbox2 in enumerate(bbox_set2):
if i!=j or diagonal:
all_pairs[(i,j)] = iou(bbox1,bbox2)
 for(index1,index2),iou_score in sorted(all_pairs.items() , key=lambda item: item[1],reverse=True):
if iou_score<iou_threshold: return relation
if(index1 not in done1)and(index2 not in done2):
relation[index2] = index1
done1[index1] = 1
done2[index2] = 1
return relation
def map_classes(row):
mapping = {}
for box,pred in zip(row.box_predictions,row.label_list):
mapping[tuple(box)] = pred
return mapping
def assign_players(df,video,LOOK_BACKWARD_LIMIT=LOOK_BACKWARD_LIMIT,IOU_THRESHOLD=IOU_THRESHOLD):
video_df = df[df.video==video].sort_values(by='frame' ).set_index('frame')
 row_to_class_map = video_df['row_to_class_map']
row_to_class_map.apply(lambda x:x.update({() :-1}))
player_to_bbox = pd.DataFrame()
bbox_to_player = {k+1:{} for k in range(video_df.shape[0])}
N_PLAYERS = 0
bboxes = video_df.loc[1,'box_predictions']
for player_id,box in enumerate(bboxes):
player_to_bbox[player_id] = None
player_to_bbox.at[1,player_id] = box
bbox_to_player[1][tuple(box)] = player_id
N_PLAYERS += 1
for frame in range(2,video_df.shape[0]+1):
player_to_bbox.loc[frame] = None
bboxes = video_df.loc[frame,'box_predictions']
assigned = {}
look_backward = 1
while(look_backward<min(frame,LOOK_BACKWARD_LIMIT+1)and len(bboxes)>0):
ref_bbox = []
for box in video_df.loc[frame-look_backward,'box_predictions']:
if bbox_to_player[frame-look_backward][tuple(box)] not in assigned:
ref_bbox.append(box)
 relation = match_boxes(ref_bbox,bboxes,iou_threshold=IOU_THRESHOLD)
 left_out_bboxes = []
for index,ref_index in enumerate(relation):
if ref_index is not None:
player_id = bbox_to_player[frame-look_backward][tuple(ref_bbox[ref_index])]
box = bboxes[index]
player_to_bbox.at[frame,player_id] = box
bbox_to_player[frame][tuple(box)] = player_id
assigned[player_id] = True
else:
left_out_bboxes.append(bboxes[index])
 bboxes = left_out_bboxes
look_backward +=1
for box in bboxes:
player_id = N_PLAYERS
player_to_bbox[player_id] = None
player_to_bbox.at[frame,player_id] = box
bbox_to_player[frame][tuple(box)] = player_id
N_PLAYERS += 1
player_to_class_map = player_to_bbox.apply(lambda row: row.fillna('' ).apply(tuple ).apply(
 lambda x:row_to_class_map.loc[row.name][x]),axis=1)
 return player_to_bbox,player_to_class_map
def filter_impacts(player_series):
impacted_player_series = pd.DataFrame(player_series.loc[player_series==2])
 impacted_player_series['indices'] = impacted_player_series.index
start_indices = impacted_player_series[impacted_player_series.indices.diff(1 ).fillna(99 ).abs() >9].indices.tolist()
end_indices = impacted_player_series[impacted_player_series.indices.diff(-1 ).fillna(99 ).abs() >9].indices.tolist()
output_series = player_series.copy(deep=True)
 output_series[player_series>0] = 1
output_series[player_series<0] = -1
for start_index,end_index in zip(start_indices,end_indices):
difference = end_index-start_index+1
n_medians = math.ceil(difference/9)
 difference_split = difference/(n_medians+1)
 centres = [int(start_index+(i+1)*difference_split)for i in range(n_medians)]
output_series.loc[centres] = 2
return output_series<prepare_output>
labels=pd.DataFrame(labels,columns=['Classes'])
 print(labels[labels['Classes']==-1] )
submission_df = pd.DataFrame(columns=['gameKey','playID','view','video','frame','left','width','top','height'])
 predictions['row_to_class_map'] = predictions.apply(map_classes,axis=1)
 entries = 0
for video in predictions.video.unique() :
assigned_players_boxes,assigned_players_classes = assign_players(predictions,video)
 keep_columns = assigned_players_classes.columns[(assigned_players_classes!=-1 ).sum() >keep_columns_threshold]
assigned_players_boxes = assigned_players_boxes[keep_columns]
assigned_players_classes = assigned_players_classes[keep_columns]
print(assigned_players_boxes.shape)
 assigned_players_classes_filtered = assigned_players_classes.apply(filter_impacts)
 n_filtered =(assigned_players_classes_filtered==2 ).sum().sum()
n_classes =(assigned_players_classes==2 ).sum().sum()
print(video,n_filtered,n_classes,(~assigned_players_boxes.isna() ).sum().sum())
 all_bboxes = assigned_players_boxes.fillna(method='ffill' ).values.reshape(-1)
 all_classes = assigned_players_classes_filtered.values.reshape(-1)
 all_frames = assigned_players_boxes.apply(lambda x: pd.DataFrame(x ).apply(
 lambda y: y.name, axis=1)).values.reshape(-1)
 bboxes_with_impact = all_bboxes[np.where(all_classes==2)]
frames_with_impact = all_frames[np.where(all_classes==2)]
gameKey,playID,view,_ = video.replace('.','_' ).split('_')
 for frame,box in zip(frames_with_impact,bboxes_with_impact):
submission_df.loc[entries,'gameKey'] = gameKey
submission_df.loc[entries,'playID'] = playID
submission_df.loc[entries,'view'] = view
submission_df.loc[entries,'video'] = video
submission_df.loc[entries,'frame'] = frame
submission_df.loc[entries,'left'] = box[0]
submission_df.loc[entries,'top'] = box[1]
submission_df.loc[entries,'width'] = box[2]-box[0]
submission_df.loc[entries,'height'] = box[3]-box[1]
entries += 1
submission_df = submission_df[(submission_df.frame<FRAME_THRESHOLD)&(submission_df.frame>20)]<create_dataframe>
train=pd.concat([train,labels],axis=1)
submission_df1 = submission_df.copy(deep=True )<feature_engineering>
train[train.Classes==-1]
SEED = 42
FOLDS = 5
FOLD = 2
EFF_DET = 5
Threshold = 0.65
LOOK_BACKWARD_LIMIT = 10
IOU_THRESHOLD = 0.4
IMAGE_SIZE = 512
NUM_CLASSES = 2
BATCH_SIZE = 16
FRAME_THRESHOLD = 240
keep_columns_threshold = 20
video_labels = pd.read_csv('off.csv')
 video_labels = video_labels.sort_values(by=['video','frame'] ).reset_index(drop=True)
 video_labels = video_labels[video_labels.frame>0]
video_labels['image'] = video_labels['video'].str.replace('.mp4', '')+ '_' + video_labels['frame'].astype(str)+ '.png'
video_labels['right'] = video_labels['left'] + video_labels['width']
video_labels['bottom'] = video_labels['top'] + video_labels['height']
video_labels['label_list'] = video_labels['labels'].apply(lambda x:[x])
 video_labels['bbox'] = pd.Series(video_labels[['left','top','right','bottom']].values.tolist() , index=video_labels.index)
 video_labels['box_predictions'] = video_labels['bbox'].apply(lambda x:[x])
 video_labels['score_list'] = video_labels['scores'].apply(lambda x:[x])
 base_cols = ['video','image','gameKey','frame']
target_cols = ['box_predictions','label_list','score_list']
predictions = video_labels[base_cols+target_cols].groupby(base_cols ).sum().reset_index()
predictions = predictions.apply(filter_duplicates_and_low_scores,axis=1)
 predictions['n_impact'] = predictions.label_list.apply(lambda x:(np.array(x)==2 ).sum())
 predictions.groupby('video' ).n_impact.sum().describe()
submission_df = pd.DataFrame(columns=['gameKey','playID','view','video','frame','left','width','top','height'])
 predictions['row_to_class_map'] = predictions.apply(map_classes,axis=1)
 entries = 0
for video in predictions.video.unique() :
assigned_players_boxes,assigned_players_classes = assign_players(predictions,video)
 keep_columns = assigned_players_classes.columns[(assigned_players_classes!=-1 ).sum() >keep_columns_threshold]
assigned_players_boxes = assigned_players_boxes[keep_columns]
assigned_players_classes = assigned_players_classes[keep_columns]
print(assigned_players_boxes.shape)
 assigned_players_classes_filtered = assigned_players_classes.apply(filter_impacts)
 n_filtered =(assigned_players_classes_filtered==2 ).sum().sum()
n_classes =(assigned_players_classes==2 ).sum().sum()
print(video,n_filtered,n_classes,(~assigned_players_boxes.isna() ).sum().sum())
 all_bboxes = assigned_players_boxes.fillna(method='ffill' ).values.reshape(-1)
 all_classes = assigned_players_classes_filtered.values.reshape(-1)
 all_frames = assigned_players_boxes.apply(lambda x: pd.DataFrame(x ).apply(
 lambda y: y.name, axis=1)).values.reshape(-1)
 bboxes_with_impact = all_bboxes[np.where(all_classes==2)]
frames_with_impact = all_frames[np.where(all_classes==2)]
gameKey,playID,view,_ = video.replace('.','_' ).split('_')
 for frame,box in zip(frames_with_impact,bboxes_with_impact):
submission_df.loc[entries,'gameKey'] = gameKey
submission_df.loc[entries,'playID'] = playID
submission_df.loc[entries,'view'] = view
submission_df.loc[entries,'video'] = video
submission_df.loc[entries,'frame'] = frame
submission_df.loc[entries,'left'] = box[0]
submission_df.loc[entries,'top'] = box[1]
submission_df.loc[entries,'width'] = box[2]-box[0]
submission_df.loc[entries,'height'] = box[3]-box[1]
entries += 1
submission_df = submission_df[(submission_df.frame<FRAME_THRESHOLD)&(submission_df.frame>120)]<create_dataframe>
train.drop([197,810,1170,1182,1298,1386,1423],axis=0,inplace=True)
submission_df2 = submission_df.copy(deep=True )<concatenate>
train = train[train.GarageArea * train.GarageCars < 3700]
train = train[(train.FullBath +(train.HalfBath*0.5)+ train.BsmtFullBath +(train.BsmtHalfBath*0.5)) <5]
submission_df = pd.concat([submission_df1,submission_df2] )<set_options>
train.isnull().sum()
test.isnull().sum()
env = nflimpact.make_env()<load_from_csv>
del test['SalePrice']
if IS_PRIVATE:
env.predict(submission_df)
 else:
sub = pd.read_csv('.. /input/nfl-impact-detection/sample_submission.csv')
 env.predict(sub )<set_options>
X=train.drop(['SalePrice','SalePrice_Log1p','Classes'],axis=1)
 y=train.SalePrice_Log1p
shap.initjs()
sns.set_style('whitegrid')
 OneHotEncoder, OrdinalEncoder)
<load_from_csv>
def overfit_reducer(df):
overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.9:
overfit.append(i)
 overfit = list(overfit)
 return overfit
overfitted_features = overfit_reducer(X )
df_train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 df_test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 df_sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv')
 SEED = 1991
sub_295 = pd.read_csv('.. /input/tp-f-my-subs/submission_001.csv')
 sub_222 = pd.read_csv('.. /input/tp-f-my-subs/submission_021.csv')
 sub_216 = pd.read_csv('.. /input/tp-f-my-subs/submission_216.csv')
 sub_257 = pd.read_csv('.. /input/tp-f-my-subs/submission_257.csv')
 sub_279 = pd.read_csv('.. /input/tp-f-my-subs/submission_012.csv' )<drop_column>
std_scaler=StandardScaler()
rbst_scaler=RobustScaler()
power_transformer=PowerTransformer()
X_std=std_scaler.fit_transform(X)
 X_rbst=rbst_scaler.fit_transform(X)
 X_pwr=power_transformer.fit_transform(X)
 test_std=std_scaler.transform(test)
 test_rbst=rbst_scaler.transform(test)
 test_pwr=power_transformer.transform(test)
 X_train,X_test,y_train,y_test=train_test_split(X_std,y,test_size=0.002,random_state=52 )
target = df_train['target']
df_train.drop('target', axis=1, inplace=True )<filter>
lgb_regressor=lgb.LGBMRegressor(objective='regression', num_leaves=5, learning_rate=0.035, n_estimators=2177, max_bin=50, bagging_fraction=0.65,bagging_freq=5, bagging_seed=7,
feature_fraction=0.201, feature_fraction_seed=7,n_jobs=-1)
 lgb_regressor.fit(X_train, y_train)
 y_head=lgb_regressor.predict(X_test)
 print('-'*10+'LGBM'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
len(target[target <= 4] )<sort_values>
gb_reg = GradientBoostingRegressor(n_estimators=1992, learning_rate=0.03005, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=14, loss='huber', random_state =42)
 gb_reg.fit(X_train, y_train)
 y_head=gb_reg.predict(X_test)
 print('-'*10+'GBR'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
CAT_01 = list(set(CAT ).difference(set(['cat6'])))
 CAT_01<categorify>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 alphas=[1e-9,1e-8,1e-7,1e-6]
ridgecv_reg= make_pipeline(RidgeCV(alphas=alphas, cv=kfolds))
ridgecv_reg.fit(X_train, y_train)
 y_head=ridgecv_reg.predict(X_test)
 print('-'*10+'RidgeCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
<categorify>
kfolds = KFold(n_splits=8, shuffle=True, random_state=42)
 lassocv_reg= make_pipeline(LassoCV(alphas=alphas, cv=kfolds))
lassocv_reg.fit(X_train, y_train)
 y_head=lassocv_reg.predict(X_test)
 print('-'*10+'LassoCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
<drop_column>
kfolds = KFold(n_splits=8, shuffle=True, random_state=42)
 alphas=[0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006]
l1ratio=[0.87, 0.9,0.92, 0.95,0.97, 0.99, 1]
elasticv_reg= make_pipeline(ElasticNetCV(alphas=alphas, cv=kfolds, l1_ratio=l1ratio))
elasticv_reg.fit(X_train, y_train)
 y_head=elasticv_reg.predict(X_test)
 print('-'*10+'ElasticNetCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
train = df_train.copy()
test = df_test.copy()
train.drop(['id'], axis=1, inplace=True)
 test.drop(['id'], axis=1, inplace=True)
 train.shape, test.shape<categorify>
estimators = [('lgbm', lgb_regressor), 
('gbr', gb_reg), 
('lasso', lassocv_reg), 
('ridge', ridgecv_reg), 
('elasticnet', elasticv_reg)]
stack_reg=StackingRegressor(estimators=estimators,final_estimator=ExtraTreesRegressor(n_estimators=50),n_jobs=-1)
 stack_reg.fit(X_train, y_train)
 y_head=stack_reg.predict(X_test)
 print('-'*10+'StackingRegressor'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))
cols = train.columns.tolist()
ct = ColumnTransformer([('onehot',OrdinalEncoder() , slice(len(CAT))), 
('quantile',QuantileTransformer(random_state=SEED, n_quantiles=1500),
slice(len(CAT),len(CAT)+ len(NUM)+ 5)) ])
 train = ct.fit_transform(train)
 test = ct.transform(test)
 train = pd.DataFrame(train, columns = cols)
 test = pd.DataFrame(test, columns = cols)
 train[CAT] = train[CAT] / 10
test[CAT] = test[CAT] / 10<feature_engineering>
y_head=pd.DataFrame(y_head,columns=['Predict'])
 y_test.reset_index(drop=True,inplace=True)
 y_test_y_head=pd.concat([y_test,y_head],axis=1)
 y_test_y_head.head()
def feat_eng(df):
df['cont001'] = df['cont8'] * df['cont0']
df['cont002'] = df['cont9'] * df['cont0']
df['cont003'] = df['cont9'] * df['cont5']
df['cont004'] = df['cont8'] * df['cont5']
df['cont005'] = df['cont2'] * df['cont4']
df['cat005'] = df['cat2'] * df['cat1']
return df<create_dataframe>
test_pred_lgb=lgb_regressor.predict(test_pwr)
 test_pred_gb=gb_reg.predict(test_pwr)
 test_pred_elastic=elasticv_reg.predict(test_pwr)
 test_pred_ridge=ridgecv_reg.predict(test_pwr)
 test_pred_lasso=lassocv_reg.predict(test_pwr)
 test_pred_stack=stack_reg.predict(test_pwr)
 test_pred_lgb=pd.DataFrame(test_pred_lgb,columns=['SalePrice'])
 test_pred_gb=pd.DataFrame(test_pred_gb,columns=['SalePrice'])
 test_pred_elastic=pd.DataFrame(test_pred_elastic,columns=['SalePrice'])
 test_pred_ridge=pd.DataFrame(test_pred_ridge,columns=['SalePrice'])
 test_pred_lasso=pd.DataFrame(test_pred_lasso,columns=['SalePrice'])
 test_pred_stack=pd.DataFrame(test_pred_stack,columns=['SalePrice'])
 test_pred_lgb.SalePrice =np.floor(np.expm1(test_pred_lgb.SalePrice))
test_pred_gb.SalePrice =np.floor(np.expm1(test_pred_gb.SalePrice))
test_pred_elastic.SalePrice =np.floor(np.expm1(test_pred_elastic.SalePrice))
test_pred_ridge.SalePrice =np.floor(np.expm1(test_pred_ridge.SalePrice))
test_pred_lasso.SalePrice =np.floor(np.expm1(test_pred_lasso.SalePrice))
test_pred_stack.SalePrice =np.floor(np.expm1(test_pred_stack.SalePrice))
test_pred_lgb.head()
def feat_eng_01(df):
es = ft.EntitySet(id = 'data')
 original_cols = to_transform
es = es.entity_from_dataframe(entity_id = 'data',
dataframe = df[original_cols],
index = 'id',
time_index = None)
 new_features, new_feature_names = ft.dfs(entityset = es, target_entity = 'data',
trans_primitives = ['multiply_numeric'])
 new_features.reset_index(drop=True, inplace=True)
 new_features.drop(original_cols, axis=1, inplace=True)
 return new_features<split>
final_pred=(test_pred_stack*0.1665)+(test_pred_lgb*0.678)+(test_pred_ridge*0.1665)
 final_pred.head()
<init_hyperparams>
sample_sub['SalePrice']=final_pred
<train_model><EOS> 
sample_sub.to_csv('sample_submission.csv',index=False )