Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<load_from_csv>
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50)
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 start = time.time()
X_testfresh = pd.DataFrame()
for i in progressbar(range(len(submission)) , "Computing: ", 40):
X1 = pd.DataFrame()
ids = pd.DataFrame({'id': [submission.index[i] for number in range(n)]}, dtype=np.int16)
 seg = pd.read_csv('.. /input/test/' + submission.index[i] + '.csv')
 x = seg['acoustic_data']
test=pd.concat([ids,times,x],axis=1)
 X2=ts_features(test,peak)
 for j in range(4):
X=ts_features(test[j * parts:(j + 1)*parts],settings ).add_prefix('q_' +str(j))
X1=pd.concat([X1,X], axis=1)
 X_testfresh=X_testfresh.append(pd.concat([X1,X2], axis=1))
end = time.time()
print("Time",end - start)
<save_to_csv>
!nvidia-smi
dfb=X_filtered.reset_index(drop=True ).append(X_testfresh)
 dfb.to_csv('unfiltered.csv',index=False)
 print(dfb.shape)
 dfb=dfb.dropna(axis='columns')
 dfb=dfb.loc[:,(dfb != dfb.iloc[0] ).any() ]
print(dfb.shape)
 dfb.head()<save_to_csv>
!nvcc --version
df=pd.concat([y_tr,dfb[:X_filtered.shape[0]]], axis=1)
 df.to_csv('tsfreshFeaturestrain.csv',index=False)
 test=dfb[-X_testfresh.shape[0]:]
test.to_csv('tsfreshFeaturestest.csv',index=False)
 X=df[df.columns.difference(['time_to_failure'])]
y=df['time_to_failure']
scaler=MinMaxScaler(feature_range=(0, 1))
scaled=scaler.fit_transform(X.append(test))
train=pd.DataFrame(scaled[:X.shape[0]], columns=X.columns.values)
 test=pd.DataFrame(scaled[-test.shape[0]:], columns=X.columns.values)
 trainX, testX, trainy, testy = train_test_split(train, y, test_size=0.0,shuffle=True)
<compute_train_metric>
torch.__version__
model = GradientBoostingRegressor(learning_rate=0.009, loss='huber',max_depth=5,max_features=150,n_estimators=650)
 num_folds = 5
cross_validate_results = cross_validate(model, trainX, trainy, n_jobs=-1, return_estimator= True, return_train_score=True, cv=num_folds, scoring="neg_mean_absolute_error")
 for i in range(num_folds):
print("Fold: {} Training MAE: {}".format(i,-cross_validate_results['train_score'][i]))
print("Fold: {} Validation MAE: {}".format(i,-cross_validate_results['test_score'][i]))
print(-cross_validate_results['train_score'].mean() , cross_validate_results['train_score'].std())
 print(-cross_validate_results['test_score'].mean() , cross_validate_results['test_score'].std())
 model.fit(trainX, trainy)
 data=pd.concat([pd.DataFrame(X.columns.values,columns={'Features'}),
pd.DataFrame(model.feature_importances_,columns={'Importance'})*100],
axis=1 ).sort_values(by='Importance', ascending=False)
 data.to_csv('FeaturesImportance.csv',index=False)
 plt.figure(figsize=(6, 20))
sns.barplot(y="Features", x="Importance", data=data[:150])
 plt.title("Features")
 plt.show()<train_model>
!pip install detectron2 -f \
https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html
mae = np.array([])
 predictions1 = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = trainX.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(trainX,trainy.values)) :
strLog = "fold {}".format(fold_)
 X_tr, X_val = trainX.iloc[trn_idx], trainX.iloc[val_idx]
y_tr, y_val = trainy.iloc[trn_idx], trainy.iloc[val_idx]
model.fit(X_tr, y_tr)
 mae = np.append(mae, mean_absolute_error(y_val, model.predict(X_val)))
 print("Fold: {} Validation MAE: {}".format(strLog,mae[fold_]))
predictions1 += model.predict(test)/ folds.n_splits
np.mean(mae )<save_to_csv>
def get_vinbigdata_dicts(
 imgdir: Path,
train_df: pd.DataFrame,
train_data_type: str = "original",
use_cache: bool = True,
debug: bool = True,
target_indices: Optional[np.ndarray] = None,
use_class14: bool = False, 
):
debug_str = f"_debug{int(debug)}"
train_data_type_str = f"_{train_data_type}"
class14_str = f"_14class{int(use_class14)}"
cache_path = Path(".")/ f"dataset_dicts_cache{train_data_type_str}{class14_str}{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 train_meta = pd.read_csv(imgdir / "train_meta.csv")
 if debug:
train_meta = train_meta.iloc[:500]
image_id = train_meta.loc[0, "image_id"]
image_path = str(imgdir / "train" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, train_meta_row in tqdm(train_meta.iterrows() , total=len(train_meta)) :
record = {}
image_id, height, width = train_meta_row.values
filename = str(imgdir / "train" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
objs = []
for index2, row in train_df.query("image_id == @image_id" ).iterrows() :
class_id = row["class_id"]
if class_id == 14:
if use_class14:
bbox_resized = [0, 0, resized_width, resized_height]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 else:
pass
else:
h_ratio = resized_height / height
w_ratio = resized_width / width
bbox_resized = [
float(row["x_min"])* w_ratio,
float(row["y_min"])* h_ratio,
float(row["x_max"])* w_ratio,
float(row["y_max"])* h_ratio,
]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 record["annotations"] = objs
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 if target_indices is not None:
dataset_dicts = [dataset_dicts[i] for i in target_indices]
return dataset_dicts
def get_vinbigdata_dicts_test(
 imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True, 
):
debug_str = f"_debug{int(debug)}"
cache_path = Path(".")/ f"dataset_dicts_cache_test{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 if debug:
test_meta = test_meta.iloc[:500]
image_id = test_meta.loc[0, "image_id"]
image_path = str(imgdir / "test" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, test_meta_row in tqdm(test_meta.iterrows() , total=len(test_meta)) :
record = {}
image_id, height, width = test_meta_row.values
filename = str(imgdir / "test" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 return dataset_dicts
submission.time_to_failure = predictions1
submission.to_csv('submissiongbm.csv',index=True)
 submission.head()<init_hyperparams>
def format_pred(labels: ndarray, boxes: ndarray, scores: ndarray)-> str:
pred_strings = []
for label, score, bbox in zip(labels, scores, boxes):
xmin, ymin, xmax, ymax = bbox.astype(np.int64)
 pred_strings.append(f"{label} {score} {xmin} {ymin} {xmax} {ymax}")
 return " ".join(pred_strings)
 def predict_batch(predictor: DefaultPredictor, im_list: List[ndarray])-> List:
with torch.no_grad() :
inputs_list = []
for original_image in im_list:
if predictor.input_format == "RGB":
original_image = original_image[:, :, ::-1]
height, width = original_image.shape[:2]
image = original_image
image = torch.as_tensor(image.astype("float32" ).transpose(2, 0, 1))
inputs = {"image": image, "height": height, "width": width}
inputs_list.append(inputs)
 predictions = predictor.model(inputs_list)
 return predictions
params={'bagging_fraction': 0.72,
'boosting': 'gbdt',
'feature_fraction' : 0.55,
'lambda_l1': 4.2030167201177235,
'lambda_l2': 2.8818379730100174,
'learning_rate': 0.04197920600013021,
'max_bin': 182,
'max_depth': 13,
'metric': 'RMSE',
'min_data_in_bin': 106,
'min_data_in_leaf': 83,
'num_leaves': 1415,
'objective': 'fair',
'subsample': 0.5807414994404817} 
<split>
def save_yaml(filepath: Union[str, Path], content: Any, width: int = 120):
with open(filepath, "w")as f:
yaml.dump(content, f, width=width)
 def load_yaml(filepath: Union[str, Path])-> Any:
with open(filepath, "r")as f:
content = yaml.full_load(f)
 return content
predictions2 = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = trainX.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(trainX,trainy.values)) :
strLog = "fold {}".format(fold_)
 X_tr, X_val = trainX.iloc[trn_idx], trainX.iloc[val_idx]
y_tr, y_val = trainy.iloc[trn_idx], trainy.iloc[val_idx]
print(strLog)
 model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=1000)
 mae = np.append(mae, mean_absolute_error(y_val, model.predict(X_val)))
 print("Fold: {} Validation MAE: {}".format(strLog,mae[fold_]))
predictions2 += model.predict(test, num_iteration=model.best_iteration_)/ folds.n_splits
np.mean(mae )<save_to_csv>
thing_classes = [
"Aortic enlargement",
"Atelectasis",
"Calcification",
"Cardiomegaly",
"Consolidation",
"ILD",
"Infiltration",
"Lung Opacity",
"Nodule/Mass",
"Other lesion",
"Pleural effusion",
"Pleural thickening",
"Pneumothorax",
"Pulmonary fibrosis"
]
category_name_to_id = {class_name: index for index, class_name in enumerate(thing_classes)}
submission.time_to_failure = predictions2
submission.to_csv('submissionlgbm.csv',index=True)
 submission.head()<save_to_csv>
@dataclass
class Flags:
debug: bool = True
outdir: str = "results/det"
imgdir_name: str = "vinbigdata-chest-xray-resized-png-256x256"
split_mode: str = "all_train"
seed: int = 111
train_data_type: str = "original"
use_class14: bool = False
iter: int = 10000
ims_per_batch: int = 2
num_workers: int = 4
lr_scheduler_name: str = "WarmupMultiStepLR"
base_lr: float = 0.00025
roi_batch_size_per_image: int = 512
eval_period: int = 10000
aug_kwargs: Dict = field(default_factory=lambda: {})
 def update(self, param_dict: Dict)-> "Flags":
for key, value in param_dict.items() :
if not hasattr(self, key):
raise ValueError(f"[ERROR] Unexpected key for flag = {key}")
 setattr(self, key, value)
 return self
blending = predictions1*0.5 + predictions2*0.5
submission['time_to_failure']=blending
submission.to_csv('submissionblend.csv',index=True)
 submission.head()<import_modules>
inputdir = Path("/kaggle/input")
 traineddir = inputdir / "vinbigdata-alb-aug-512-cos"
flags: Flags = Flags().update(load_yaml(str(traineddir/"flags.yaml")))
 print("flags", flags)
 debug = flags.debug
outdir = Path(flags.outdir)
 os.makedirs(str(outdir), exist_ok=True)
 datadir = inputdir / "vinbigdata-chest-xray-abnormalities-detection"
if flags.imgdir_name == "vinbigdata-chest-xray-resized-png-512x512":
imgdir = inputdir/ "vinbigdata"
else:
imgdir = inputdir / flags.imgdir_name
test_meta = pd.read_csv(inputdir / "vinbigdata-testmeta" / "test_meta.csv")
 sample_submission = pd.read_csv(datadir / "sample_submission.csv" )
import gc
import numpy as np
import pandas as pd
import lightgbm as lgbm
from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error<load_from_csv>
cfg = get_cfg()
original_output_dir = cfg.OUTPUT_DIR
cfg.OUTPUT_DIR = str(outdir)
 print(f"cfg.OUTPUT_DIR {original_output_dir} -> {cfg.OUTPUT_DIR}")
 cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN =("vinbigdata_train",)
 cfg.DATASETS.TEST =()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml")
 cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = flags.base_lr
cfg.SOLVER.MAX_ITER = flags.iter
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = flags.roi_batch_size_per_image
cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(thing_classes)
 cfg.MODEL.WEIGHTS = str(traineddir/"model_final.pth")
 print("Original thresh", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)
 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.0
print("Changed thresh", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)
 predictor = DefaultPredictor(cfg)
 DatasetCatalog.register(
 "vinbigdata_test", lambda: get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
)
 MetadataCatalog.get("vinbigdata_test" ).set(thing_classes=thing_classes)
 metadata = MetadataCatalog.get("vinbigdata_test")
 dataset_dicts = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
 if debug:
dataset_dicts = dataset_dicts[:100]
results_list = []
index = 0
batch_size = 4
for i in tqdm(range(ceil(len(dataset_dicts)/ batch_size))):
inds = list(range(batch_size * i, min(batch_size *(i + 1), len(dataset_dicts))))
dataset_dicts_batch = [dataset_dicts[i] for i in inds]
im_list = [cv2.imread(d["file_name"])for d in dataset_dicts_batch]
outputs_list = predict_batch(predictor, im_list)
 for im, outputs, d in zip(im_list, outputs_list, dataset_dicts_batch):
resized_height, resized_width, ch = im.shape
if index < 5:
v = Visualizer(
 im[:, :, ::-1],
metadata=metadata,
scale=0.5,
instance_mode=ColorMode.IMAGE_BW 
)
 out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
cv2.imwrite(str(outdir / f"pred_{index}.jpg"), out.get_image() [:, :, ::-1])
 image_id, dim0, dim1 = test_meta.iloc[index].values
instances = outputs["instances"]
if len(instances)== 0:
result = {"image_id": image_id, "PredictionString": "14 1.0 0 0 1 1"}
else:
fields: Dict[str, Any] = instances.get_fields()
pred_classes = fields["pred_classes"]
pred_scores = fields["scores"]
pred_boxes = fields["pred_boxes"].tensor
h_ratio = dim0 / resized_height
w_ratio = dim1 / resized_width
pred_boxes[:, [0, 2]] *= w_ratio
pred_boxes[:, [1, 3]] *= h_ratio
pred_classes_array = pred_classes.cpu().numpy()
pred_boxes_array = pred_boxes.cpu().numpy()
pred_scores_array = pred_scores.cpu().numpy()
result = {
"image_id": image_id,
"PredictionString": format_pred(
 pred_classes_array, pred_boxes_array, pred_scores_array 
),
}
results_list.append(result)
 index += 1
<init_hyperparams><EOS> 
submission_det = pd.DataFrame(results_list, columns=['image_id', 'PredictionString'])
 submission_det.to_csv(outdir/"submission.csv", index=False)
 submission_det