Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<train_model>
import numpy as np
import pandas as pd
import seaborn as sns
import lightgbm as lgb
import time
from sklearn.model_selection import KFold,StratifiedKFold
train()<load_from_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()
num_test_images = count_data_items(test_filenames)
 submission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df.copy()
del temp['target']
submission_df['target'] += temp.merge(pred_df,on="image_name")['target']/TTA<save_to_csv>
train_df['target'].value_counts()
submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.round(submission_df['target'].values)).value_counts()<set_options>
X_test = test_df.drop('ID_code',axis=1)
 X = train_df.drop(['ID_code','target'],axis=1)
 y = train_df['target']
warnings.filterwarnings('ignore')
<load_from_csv>
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42 )
yolo = pd.read_csv('.. /input/vinbigdatastack/yolov5.csv')
 detectron = pd.read_csv('.. /input/vinbigdatastack/detectron2.csv')
 fasterrcnn = pd.read_csv('.. /input/vinbigdatastack/fasterrcnn.csv')
 image_ids = yolo.image_id.values<feature_engineering>
params = {'num_leaves': 8,'min_data_in_leaf': 42,'objective': 'binary','max_depth': 16,'learning_rate': 0.0123,'boosting': 'gbdt','bagging_freq': 5,
'bagging_fraction': 0.8,'feature_fraction': 0.8201,'bagging_seed': 11,'reg_alpha': 1.728910519108444,'reg_lambda': 4.9847051755586085,'random_state': 42,
'metric': 'auc','verbosity': -1,'subsample': 0.81,'min_gain_to_split': 0.01077313523861969,'min_child_weight': 19.428902804238373,'num_threads': 4}
def getitem(dataframe, img_id):

 pred = list(dataframe.loc[dataframe.image_id == img_id, "PredictionString"])[0].split(' ')
 nb_elm = len(pred)//6
output = {}
for elm in range(nb_elm):
output[f'elm_{elm}'] = pred[elm*6 :(elm+1)*6]
return output
def sortDictByProba(dict_):

 for key in dict_.keys() :
dict_[key] = list(map(lambda x: float(x), dict_[key]))
return {k: v for k, v in sorted(dict_.items() , key=lambda item: item[1][1], reverse = True)}
def getHighestProba(*list_of_dicts, n=3):

 output = {}
for index, dict_ in enumerate(list_of_dicts):
dict_length = len(dict_)
 for i in range(dict_length):
if i < n:
output[f"elm_{i}_dict_{index}"] =list(dict_.values())[i]
return output
def getUnique(dict_):

 dict_length = len(dict_)
 classes_non_unique = [list(dict_.values())[index][0] for index in range(dict_length)]
classes_unique = list(set(classes_non_unique))
uniques, counts = np.unique(classes_non_unique, return_counts=True)
 duplicates = uniques[counts > 1]
singles = np.setdiff1d(classes_unique, duplicates)
 return singles, duplicates
def getKeysByValue(dictOfElements, valueToFind):

 output = list()
listOfItems = dictOfElements.items()
for item in listOfItems:
if item[1][0] == valueToFind:
output.append(item[0])
 return output
def getListKeysByValue(dictOfElements, valuesToFind):

 output = []
for value in valuesToFind:
output.append(getKeysByValue(dictOfElements, value))
return output
def averaging(from_dict, single_keys, dupl_keys):

 output = {}
if len(np.ravel(single_keys)) != 0:
for single in np.ravel(single_keys):
output[single] = from_dict[single]
if len(np.ravel(dupl_keys)) != 0:
for index, list_of_duplicate_class in enumerate(dupl_keys):
probs = []
boxing1 = []
boxing2 = []
boxing3 = []
boxing4 = []
for elm in list_of_duplicate_class:
probs.append(from_dict[elm][1])
 boxing1.append(from_dict[elm][2])
 boxing2.append(from_dict[elm][3])
 boxing3.append(from_dict[elm][4])
 boxing4.append(from_dict[elm][5])
 output[f"elm_{index}"] = [from_dict[list_of_duplicate_class[0]][0],
np.mean(probs),
np.mean(boxing1),
np.mean(boxing2),
np.mean(boxing3),
np.mean(boxing4)]
return output
def toString(pred_list):

 castedList = []
for index, elm in enumerate(pred_list):
if index%6 == 0:
castedList.append(str(int(elm)))
 else:
castedList.append(str(elm))
output = " ".join(castedList)
 return output<create_dataframe>
prediction = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 prediction += model.predict(X_test, num_iteration=model.best_iteration)/5
def main() :
output = pd.DataFrame(columns = ["image_id", "PredictionString"])
 for image_id in tqdm(image_ids):
fasterrcnn_pred = getitem(fasterrcnn, image_id)
 detectron_pred = getitem(detectron, image_id)
 yolo_pred = getitem(yolo, image_id)
 sorted_fasterrcnn = sortDictByProba(fasterrcnn_pred)
 sorted_detectron = sortDictByProba(detectron_pred)
 sorted_yolo = sortDictByProba(yolo_pred)
 highest_probs = getHighestProba(sorted_fasterrcnn,
sorted_detectron,
sorted_yolo,
n = 3)
 singles, duplicates = getUnique(highest_probs)
 single_keys = getListKeysByValue(highest_probs, singles)
 dupl_keys = getListKeysByValue(highest_probs, duplicates)
 stacked_dict = averaging(highest_probs, single_keys, dupl_keys)
 prediction_int = np.ravel(list(stacked_dict.values()))
prediction_string = toString(prediction_int)
 output = output.append({"image_id": image_id,
"PredictionString": prediction_string},
ignore_index=True)
 return output<save_to_csv>
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_
final_sub = main()
final_sub.to_csv("submission.csv", index=False )<install_modules>
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )
<set_options><EOS> 
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )