Kaggle competition: RANZCR CLiP - Catheter and Line Position Challenge
<SOS> metric: MCAUC Kaggle data source: ranzcr-clip-catheter-and-line-position-challenge<feature_engineering>
import pandas as pd
import numpy as np
import os
import tensorflow as tf
all_data = pd.concat(objs=[train_data, test_data], axis=0 ).reset_index(drop=True)
 def extract_title(Name):
if '.' in Name:
return Name.split(',')[1].split('.')[0].strip()
else:
return 'None'
all_data['Title'] = all_data['Name'].map(lambda x: extract_title(x))
train_data['Title'] = train_data['Name'].map(lambda x: extract_title(x))
test_data['Title'] = test_data['Name'].map(lambda x: extract_title(x))
title_list = all_data['Title'].unique()
print("Title list:", title_list)
 print("
")
 def identify_title_category(cols):
Parch = cols[0]
Title = cols[1]
if Title in ['Miss', 'Master'] and Parch == 0:
return 'Young without parent-child'
elif Title in ['Miss', 'Master'] and Parch > 0:
return 'Young with parent-child'
elif Title not in ['Miss', 'Master'] and Parch == 0:
return 'Old without parent-child'
else:
return 'Old with parent-child'
all_data['Title'] = all_data[["Parch","Title"]].apply(identify_title_category, axis=1)
 train_data['Title'] = train_data[["Parch","Title"]].apply(identify_title_category, axis=1)
 test_data['Title'] = test_data[["Parch","Title"]].apply(identify_title_category, axis=1)
 def replace_null_age(cols):
Age = cols[0]
Pclass = cols[1]
Sex = cols[2]
Title = cols[3]
if pd.isnull(Age):
return float(round(all_data[(( all_data["Pclass"] == Pclass)&(all_data["Sex"] == Sex)&(all_data["Title"] == Title)) ]["Age"].mean()))
else:
return Age
train_data["Age"] = train_data[["Age", "Pclass", "Sex", "Title"]].apply(replace_null_age,axis=1)
 test_data["Age"] = test_data[["Age", "Pclass", "Sex", "Title"]].apply(replace_null_age,axis=1)
 grouped_average_age = all_data.groupby(['Pclass', 'Sex', 'Title'])["Age"].mean()
print("grouped_average_age:", "
", grouped_average_age)
 train_data.head(n=10)
<categorify>
img_size = 900
test_data = test_data.fillna(test_data.mean())
 train_data.fillna(train_data.mean())
 survived_data = train_data["Survived"]
relevant_features = ["Pclass", "Sex", "SibSp", "Parch", "Embarked", "Age", "Fare"]
X_train = pd.get_dummies(train_data[relevant_features])
 X_test = pd.get_dummies(test_data[relevant_features])
 k_fold = KFold(n_splits=10, shuffle=True, random_state=1)
 model = RandomForestClassifier(n_estimators=100)
 scoring = "accuracy"
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("RF score:", score)
 print(round(np.mean(score)*100, 2))
model = SVC()
scoring = "accuracy"
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("SVC score:", score)
 print(round(np.mean(score)*100, 2))
model = KNeighborsClassifier(n_neighbors = 10)
 scoring = 'accuracy'
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("K Neighbor score", score)
 print(round(np.mean(score)*100, 2))
model = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("Gaussian score", score)
 print(round(np.mean(score)*100, 2))
model = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("Decision Tree score", score)
 print(round(np.mean(score)*100, 2))
model = RandomForestClassifier(n_estimators=200, max_depth=6, random_state=100)
 scoring = "accuracy"
score = cross_val_score(model, X_train, survived_data, cv=k_fold, n_jobs=1, scoring=scoring)
 print("New RF score:", score)
 print(round(np.mean(score)*100, 2))
model.fit(X_train, survived_data)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission_7_1.csv', index=False)
 print("Submission done!")
<import_modules>
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(img_size, img_size), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 img = tf.image.random_saturation(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.image.random_contrast(img, 0.9, 1.2)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
from xgboost import XGBClassifier
from xgboost import plot_importance
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix,classification_report, mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, cross_val_score, GridSearchCV 
<load_from_csv>
COMPETITION_NAME = "ranzcr-clip-catheter-line-classification"
strategy = auto_select_accelerator()
batch_size = strategy.num_replicas_in_sync * 16
print('batch size', batch_size )
def load_csv(dataset, datadir):

 df = pd.read_csv(
 f"{datadir}/{dataset}.csv",
dtype={"Country/Region": "category"},
parse_dates=["Date"], 
)
 df["DayOfYear"] = df["Date"].dt.dayofyear
df["Province/State"].fillna(df["Country/Region"], inplace=True)
 return df
def sigmoid(x, x0, L, k):
return L /(1 + np.exp(-k *(x - x0)))
 def fit(df: pd.DataFrame, index: str, channel: str):
x = df[index].values
y = df[channel].values
try:
p, pc = curve_fit(
 sigmoid,
x,
y,
p0=(np.median(x), y[-1], 0.1),
maxfev=100000,
bounds=([0, y[-1], 0], [np.inf, np.inf, 2]),
ftol=1e-5, 
)
 except RuntimeError:
return [-1] * 9
return list(p)+ pc[np.triu_indices_from(pc)].tolist()
def apply_fit(df: pd.DataFrame, index: str, channel: str)-> pd.DataFrame:

 fits = df.groupby(["Country/Region", "Province/State"])[
[index, channel]
].apply(lambda x: fit(x, index=index, channel=channel))
return pd.DataFrame(
 fits.tolist() ,
columns=["x0", "L", "k", "sx0", "sx0L", "sx0k", "sL", "sLk", "sk"],
index=fits.index,
).add_suffix(f"_{channel}")
 def prep_pop(filename: str)-> pd.DataFrame:

 df = pd.read_csv(
 filename,
engine="python",
skiprows=901,
index_col=0,
names=(
 "idx",
"country",
"Year",
"Series",
"Value",
"Footnotes",
"Source", 
),
usecols=["country", "Year", "Series", "Value"], 
)
 pivot =(
 df[df["Year"] == 2019] 
.pivot(columns="Series", values="Value")
.dropna() 
.rename(
 columns={
"Population aged 0 to 14 years old(percentage)": "young%",
"Population aged 60+ years old(percentage)": "old%",
"Population density": "density",
"Population mid-year estimates(millions)": "population",
"Population mid-year estimates for females(millions)": "fem",
"Population mid-year estimates for males(millions)": "male",
"Sex ratio(males per 100 females)": "ratio",
} 
)
)
 pivot["fem%"] = pivot["fem"] * 100.0 / pivot["population"]
return pivot.drop(columns=["fem", "male", "ratio"])
 def prep_hh(filename: str)-> pd.DataFrame:

 return(
 pd.read_csv(
 filename,
usecols=[0, 3, 4, 5, 6, 7, 8, 18],
names=[
"country",
"date",
"avg_hh",
"hh%1",
"hh%2-3",
"hh%4-5",
"hh%6+",
"hh65+",
],
na_values=".. ",
header=0,
parse_dates=["date"], 
)
.sort_values(by=["country", "date"])
.groupby("country")
.tail(1)
.drop(columns=["date"])
.set_index("country")
)
 def predict(df):
df["y_ConfirmedCases"] =(
(
 df["L_ConfirmedCases"]
/(
 1
+ np.exp(
 -df["k_ConfirmedCases"]
*(df["DayOfYear"] - df["x0_ConfirmedCases"])
)
)
)
.round() 
.astype(int)
)
 df["y_Fatalities"] =(
(
 df["L_Fatalities"]
/(
 1
+ np.exp(
 -df["k_Fatalities"]
*(df["DayOfYear"] - df["x0_Fatalities"])
)
)
)
.round() 
.astype(int)
)
 return df
def merge_support_main(main_df, supp_df):
return pd.merge(
 main_df,
supp_df,
left_on="Country/Region",
right_on="country",
how="left", 
)
 def augment_dataset(DATADIR="data/"):
train = load_csv("train", DATADIR)
 test = load_csv("test", DATADIR)
 fits_cc = apply_fit(train, index="DayOfYear", channel="ConfirmedCases")
 fits_f = apply_fit(train, index="DayOfYear", channel="Fatalities")
 fits = fits_cc.join(fits_f)
 train = pd.merge(
 train, fits.reset_index() , on=["Country/Region", "Province/State"] 
)
 test = pd.merge(
 test, fits.reset_index() , on=["Country/Region", "Province/State"] 
)
 sf = pd.read_csv(f"{DATADIR}/supporting_features.csv", index_col=0 ).drop(
 columns=["avg_HH"] 
)
 sf.index.name = "country"
train = merge_support_main(train, sf)
 test = merge_support_main(test, sf)
 hh = prep_hh(
 f"{DATADIR}/population_division_UN_Houseshold_Size_and_Composition_2019.csv" 
)
 train = pd.merge(
 train, hh, left_on="Country/Region", right_on="country", how="left" 
)
 test = pd.merge(
 test, hh, left_on="Country/Region", right_on="country", how="left" 
)
 pop = prep_pop(
 f"{DATADIR}/SYB62_1_201907_Population, Surface Area and Density.csv" 
)
 train = pd.merge(
 train, pop, left_on="Country/Region", right_on="country", how="left" 
)
 test = pd.merge(
 test, pop, left_on="Country/Region", right_on="country", how="left" 
)
 train = predict(train)
 test = predict(test)
 return train, test<load_from_csv>
with strategy.scope() :
model = tf.keras.models.load_model('.. /input/tensorflow-data-dataset-tpu-part1/model.h5', custom_objects={'FixedDropout': tf.keras.layers.Dropout} )
train = pd.read_csv("/kaggle/input/covid19aug/augmented/train_aug.csv")
 test = pd.read_csv("/kaggle/input/covid19aug/augmented/test_aug.csv" )<create_dataframe>
test_decoder = build_decoder(with_labels=False, target_size=(img_size, img_size))
load_dir = '/kaggle/input/ranzcr-clip-catheter-line-classification/'
df_sub = pd.read_csv(load_dir + 'sample_submission.csv')
 test_paths = load_dir + "test/" + df_sub['StudyInstanceUID'] + '.jpg'
dtest = build_dataset(
 test_paths, bsize=batch_size, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder
)
def prepare_data(df: pd.DataFrame):
df = df[df['ConfirmedCases'] > 100].reset_index()
df['days_since_first100cases'] = df.groupby(['Province/State', 'Country/Region'] 
)['ConfirmedCases'].transform(lambda x: range(len(x)))
 return df.drop(['x0_ConfirmedCases', 'L_ConfirmedCases', 'k_ConfirmedCases', 'sx0_ConfirmedCases',
'sx0L_ConfirmedCases', 'sx0k_ConfirmedCases', 'sL_ConfirmedCases',
'sLk_ConfirmedCases', 'sk_ConfirmedCases', 'x0_Fatalities',
'L_Fatalities', 'k_Fatalities', 'sx0_Fatalities', 'sx0L_Fatalities',
'sx0k_Fatalities', 'sL_Fatalities', 'sLk_Fatalities', 'sk_Fatalities',
'DayOfYear', 'index', 'Id', 'Date'], axis=1)
 def impute_values(df: pd.DataFrame):
features = ['life_expectancy_years', 'literacy_rates', 'veg_supply_person_kg_year',
'respiratory_infections_death', 'deaths_from_smoking%', 'young%',
'old%', 'density', 'population', 'fem%', 'number_doc_per1000',
'surgeons_act_working', 'obstetricians_act_working',
'anaesthesiologists_act_working', 'avg_hh', 'hh%1', 'hh%2-3', 'hh%4-5',
'hh%6+', 'hh65+']
df_per_country = df.groupby(
 ['Province/State', 'Country/Region'], as_index=False ).min()
df_per_country = df_per_country[[
'Province/State', 'Country/Region']+features].replace(0, np.nan)
 column_null = df_per_country.isnull().mean()
column_null_drop = list(column_null[column_null > 0.5].index)
 df_per_country_clean = df_per_country.drop(
 column_null_drop, axis=1 ).fillna(df_per_country.median())
 return df.drop(features, axis=1 ).merge(df_per_country_clean,
on=['Province/State', 'Country/Region'], how='left' )<split>
y_preds = model.predict(dtest, verbose=1 )
def model(df, ylab):
features = df.drop(['Country/Region','Province/State',
'ConfirmedCases', 'Fatalities', 'Long', 'y_Fatalities', 'y_ConfirmedCases'
], axis=1)
 label = df[ylab]
train_df, test_df, train_label, test_label = train_test_split(
 features, label, test_size=0.2)
 model = XGBClassifier(objective='reg:linear')
 model.fit(train_df, train_label)
 test_pred = model.predict(test_df)
 return model, np.sqrt(mean_squared_error(test_label, test_pred))<feature_engineering>
df_sub.iloc[:, 1:] = y_preds
display(df_sub )
df = impute_values(prepare_data(train))
df.head(3 )<choose_model_class>
df_sub.to_csv('submission.csv',index=False )