Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<create_dataframe>
warnings.filterwarnings("ignore")
df_test_predict_proba_1 = pd.concat(
 [test, pd.DataFrame(softmax(tst_preds, axis = 1)) ],
axis=1
).sort_values(["image_id"] )<drop_column>
raw_train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv', index_col='id')
 raw_test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv', index_col='id')
 print(raw_train.shape, raw_test.shape )
variable_list = %who_ls
for _ in variable_list:
if _ is not "df_test_predict_proba_1":
del globals() [_]
%who_ls<define_variables>
df_train = raw_train.dropna(subset=['month', 'day'])[['day', 'month', 'target']]
df_test = raw_test.dropna(subset=['month', 'day'])[['day', 'month']]
df0 = df_train[df_train.target == 0]
df1 = df_train[df_train.target == 1]
def number2datetime(df):
time_col = '2019/' + df.month.astype(int ).astype(str)+ '/' + df.day.astype(int ).astype(str)
 df['time'] = pd.to_datetime(time_col , format = '%Y/%m/%d')
 df = df.drop(columns=['day', 'month'])
 return df
df0 = number2datetime(df0)
 df1 = number2datetime(df1)
 df_test = number2datetime(df_test )
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/cassava-resnext50-32x4d-weights/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<define_search_space>
count0 = df0.time.value_counts() /len(df0)
 count0 = count0.sort_index()
count1 = df1.time.value_counts() /len(df1)
 count1 = count1.sort_index()
count_test = df_test.time.value_counts(normalize=True )
class CFG:
debug=False
num_workers=8
model_name='resnext50_32x4d'
size=512
batch_size=32
seed=2020
target_size=5
target_col='label'
n_fold=5
trn_fold=[0, 1, 2, 3, 4]
inference=True<import_modules>
full_data = pd.concat([raw_train, raw_test], sort=False ).drop(columns='target')
 full_data.shape
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 warnings.filterwarnings('ignore' )<compute_test_metric>
cate_columns = full_data.select_dtypes(include=['object'] ).columns
for col in cate_columns:
train_values = set(raw_train[col].unique())
 test_values = set(raw_test[col].unique())
 xor_values = test_values - train_values
if xor_values:
print(f'Replace {len(xor_values)} in {col} column')
 print('They are: ', xor_values)
 print()
full_data.loc[full_data[col].isin(xor_values), col] = 'xor'
def get_score(y_true, y_pred):
return accuracy_score(y_true, y_pred)
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed )<load_from_csv>
map_ord1 = {'Novice':1,
'Contributor':2,
'Expert':4,
'Master':5,
'Grandmaster':6}
full_data.ord_1 = full_data.ord_1.map(map_ord1 )
test = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 test.head()<normalization>
map_ord2 = {'Freezing':1,
'Cold':2,
'Warm':3,
'Hot':4,
'Boiling Hot':5,
'Lava Hot':6}
full_data.ord_2 = full_data.ord_2.map(map_ord2 )
def get_transforms(*, data):
if data == 'valid':
return A.Compose([
A.Resize(CFG.size, CFG.size),
A.Transpose(p=0.5),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.ShiftScaleRotate(p=0.5),
A.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2()
] )<choose_model_class>
full_data['ord_3_by_ord'] = full_data.ord_3.map(ord, na_action='ignore')
 map_ord3 = {key:value for value,key in enumerate(sorted(full_data.ord_3.dropna().unique())) }
full_data.ord_3 = full_data.ord_3.map(map_ord3 )
class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x<load_pretrained>
full_data['ord_4_by_ord'] = full_data.ord_4.map(ord, na_action='ignore')
 map_ord4 = {key:value for value,key in enumerate(sorted(full_data.ord_4.dropna().unique())) }
full_data.ord_4 = full_data.ord_4.map(map_ord4 )
def load_state(model_path):
model = CustomResNext(CFG.model_name, pretrained=False)
 try:
model.load_state_dict(torch.load(model_path)['model'], strict=True)
 state_dict = torch.load(model_path)['model']
except:
state_dict = torch.load(model_path)['model']
state_dict = {
k[7:]
if k.startswith('module.')
 else k: state_dict[k]
for k in state_dict.keys()
}
return state_dict
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state)
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<load_pretrained>
full_data['ord_5_1'] = full_data.ord_5.map(lambda string: ord(string[0]), na_action='ignore')
 full_data['ord_5_2'] = full_data.ord_5.map(lambda string: ord(string[1]), na_action='ignore')
 map_ord5 = {key:value for value,key in enumerate(sorted(full_data.ord_5.dropna().unique())) }
full_data.ord_5 = full_data.ord_5.map(map_ord5 )
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(
 test_dataset,
batch_size=CFG.batch_size,
shuffle=False,
num_workers=CFG.num_workers,
pin_memory=True 
)
 predictions = inference(model, states, test_loader, device )<create_dataframe>
num_columns = full_data.select_dtypes(exclude=['object'] ).columns.drop(['bin_0', 'bin_1', 'bin_2'])
 cate_columns = full_data.columns.drop(num_columns )
df_test_predict_proba_2 = pd.concat(
 [test["image_id"], pd.DataFrame(softmax(predictions, axis = 1)) ],
axis=1
).sort_values(["image_id"] )<feature_engineering>
missing_num_columns = [col for col in num_columns if any(full_data[col].isnull())]
for col in missing_num_columns:
full_data[col+'_is_missing'] = full_data[col].isnull().astype(int )
submission = df_test_predict_proba_1[["image_id"]]
submission["label"] =(
 df_test_predict_proba_1.drop(["image_id"], axis=1)* 0.5
+ df_test_predict_proba_2.drop(["image_id"], axis=1)* 0.5
).to_numpy().argmax(1)
<save_to_csv>
time_cols = ['day', 'month']
for col in time_cols:
full_data[col+'_sin'] = np.sin(2*np.pi*full_data[col]/7)
 full_data[col+'_cos'] = np.cos(2*np.pi*full_data[col]/12)
 full_data = full_data.drop(columns=time_cols )
submission.to_csv("submission.csv", index=False )<save_to_csv>
print(f"One-Hot encoding {len(OH_cols)} columns")
 OH_full = pd.get_dummies(
 full_data,
columns=OH_cols,
drop_first=True,
dummy_na=True,
sparse=True,
).sparse.to_coo()
submission.to_csv("submission.csv", index=False )<import_modules>
imputer = SimpleImputer(strategy='mean')
 retain_full = pd.DataFrame(imputer.fit_transform(full_data[retain_cols]), columns=retain_cols)
 retain_full = retain_full/retain_full.max()
import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn import linear_model
import os<load_from_csv>
encoded_full = scipy.sparse.hstack([OH_full, retain_full, retain_full**2] ).tocsr()
print(encoded_full.shape)
 encoded_train = encoded_full[:len(raw_train)]
encoded_test = encoded_full[len(raw_train):]
train = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv")
 sample = pd.read_csv(".. /input/cat-in-the-dat-ii/sample_submission.csv")
 train.tail()<categorify>
model = LogisticRegression(C=0.03, max_iter=300 )
class CategoricalFeatures:
def __init__(self, df, categorical_features, encoding_type, handle_na=False):

 self.df = df
self.cat_feats = categorical_features
self.enc_type = encoding_type
self.handle_na = handle_na
self.label_encoders = dict()
self.binary_encoders = dict()
self.ohe = None
if self.handle_na:
for c in self.cat_feats:
self.df.loc[:, c] = self.df.loc[:, c].astype(str ).fillna("-9999999")
 self.output_df = self.df.copy(deep=True)
 def _label_encoding(self):
for c in self.cat_feats:
lbl = preprocessing.LabelEncoder()
lbl.fit(self.df[c].values)
 self.output_df.loc[:, c] = lbl.transform(self.df[c].values)
 self.label_encoders[c] = lbl
return self.output_df
def _label_binarization(self):
for c in self.cat_feats:
lbl = preprocessing.LabelBinarizer()
lbl.fit(self.df[c].values)
 val = lbl.transform(self.df[c].values)
 self.output_df = self.output_df.drop(c, axis=1)
 for j in range(val.shape[1]):
new_col_name = c + f"__bin_{j}"
self.output_df[new_col_name] = val[:, j]
self.binary_encoders[c] = lbl
return self.output_df
def _one_hot(self):
ohe = preprocessing.OneHotEncoder()
ohe.fit(self.df[self.cat_feats].values)
 return ohe.transform(self.df[self.cat_feats].values)
 def fit_transform(self):
if self.enc_type == "label":
return self._label_encoding()
elif self.enc_type == "binary":
return self._label_binarization()
elif self.enc_type == "ohe":
return self._one_hot()
else:
raise Exception("Encoding type not understood")
 def transform(self, dataframe):
if self.handle_na:
for c in self.cat_feats:
dataframe.loc[:, c] = dataframe.loc[:, c].astype(str ).fillna("-9999999")
 if self.enc_type == "label":
for c, lbl in self.label_encoders.items() :
dataframe.loc[:, c] = lbl.transform(dataframe[c].values)
 return dataframe
elif self.enc_type == "binary":
for c, lbl in self.binary_encoders.items() :
val = lbl.transform(dataframe[c].values)
 dataframe = dataframe.drop(c, axis=1)
 for j in range(val.shape[1]):
new_col_name = c + f"__bin_{j}"
dataframe[new_col_name] = val[:, j]
return dataframe
elif self.enc_type == "ohe":
return self.ohe(dataframe[self.cat_feats].values)
 else:
raise Exception("Encoding type not understood" )<concatenate>
train_len = len(train)
 test["target"] = -1
full_data = pd.concat([train, test])
<categorify>
%%time
model = LogisticRegression(C=0.03, max_iter=300)
 model.fit(encoded_train, raw_train.target)
 test_pred = model.predict_proba(encoded_test)[:, 1]
<prepare_x_and_y><EOS> 
%%time
submiss = pd.DataFrame({"id": raw_test.index, "target": test_pred})
 submiss.to_csv('Phan_Viet_Hoang.csv', index=False )