Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<set_options>
warnings.simplefilter(action="ignore")
 seed = 13
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed )
del gp
del gp_df
gc.collect()<merge>
class TargetEncoder:
def __init__(self, target, alpha=5):
self.target = target
self.alpha = alpha
def fit_transform(self, train, categorical):
self.train = train
self.categorical = categorical
kf = KFold(n_splits=5, random_state=1, shuffle=True)
 train_feature = np.zeros(len(train))
for train_index, test_index in kf.split(train):
cv_train, cv_test = train.iloc[train_index], train.iloc[test_index]
cv_test_feature = self._test_mean_target_encoding(cv_train, cv_test)
 train_feature[test_index] = cv_test_feature
return train_feature
def transform(self, test):
test_feature = self._test_mean_target_encoding(self.train, test)
 return test_feature
def _test_mean_target_encoding(self, train, test):
global_mean = train[self.target].mean()
train_groups = train.groupby(self.categorical)
 category_sum = train_groups[self.target].sum()
category_size = train_groups.size()
train_statistics =(category_sum + global_mean * self.alpha)/(category_size + self.alpha)
 test_feature = test[self.categorical].map(train_statistics ).fillna(global_mean)
 return test_feature.values
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average="macro")
 return("macroF1", f1, True )
all_periods = all_periods.merge(all_samples, on='item_id', how='left')
 all_periods.head()<groupby>
def read_data(train_path, test_path):
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
 return train, test
gp = all_periods.groupby(['user_id'])[['days_up_sum', 'times_put_up']].mean().reset_index() \ 
.rename(index=str, columns={
'days_up_sum': 'avg_days_up_user',
'times_put_up': 'avg_times_up_user'
})
 gp.head()<merge>
def preprocess_data(train, test):
data = pd.concat([train, test], sort=False)
 features_to_drop = ["area2", "tamhog", "hhsize", "agesq"]
features_to_drop += [x for x in data.columns if "SQB" in x]
data.drop(features_to_drop, axis=1, inplace=True)
 for ohe in [
"pared",
"piso",
"techo",
"abastagua",
"sanitario",
"energcocinar",
"elimbasu",
"epared",
"etecho",
"eviv",
"lugar",
"tipovivi",
"electricity",
]:
if ohe != "electricity":
ohe_cols = [x for x in train.columns if x.startswith(ohe)]
else:
ohe_cols = ["public", "planpri", "noelec", "coopele"]
data[ohe] = np.where(
 data[ohe_cols].sum(axis=1)== 0, "NEW_CAT", data[ohe_cols].idxmax(axis=1)
)
 data.drop(ohe_cols, axis=1, inplace=True)
 data.fillna(-999, inplace=True)
 train = data[: len(train)]
test = data[-len(test):]
return train, test
n_user_items = all_samples.groupby(['user_id'])[['item_id']].count().reset_index() \ 
.rename(index=str, columns={
'item_id': 'n_user_items'
})
 gp = gp.merge(n_user_items, on='user_id', how='outer')
 gp.head()<save_to_csv>
def generate_features(train, test):
data = pd.concat([train, test], sort=False)
 data["adult"] = data["hogar_adul"] - data["hogar_mayor"]
data["dependency_count"] = data["hogar_nin"] + data["hogar_mayor"]
data["dependency"] = np.where(data["adult"] == 0, 1, data["dependency_count"] / data["adult"])
 data["child_percent"] = data["hogar_nin"] / data["hogar_total"]
data["elder_percent"] = data["hogar_mayor"] / data["hogar_total"]
data["adult_percent"] = data["hogar_adul"] / data["hogar_total"]
data["rent_per_bedroom"] = data["v2a1"] / data["bedrooms"]
data["male_per_bedroom"] = data["r4h3"] / data["bedrooms"]
data["female_per_bedroom"] = data["r4m3"] / data["bedrooms"]
data["bedrooms_per_person_household"] = data["hogar_total"] / data["bedrooms"]
data["escolari_age"] = data["escolari"] / data["age"]
aggr_mean_list = ["rez_esc", "dis", "male", "female"]
aggr_mean_list += [f"estadocivil{x}" for x in range(1, 8)]
aggr_mean_list += [f"parentesco{x}" for x in range(2, 13)]
aggr_mean_list += [f"instlevel{x}" for x in range(1, 10)]
other_list = ["escolari", "age", "escolari_age"]
for item in aggr_mean_list:
data[item + "_mean"] = data.groupby("idhogar")[item].transform("mean")
 for item in other_list:
for function in ["mean", "std", "min", "max", "sum"]:
data[item + "_" + function] =(
 data.groupby("idhogar")[item].transform(function ).fillna(0)
)
 train = data[: len(train)]
test = data[-len(test):]
return train, test
gp.to_csv('aggregated_features.csv', index=False )<drop_column>
train, test = read_data(
 train_path=".. /input/costa-rican-household-poverty-prediction/train.csv",
test_path=".. /input/costa-rican-household-poverty-prediction/test.csv",
)
del all_samples
del all_periods
del train
del test
gc.collect()<load_from_csv>
train, test = preprocess_data(train, test )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train = train.merge(gp, on='user_id', how='left')
 test = test.merge(gp, on='user_id', how='left')
 agg_cols = list(gp.columns)[1:]
del gp
gc.collect()
train.head()<count_missing_values>
train, test = generate_features(train, test )
train[agg_cols].isna().any(axis=1 ).sum() / len(train)* 100<count_missing_values>
train = train[train["parentesco1"] == 1]
target_encoder = preprocessing.LabelEncoder()
y = target_encoder.fit_transform(train["Target"])
 X = train.drop(["Id", "idhogar", "parentesco1"], axis=1)
 X_test = test.drop(["Id", "idhogar", "parentesco1", "Target"], axis=1 )
test[agg_cols].isna().any(axis=1 ).sum() / len(test)* 100<feature_engineering>
params = {
"learning_rate": 0.1,
"objective": "multiclass",
"metric": "multi_logloss",
"n_estimators": 1000,
"class_weight": "balanced",
"colsample_bytree": 0.9,
"subsample": 0.8,
"subsample_freq": 1,
"num_class": 4,
"lambda_l2": 1,
}
count = lambda l1,l2: sum([1 for x in l1 if x in l2])
 for df in [train, test]:
df['description'].fillna('unknowndescription', inplace=True)
 df['title'].fillna('unknowntitle', inplace=True)
 df['weekday'] = pd.to_datetime(df['activation_date'] ).dt.day
for col in ['description', 'title']:
df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))
df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100
df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100
df['city'] = df['region'] + '_' + df['city']
df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))
 for col in agg_cols:
df[col].fillna(-1, inplace=True )<feature_engineering>
num_folds = 5
skf = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=1)
 y_preds = np.zeros(( len(X_test), 4))
val_scores = []
for fold_n,(train_index, valid_index)in enumerate(skf.split(X, y)) :
X_train = X.iloc[train_index]
X_valid = X.iloc[valid_index]
y_train = y[train_index]
y_valid = y[valid_index]
for f in X_train.columns:
if X_train[f].dtype == "object" and f not in ["Id", "idhogar", "Target"]:
te = TargetEncoder(target="Target", alpha=5)
 X_train[f] = te.fit_transform(X_train, f)
 X_valid[f] = te.transform(X_valid)
 X_test[f] = te.transform(X_test)
 X_train = X_train.drop(["Target"], axis=1)
 X_valid = X_valid.drop(["Target"], axis=1)
 clf = lgb.LGBMClassifier(**params)
 clf = clf.fit(
 X_train,
y_train,
eval_set=[(X_valid, y_valid)],
verbose=100,
early_stopping_rounds=200, 
)
 y_pred_valid = clf.predict(X_valid)
 importances = pd.DataFrame({"feature": X_train.columns, "importance": clf.feature_importances_})
 score = f1_score(y_valid, y_pred_valid, average="macro")
 val_scores.append(score)
 print(f"Fold {fold_n}.F1 Score: {score:.5f}
")
 y_preds += clf.predict_proba(X_test)/ num_folds
print("Overall F1 Score: {:.3f}".format(np.mean(val_scores)+ np.std(val_scores)) )
count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=25)
 title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))
train_title_counts = title_counts[:len(train)]
test_title_counts = title_counts[len(train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'),
lowercase=True, ngram_range=(1, 2),
max_features=15000)
 desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))
train_desc_counts = desc_counts[:len(train)]
test_desc_counts = desc_counts[len(train):]
train_title_counts.shape, train_desc_counts.shape<define_variables>
y_preds = np.argmax(y_preds, axis=1)
 submission = pd.DataFrame(
 {
"Id": test["Id"],
"Target": target_encoder.inverse_transform(y_preds ).astype(int),
} 
)
 submission.to_csv("submission.csv", index=False )
target = 'deal_probability'
predictors = [
'num_desc_punct',
'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',
'avg_times_up_user', 'avg_days_up_user', 'n_user_items',
'price', 'item_seq_number'
]
categorical = [
'image_top_1', 'param_1', 'param_2', 'param_3',
'city', 'region', 'category_name', 'parent_category_name', 'user_type'
]
predictors = predictors + categorical<categorify>
train.Target.value_counts()
for feature in categorical:
print(f'Transforming {feature}...')
 encoder = LabelEncoder()
encoder.fit(train[feature].append(test[feature] ).astype(str))
train[feature] = encoder.transform(train[feature].astype(str))
test[feature] = encoder.transform(test[feature].astype(str))<init_hyperparams>
submission.Target.value_counts()