Kaggle competition: Santander Product Recommendation
def decision(cabin_val, num):
if cabin_val == 'X':
return -1
elif num > 1:
return 1
else:
return 0
def detect_share(data, col_analysis, new_column):
_g = data.groupby(by=[col_analysis] ).agg({
'PassengerId': 'size',
} ).reset_index()
_g[new_column] = _g[[col_analysis, 'PassengerId']].apply(
 lambda _: decision(_[0], _[1]), axis=1 
)
 data = data.merge(_g.drop(['PassengerId'], axis=1), on=col_analysis, how='left')
 del _g
return data<drop_column>
start_time = datetime.datetime.now()
demographic_cols = ['fecha_dato',
'ncodpers','ind_empleado','pais_residencia','sexo','age','fecha_alta','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov"]
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
train_cols = demographic_cols + product_col
del train['Ticket']
del test['Ticket']
del train['Cabin']
del test['Cabin']<feature_engineering>
df_train = pd.read_csv('.. /input/makejuneextradata-fff8dd/juneExtra.csv', usecols=train_cols )
train['total_family'] = train['SibSp'] + train['Parch']
test['total_family'] = test['SibSp'] + test['Parch']
view_numeric(train, 'total_family', col_target )<feature_engineering>
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )
train['family_greater_than_3'] = train[['SibSp', 'Parch']].apply(lambda _: 1 if _[0] + _[1] > 3 else 0, axis=1)
 test['family_greater_than_3'] = test[['SibSp', 'Parch']].apply(lambda _: 1 if _[0] + _[1] > 3 else 0, axis=1)
 view_cat(train, 'family_greater_than_3', col_target )<feature_engineering>
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']
def _range_edad(_):
if _ < 0:
return 'sin edad'
elif _ < 10:
return 'ninno'
elif _ < 20:
return 'adolescente'
elif _ < 30:
return 'joven'
elif _ < 40:
return 'adulto'
elif _ < 50:
return 'adultomayor'
else:
return 'anciano'
train['range_edad'] = train['Age'].apply(_range_edad)
 test['range_edad'] = test['Age'].apply(_range_edad)
 view_cat(train, 'range_edad', 'Survived' )<prepare_x_and_y>
df_train.isnull().sum()
y_train = train['Survived'].copy()
train = train.drop(['PassengerId', 'Survived'], axis=1)
 test = test.drop(['PassengerId'], axis=1 )<categorify>
df_train.tiprel_1mes.value_counts()
train = pd.get_dummies(train, drop_first=True, columns=['Sex'])
 test = pd.get_dummies(test, drop_first=True, columns=['Sex'] )<categorify>
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )
train = pd.get_dummies(train, drop_first=False)
 test = pd.get_dummies(test, drop_first=False )<sort_values>
df_train["renta"] = pd.to_numeric(df_train["renta"], errors="coerce")
 df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce" )
abs(_corr['Survived'] ).sort_values(ascending=False )<define_variables>
for cod in unique_prov:
df_test.loc[df_test['cod_prov']==cod,['renta']] = df_test.loc[df_test['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df_train.loc[df_train['cod_prov']==cod,['renta']] = df_train.loc[df_train['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
list_var_relevants = ['Age', 'SibSp', 'Fare', 'Mr', 'Mrs', 'shared_cabin', 'family_greater_than_3',
'total_family', 'words_in_name', 'Sex_male', 'Pclass_1', 'Pclass_3', 'Embarked_S',
'range_edad_joven', 'range_edad_ninno']<define_search_space>
df_test.renta.fillna(df_test["renta"].median() , inplace=True )
c_values = list(np.logspace(-1, 0.5, 50))
print(c_values )<choose_model_class>
df_train.isnull().sum()
param_grid_log = {
'penalty': ['l2'],
'C': c_values,
'class_weight': ['balanced', None],
'max_iter': [75],
'solver': ['lbfgs']
}
kfold_on_rf = StratifiedKFold(
 n_splits=3,
shuffle=False,
random_state=SEED 
)
 model_log = LogisticRegression(random_state=SEED, n_jobs = 4 )<train_on_grid>
df_test.isnull().sum()
def apply_grid(X_train, y_train, model, param_grid, kfold_):
grid = RandomizedSearchCV(
 model,
{k: [v] if not isinstance(v, list)else v for k, v in param_grid.items() },
cv=kfold_,
n_jobs=-1,
scoring='accuracy',
verbose=1,
n_iter=1500 
)
 grid.fit(X_train, y_train)
 print(grid.best_score_, end=' / ')
 return grid.best_estimator_, grid.best_score_, grid.best_params_<compute_train_metric>
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )
result_0_1_train = best_model.predict(train[list_var_relevants])
 acu = accuracy_score(y_train, result_0_1_train)
 rec = recall_score(y_train, result_0_1_train)
 print(acu, rec)
 print(classification_report(y_train, result_0_1_train))<count_values>
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']
sub['Survived'] = best_model.predict(test[list_var_relevants])
 sub['Survived'].value_counts()<save_to_csv>
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 print(use_col)
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
print(row_index)
 df_train.loc[row_index,col] = np.nan
print(df_train.loc[row_index,col])
 for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan
_date = str(datetime.now() ).split('.')[0].replace('-', '_' ).replace(' ', '_' ).replace(':', '_')
 sub.to_csv('result_log_lassocv_{}_{}_{}.csv'.format(
 round(acu, 4),round(rec, 4),_date 
), index=False )<predict_on_test>
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )
result_prob_train = best_model.predict_proba(train[list_var_relevants])[:,1]
accuracy_score(y_train, np.array([0 if _ < 0.5 else 1 for _ in result_prob_train]))<compute_test_metric>
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )
optimization = differential_evolution(
 lambda c: -1*accuracy_score(y_train, np.array([0 if _ < c[0] else 1 for _ in result_prob_train])) ,
[(0, 1)] 
)
 optimization<compute_test_metric>
models = {}
model_preds = {}
id_preds = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = ensemble.RandomForestClassifier(n_estimators=60, n_jobs=-1,max_depth=10, min_samples_split=10, verbose=1)
 clf.fit(x_train, y_train)
 p_train1 = clf.predict_proba(x_train)[:,1]
x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds[id].append(p)
result_0_1_train_opt = np.array([0 if _ < optimization["x"][0] else 1 for _ in result_prob_train])
 acu = accuracy_score(y_train, result_0_1_train_opt)
 rec = recall_score(y_train, result_0_1_train_opt)
 print(acu, rec)
 print(classification_report(y_train, result_0_1_train_opt))<predict_on_test>
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
 df_recent = df_recent.drop_duplicates(['ncodpers'], keep='last' )
result_0_1_test_opt = np.array(
 [0 if _ < optimization["x"][0] else 1 for _ in best_model.predict_proba(test[list_var_relevants])[:,1]] 
)
 sub['Survived'] = result_0_1_test_opt
sub['Survived'].value_counts()<save_to_csv>
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col )
_date = str(datetime.now() ).split('.')[0].replace('-', '_' ).replace(' ', '_' ).replace(':', '_')
 sub.to_csv('result_loglassocv_opt_{}_{}_{}.csv'.format(
 round(acu, 5),round(rec, 5),_date 
), index=False )<import_modules>
sample = pd.read_csv('.. /input/santander-product-recommendation/sample_submission.csv' )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<set_options>
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p))
sns.set_style('darkgrid')
 pd.set_option('display.max_columns', 500 )<load_from_csv>
sample['added_products'] = test_preds
<sort_values><EOS> 
sample.to_csv('RForest_demographic_sub.csv', index=False)
 print(datetime.datetime.now() -start_time)