Kaggle competition: Santander Product Recommendation
col_corr = []
for col in num_col:
na_idx = pd.isnull(train[col])
 corr = np.corrcoef(x= train[col][~na_idx], y=train['SalePrice'][~na_idx])[0,1]
col_corr.append(( col, corr))
col_corr.sort(key=lambda x : -abs(x[1]))<statistical_test>
start_time = datetime.datetime.now()
demographic_cols = ['fecha_dato',
'ncodpers','ind_empleado','pais_residencia','sexo','age','fecha_alta','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov"]
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
train_cols = demographic_cols + product_col
col_aov = []
for col in cat_col:
result = ols('SalePrice ~ {}'.format(col), data = train ).fit()
aov_table = sm.stats.anova_lm(result)
 aov_pr = aov_table['PR(>F)'][0]
col_aov.append(( col, aov_pr))
col_aov.sort(key = lambda x: x[1] )<sort_values>
df_train = pd.read_csv('.. /input/makejuneextradata-fff8dd/juneExtra.csv', usecols=train_cols )
high_cor_list = []
tmp = corr_table[abs(corr_table)>0.4]
for col in tmp.columns:
for row in tmp[col][~pd.isnull(tmp[col])].index:
if col == row:
break
high_cor_list.append(( col,row, tmp[col][row]))
high_cor_list.sort(key = lambda x : -x[2])
 high_cor_list<sort_values>
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )
high_cor_list = []
tmp = corr_table[abs(corr_table)>0.4]
for col in tmp.columns:
for row in tmp[col][~pd.isnull(tmp[col])].index:
if col == row:
break
high_cor_list.append(( col,row, tmp[col][row]))
high_cor_list.sort(key = lambda x : -x[2])
 high_cor_list<normalization>
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']
scaler = StandardScaler()
tmp = scaler.fit_transform(train[['SalePrice']])
 tmp_sorted = sorted(np.squeeze(tmp))
for a, b in zip(tmp_sorted[:10], tmp_sorted[-10:]):
print('{} {} {}'.format(round(a, 5), ' '*10, round(b,5)) )<filter>
df_train.isnull().sum()
train[tmp>7]<load_from_csv>
df_train.tiprel_1mes.value_counts()
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<concatenate>
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )
test['SalePrice'] = 0
data = pd.concat([train,test], axis=0)
 data.reset_index(drop=True, inplace=True )<data_type_conversions>
df_train["renta"] = pd.to_numeric(df_train["renta"], errors="coerce")
 df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce" )
data['MSZoning'].fillna(data['MSZoning'].mode() [0], inplace=True)
 data['Utilities'].fillna(data['Utilities'].mode() [0], inplace=True)
 data['Exterior1st'].fillna(data['Exterior1st'].mode() [0], inplace=True)
 data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0], inplace=True)
 data['MasVnrType'].fillna('None', inplace=True)
 data['MasVnrArea'].fillna(0, inplace=True)
 data['BsmtQual'].fillna('None', inplace=True)
 data['BsmtCond'].fillna('None', inplace=True)
 data['BsmtExposure'].fillna('None', inplace=True)
 data['BsmtFinType1'].fillna('None', inplace=True)
 data['BsmtFinSF1'].fillna(0, inplace=True)
 data['BsmtFinType2'].fillna('None', inplace=True)
 data['BsmtFinSF2'].fillna(0, inplace=True)
 data['BsmtUnfSF'].fillna(0, inplace=True)
 data['TotalBsmtSF'].fillna(0, inplace=True)
 data['Electrical'].fillna(data['Electrical'].mode() [0], inplace=True)
 data['BsmtFullBath'].fillna(0, inplace=True)
 data['BsmtHalfBath'].fillna(0, inplace=True)
 data['KitchenQual'].fillna(data['KitchenQual'].mode() [0], inplace=True)
 data['Functional'].fillna('None', inplace=True)
 data['GarageType'].fillna('None', inplace=True)
 data['GarageFinish'].fillna('None', inplace=True)
 data['GarageYrBlt'].fillna(data['GarageYrBlt'].mode() [0], inplace=True)
 data['GarageArea'].fillna(0, inplace=True)
 data['GarageCars'].fillna(0, inplace=True)
 data['GarageQual'].fillna('None', inplace=True)
 data['SaleType'].fillna(data['SaleType'].mode() [0], inplace=True)
 data['GarageCond'].fillna('None', inplace=True)
 data['PoolQC'].fillna('None', inplace=True)
 data['Fence'].fillna('None', inplace=True)
 data['MiscFeature'].fillna('None', inplace=True)
 data['LotFrontage'].fillna(0, inplace=True)
 data['Alley'].fillna('None', inplace=True)
 data['FireplaceQu'].fillna('None', inplace=True )<drop_column>
for cod in unique_prov:
df_test.loc[df_test['cod_prov']==cod,['renta']] = df_test.loc[df_test['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df_train.loc[df_train['cod_prov']==cod,['renta']] = df_train.loc[df_train['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
cols = ['MiscFeature', 'Alley', 'PoolQC', 'Fence', 'FireplaceQu', 'GarageCond', 'LotFrontage']
data.drop(cols, axis=1, inplace=True )<feature_engineering>
df_test.renta.fillna(df_test["renta"].median() , inplace=True )
data['Total_SF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
data['Total_Bath'] = data['BsmtFullBath'] + data['BsmtHalfBath']*0.5 + data['FullBath'] + data['HalfBath']*0.5
data['Total_Footage'] = data['BsmtFinSF1'] + data['BsmtFinSF2'] + data['1stFlrSF'] + data['2ndFlrSF']
data['Age'] = 2020 - data['YearBuilt']
data['Age_rmd'] = 2020 - data['YearRemodAdd']
data['Age_Garage'] = 3020 - data['GarageYrBlt']
data['qul_grliv'] = data['OverallQual'] * data['GrLivArea']
data['garage'] = data['GarageCars'] * data['GarageArea']
data['Years'] = data['YearBuilt'] + data['YearRemodAdd'] + data['GarageYrBlt']
data['Age_not_rmd'] = data['Age'] - data['Age_rmd']
data['Overall'] = data['OverallQual'] + data['OverallCond']
data['porch'] = data['OpenPorchSF'] + data['EnclosedPorch'] + data['ScreenPorch']
data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 data['Season'] = data['MoSold'].apply(lambda x: 'Spring' if x<= 3 else('Summer' if x <=6 else('Fall' if x <=9 else 'Winter')))
 data.drop(['MoSold'], axis=1, inplace=True )<data_type_conversions>
df_train.isnull().sum()
to_cat = ['MSSubClass']
data[to_cat] = data[to_cat].astype(str )<define_variables>
df_test.isnull().sum()
num_col = []
cat_col = []
for col in data.columns:
if col == "Id" or col == 'SalePrice':
continue
elif data[col].dtype == 'object':
cat_col.append(col)
 else:
num_col.append(col)
 print(len(num_col))
print(len(cat_col))
print(len(data.columns))<categorify>
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )
for col in cat_col:
data = pd.concat([data,pd.get_dummies(data[col], prefix=(col)) ], axis=1)
 data.drop([col], axis=1, inplace=True )<feature_engineering>
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']
skews = []
for col in num_col:
skews.append(( col, skew(data[col])))
 skews.sort(key=lambda x : -abs(x[1]))
for col, value in skews:
if abs(value)>0.5:
data[col] = boxcox1p(data[col], 0.15)
 data['SalePrice'] = np.log1p(data['SalePrice'] )<drop_column>
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 print(use_col)
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
print(row_index)
 df_train.loc[row_index,col] = np.nan
print(df_train.loc[row_index,col])
 for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan
cols = ['GarageArea', 'GarageYrBlt', 'TotRmsAbvGrd', '2ndFlrSF', 'BsmtFullBath']
data.drop(cols, axis=1, inplace=True)
 num_col = list(set(num_col)-set(cols))<count_values>
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )
cols = []
for col in data.columns:
major_ratio = data[col].value_counts().iloc[0]/len(data[col])
 if major_ratio > 0.999:
cols.append(col)
 data.drop(cols, axis=1, inplace=True)
 print('<prepare_x_and_y>
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )
data.reset_index(drop=True, inplace=True)
 train = data[data['Id']<=1460]
test = data[data['Id'] >1460]
X = list(set(train.columns)- set(['SalePrice']))
y = 'SalePrice'<compute_train_metric>
models = {}
model_preds = {}
id_preds1 = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = LogisticRegression()
clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds1[id].append(p)
def rmse_cv(model):
result = cross_val_score(cv=5, estimator=model, X=train[X].values, y=train[y].values, scoring='neg_mean_squared_error')
 result =(-result)**(1/2)
 return np.mean(result), np.std(result)
<compute_test_metric>
models = {}
model_preds = {}
id_preds3 = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = ensemble.RandomForestClassifier(n_estimators=60, n_jobs=-1,max_depth=10, min_samples_split=10, verbose=1)
 clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds3[id].append(p)
def rmsle(pred, actual):
result = 0
for i in range(len(pred)) :
result +=(pred[i] - actual[i])**2
result /= len(pred)
 return result**(1/2 )<choose_model_class>
fraction1 = 0.4
fraction3 = 0.6
id_preds_combined = {}
for uid, p in id_preds1.items() :
id_preds_combined[uid] = fraction1*np.asarray(id_preds1[uid])+ fraction3*np.asarray(id_preds3[uid])
 id_preds = id_preds_combined
EN = make_pipeline(RobustScaler() , ElasticNet(alpha=1e-3, l1_ratio=0.7, max_iter=1e+4))
rmse_cv(EN)
<choose_model_class>
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
lasso = make_pipeline(RobustScaler() , Lasso(alpha=5e-04, max_iter=1e+04))
rmse_cv(lasso )<choose_model_class>
sample = pd.read_csv('.. /input/santander-product-recommendation/sample_submission.csv' )
KRR = KernelRidge(alpha=1e-2, kernel='polynomial', degree=1, coef0=2)
 rmse_cv(KRR )<choose_model_class>
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p))
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=30, min_samples_split=10,
loss='huber', random_state =5)
<choose_model_class>
sample['added_products'] = test_preds
model_xgb = xgb.XGBRegressor(colsample_bytree=0.3, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.3, n_estimators=1024,
reg_alpha=0.3, reg_lambda=0.4,
subsample=0.5, silent=1,
random_state =7, nthread = -1)
<choose_model_class>
sample.to_csv('LR-RF_sub.csv', index=False)
 print(datetime.datetime.now() -start_time)