Kaggle competition: Santander Product Recommendation
submission.to_csv("submission_BERT.csv", index=False, header=True )<set_options>
start_time = datetime.datetime.now()
demographic_cols = ['ncodpers','fecha_alta','ind_empleado','pais_residencia','sexo','age','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov",'fecha_dato']
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
warnings.filterwarnings('ignore')
 seed = 12345
np.random.seed(seed)
 random.seed(seed)
 pd.set_option('display.expand_frame_repr', False)
 pd.set_option('max_colwidth', -1)
 plt.style.use('ggplot' )<load_from_csv>
df_train = pd.read_csv('.. /input/datamulticlass-6-withpast2/DataMulticlass_6_withpast2.csv' )
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 full = train.append(test)
 full.sort_values('PassengerId', inplace=True)
 full.reset_index(drop=True, inplace=True )<feature_engineering>
df_test = pd.read_csv('.. /input/testset-withpast3/TestSet_withpast3.csv' )
full[['Last','Full']] = full['Name'].str.split(r", ",expand=True)
 full[['Title','Rest']] = full['Full'].str.split(r'(?<=\.) \s',n=1, expand=True)
 full[['First', 'Parenthesis', 'blank']] = full['Rest'].str.split(r"\((.*)\)", expand=True)
 full['blank'].unique()
full = full.drop('blank', axis=1)
 full[['drop', 'Parenthesis2', 'blank']] = full['Rest'].str.split(r'\(" (.*)"\)', expand=True)
 full['blank'].unique()
full = full.drop(['blank','drop'], axis=1)
 full[['First', 'Quote', 'blank']] = full['First'].str.split(r"" (.*)"", expand=True)
 full['Quote'].replace('', None, inplace=True)
 full['blank'].unique()
full = full.drop('blank', axis=1)
 full['First'] = full['First'].str.strip()
full['Last'] = full['Last'].str.strip()
full['Title'] = full['Title'].str.strip()
full.drop(['Full', 'Rest'], axis=1, inplace=True)
 full['hasNickname'] = ~full['Quote'].isnull()
full['hasParenthesis'] = ~full['Parenthesis'].isnull()
print(full.hasNickname.value_counts())
 print(full.hasParenthesis.value_counts())
 full = full.drop(['Parenthesis','Parenthesis2', 'Quote'], axis='columns' )<count_unique_values>
pd.set_option('display.max_columns', None )
cab_dum = full['Cabin'].str.get_dummies(' ')
 cab_occupancy =cab_dum.sum(axis=0)
 occ_multiple = cab_occupancy[cab_occupancy.values>1].index
cab_cSum = cab_dum.sum(axis=1)
 cab_multiple = cab_cSum[cab_cSum.values>1].index
full_cab = pd.concat([full, cab_dum.loc[:, occ_multiple]], axis=1)
 df_temp = pd.DataFrame([])
 for c in occ_multiple:
n = full[full_cab[c]>0].Cabin.nunique()
if n>1:
df_temp = df_temp.append(full[full_cab[c]>0])
 df_temp[['Cabin','Embarked', 'Fare', 'Pclass','Name', 'Ticket']].head(20 )<feature_engineering>
def filter_data(df):
df = df[df['ind_nuevo'] == 0]
df = df[df['antiguedad'] != -999999]
df = df[df['indrel'] == 1]
df = df[df['indresi'] == 'S']
df = df[df['indfall'] == 'N']
df = df[df['tipodom'] == 1]
df = df[df['ind_empleado'] == 'N']
df = df[df['pais_residencia'] == 'ES']
df = df[df['indrel_1mes'] == 1]
df = df[df['tiprel_1mes'] ==('A' or 'I')]
df = df[df['indext'] == 'N']
full.Cabin = full.Cabin.str.replace('F ', 'F' )<count_unique_values>
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )
temp = full.groupby('Ticket' ).Cabin.nunique().sort_values(ascending=False)
 print(temp.value_counts() )<sort_values>
df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce")
 unique_prov = df_test[df_test.cod_prov.notnull() ].cod_prov.unique()
grouped = df_test.groupby("cod_prov")["renta"].median()
def impute_renta(df):
df["renta"] = pd.to_numeric(df["renta"], errors="coerce")
 for cod in unique_prov:
df.loc[df['cod_prov']==cod,['renta']] = df.loc[df['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df.renta.fillna(df_test["renta"].median() , inplace=True)
 impute_renta(df_train)
 impute_renta(df_test )
temp = full[full.groupby('Ticket' ).Cabin.transform(lambda x: x.fillna('' ).nunique())>1]
temp.sort_values(by=['Ticket','Cabin'], inplace=True)
 temp[['Cabin','Embarked', 'Fare', 'Pclass','Name', 'Ticket']].head(10 )<data_type_conversions>
def drop_na(df):
df.dropna(axis = 0, subset = ['ind_actividad_cliente'], inplace = True)
 drop_na(df_train )
full['Embarked'] = full.Embarked.fillna(full.Embarked.mode() [0])
 full['ppFare'] = full.groupby(['Pclass', 'Embarked'] ).ppFare.transform(lambda x: x.fillna(x.median()))
FareToFill = full.ppFare * full.ppTicket
full.loc[full.Fare.isna() , 'Fare'] = FareToFill[full.Fare.isna() ]<categorify>
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']
dummy_col_select = ['canal_entrada','cod_prov']
full['nFamily'] = full.groupby(['Last', 'Ticket_Type', 'Embarked'] ).Last.transform('count')
 to_impute = full[['Age', 'ppFare', 'Parch', 'Pclass', 'SibSp', 'hasNickname', 'hasParenthesis', 'ppTicket', 'nFamily']]
dummy = pd.get_dummies(data=full[['Title','Embarked', 'Sex','Ticket_Type']])
 to_impute = pd.concat([to_impute,dummy], axis=1)
 col = to_impute.columns
imp = IterativeImputer(estimator=ExtraTreesRegressor(n_estimators=100), max_iter=50, verbose=2)
 imp.fit(to_impute)
 imputed = pd.DataFrame(imp.transform(to_impute))
imputed.columns = col
imputed['PassengerId'] = full['PassengerId']
imputed['Age']=np.round(imputed['Age'], decimals=1)
 full.update(imputed[['PassengerId','Age']])
 full.info()<feature_engineering>
limit = int(0.01 * len(df_train.index))
use_dummy_col = {}
for col in dummy_col_select:
trainlist = df_train[col].value_counts()
use_dummy_col[col] = []
for i,item in enumerate(trainlist):
if item > limit:
use_dummy_col[col].append(df_train[col].value_counts().index[i] )
full['Title'] = full['Title'].replace(['Don.', 'Rev.', 'Jonkheer.', 'Sir.'], 'Honor')
 full['Title'] = full['Title'].replace(['Dr.', 'Col.', 'Major.', 'Capt.'], 'Profession')
 full['Title'] = full['Title'].replace(['Ms.', 'Mlle.', 'the Countess.', 'Lady.'], 'Miss.')
 full['Title'] = full['Title'].replace(['Mme.', 'Dona.'], 'Mrs.')
 full.Title.value_counts()
full['Cabin_Level'] = full.groupby(['ppFare', 'Embarked', 'Pclass'] ).Cabin_Level.transform(lambda x: x.replace('T', x.mode() [0]))
full['Cabin_Level'].cat.remove_unused_categories(inplace=True)
 print(full.Cabin_Level.value_counts())
 full['Cabin_Level'] = full['Cabin_Level'].replace('G', 'F')
 full['Cabin_Level'].cat.remove_unused_categories(inplace=True)
<feature_engineering>
def get_dummy(df):
for col in dummy_col_select:
for item in df[col].unique() :
if item not in use_dummy_col[col]:
row_index = df[col] == item
df.loc[row_index,col] = np.nan
return pd.get_dummies(df, prefix=dummy_col, columns = dummy_col)
 df_train = get_dummy(df_train)
 df_test = get_dummy(df_test )
full['isMarried'] = full.Title.isin(['Mrs.', 'Mme.', 'Dona.'])
 full['FamilySize'] = full ['SibSp'] + full['Parch'] + 1
full['CabinGroup'] = full['Cabin_Level'].replace(['A','B'], 'AB')
 full['CabinGroup'] = full['CabinGroup'].replace(['C','F'], 'CF')
 full['CabinGroup'] = full['CabinGroup'].replace(['D','E'], 'DE')
 full.CabinGroup.value_counts()
full.loc[(full['FamilySize']==1)&(full['ppTicket']==1)&(full['nFamily']==1), 'GroupCat'] = 'single'
full.loc[(full['FamilySize']==2)|(full['ppTicket']==2)|(full['nFamily']==2), 'GroupCat'] = 'couple'
full.loc[(full['FamilySize'].isin([3,4])) |(full['ppTicket'].isin([3,4])) |(full['nFamily'].isin([3,4])) , 'GroupCat'] = 'small'
full.loc[(full['FamilySize']>4)|(full['ppTicket']>4)|(full['nFamily']>4), 'GroupCat'] = 'large'
full['Pclass_cat'] = full['Pclass'].astype('category' )<prepare_x_and_y>
def clean_age(df):
df["age"] = pd.to_numeric(df["age"], errors="coerce")
 max_age = 80
df["age"] = df['age'].apply(lambda x: min(x ,max_age))
df["age"] = df['age'].apply(lambda x: round(x/max_age, 6))
def clean_renta(df):
max_renta = 1.0e6
df["renta"] = df['renta'].apply(lambda x: min(x ,max_renta))
df["renta"] = df['renta'].apply(lambda x: round(x/max_renta, 6))
def clean_antigue(df):
df["antiguedad"] = pd.to_numeric(df["antiguedad"], errors="coerce")
 df["antiguedad"] = df["antiguedad"].replace(-999999, df['antiguedad'].median())
 max_antigue = 256
df["antiguedad"] = df['antiguedad'].apply(lambda x: min(x ,max_antigue))
df["antiguedad"] = df['antiguedad'].apply(lambda x: round(x/max_antigue, 6))
full2 = full[['PassengerId','Survived', 'Age','Pclass_cat', 'Embarked', 'Sex', 'Title', 'Cabin_Level', 'ppTicket', 'ppFare', 'Survival_Rate', 'isMarried', 'hasSurvivalInfo', 'FamilySize', 'Parch', 'SibSp', 'nFamily', 'GroupCat']]
dummyVars = full2.select_dtypes(include=['object','bool','category'] ).columns
full3 = pd.get_dummies(full2, columns = dummyVars, drop_first=True)
 train = full3[~full3['Survived'].isna() ]
test = full3[full3['Survived'].isna() ]
X_train = train.drop(["Survived","PassengerId"],axis=1)
 Y_train = train['Survived'].astype(int)
 X_test = test.drop(["Survived","PassengerId"],axis=1 )<train_on_grid>
clean_age(df_train)
 clean_age(df_test)
 clean_renta(df_train)
 clean_renta(df_test)
 clean_antigue(df_train)
 clean_antigue(df_test)
clf_ET = ExtraTreesClassifier(random_state=0, bootstrap=True, oob_score=True)
 sss = model_selection.StratifiedShuffleSplit(n_splits=10, test_size=0.33, random_state= 0)
 sss.get_n_splits(X_train, Y_train)
 parameters = {'n_estimators' : np.r_[10:210:10],
'max_depth': np.r_[1:6]
}
grid = model_selection.GridSearchCV(clf_ET, param_grid=parameters, scoring = 'accuracy', cv = sss, return_train_score=True, n_jobs=4, verbose=2)
 grid.fit(X_train,Y_train )<save_to_csv>
product_col_5 = [col for col in df_train.columns if '_ult1_5' in col]
product_col_4 = [col for col in df_train.columns if '_ult1_4' in col]
product_col_3 = [col for col in df_train.columns if '_ult1_3' in col]
product_col_2 = [col for col in df_train.columns if '_ult1_2' in col]
product_col_1 = [col for col in df_train.columns if '_ult1_1' in col]
df_train['tot5'] = df_train[product_col_5].sum(axis=1)
 df_test['tot5'] = df_test[product_col_5].sum(axis=1)
 df_train['tot4'] = df_train[product_col_4].sum(axis=1)
 df_test['tot4'] = df_test[product_col_4].sum(axis=1)
 df_train['tot3'] = df_train[product_col_3].sum(axis=1)
 df_test['tot3'] = df_test[product_col_3].sum(axis=1)
 df_train['tot2'] = df_train[product_col_2].sum(axis=1)
 df_test['tot2'] = df_test[product_col_2].sum(axis=1)
 df_train['tot1'] = df_train[product_col_1].sum(axis=1)
 df_test['tot1'] = df_test[product_col_1].sum(axis=1 )
p = a[a.param_max_depth==4].sort_values(by='mean_test_score', ascending=False ).iloc[0]
print("Best mean test score: %f using %s" %(p.mean_test_score, p.params))
bst_ET = grid.best_estimator_
bst_ET.set_params(**p.params)
 bst_ET.fit(X_train,Y_train)
 pred_ET = bst_ET.predict(X_test)
 test['Survived'] = pred_ET.astype(int)
 submission = test[['PassengerId', 'Survived']]
submission.PassengerId = submission.PassengerId.astype(int)
 submission.sort_values(by=['PassengerId'], inplace=True)
 submission.to_csv('pred_final.csv', index=False)
 submission.head(10 )<set_options>
for col in product_col[2:]:
df_train[col+'_past'] =(df_train[col+'_5']+df_train[col+'_4']+df_train[col+'_3']+df_train[col+'_2']+df_train[col+'_1'])/5
df_test[col+'_past'] =(df_test[col+'_5']+df_test[col+'_4']+df_test[col+'_3']+df_test[col+'_2']+df_test[col+'_1'])/5
warnings.filterwarnings('ignore' )<set_options>
for pro in product_col[2:]:
df_train[pro+'_past'] = df_train[pro+'_past']*(1-df_train[pro+'_5'])
 df_test[pro+'_past'] = df_test[pro+'_past']*(1-df_test[pro+'_5'] )
sns.set_style('whitegrid' )<load_from_csv>
for col in product_col[2:]:
for month in range(2,6):
df_train[col+'_'+str(month)+'_diff'] = df_train[col+'_'+str(month)] - df_train[col+'_'+str(month-1)]
df_test[col+'_'+str(month)+'_diff'] = df_test[col+'_'+str(month)] - df_test[col+'_'+str(month-1)]
df_train[col+'_'+str(month)+'_add'] = df_train[col+'_'+str(month)+'_diff'].apply(lambda x: max(x,0))
df_test[col+'_'+str(month)+'_add'] = df_test[col+'_'+str(month)+'_diff'].apply(lambda x: max(x,0))
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )<data_type_conversions>
product_col_5_diff = [col for col in df_train.columns if '5_diff' in col]
product_col_4_diff = [col for col in df_train.columns if '4_diff' in col]
product_col_3_diff = [col for col in df_train.columns if '3_diff' in col]
product_col_2_diff = [col for col in df_train.columns if '2_diff' in col]
product_col_5_add = [col for col in df_train.columns if '5_add' in col]
product_col_4_add = [col for col in df_train.columns if '4_add' in col]
product_col_3_add = [col for col in df_train.columns if '3_add' in col]
product_col_2_add = [col for col in df_train.columns if '2_add' in col]
product_col_all_diff = [col for col in df_train.columns if '_diff' in col]
product_col_all_add = [col for col in df_train.columns if '_add' in col]
train_data['Embarked'] = train_data['Embarked'].fillna('S')
 test_data['Embarked'] = test_data['Embarked'].fillna('S' )<categorify>
df_train['tot5_add'] = df_train[product_col_5_add].sum(axis=1)
 df_test['tot5_add'] = df_test[product_col_5_add].sum(axis=1)
 df_train['tot4_add'] = df_train[product_col_4_add].sum(axis=1)
 df_test['tot4_add'] = df_test[product_col_4_add].sum(axis=1)
 df_train['tot3_add'] = df_train[product_col_3_add].sum(axis=1)
 df_test['tot3_add'] = df_test[product_col_3_add].sum(axis=1)
 df_train['tot2_add'] = df_train[product_col_2_add].sum(axis=1)
 df_test['tot2_add'] = df_test[product_col_2_add].sum(axis=1 )
def fix_age(blob):
Age = blob[0]
Pclass = blob[1]
Sex = blob[2]
if pd.isnull(Age):
if Sex == 'male':
if Pclass == 1:
return 40
elif Pclass == 2:
return 30
else:
return 25
else:
if Pclass == 1:
return 35
elif Pclass == 2:
return 27
else:
return 22
else:
return Age<feature_engineering>
cols = list(df_train.drop(['target','ncodpers']+product_col_all_diff+product_col_all_add, 1 ).columns.values)
 id_preds = defaultdict(list)
 ids = df_test['ncodpers'].values
y_train = pd.get_dummies(df_train['target'].astype(int))
x_train = df_train[cols]
train_data['Age'] = train_data[['Age','Pclass','Sex']].apply(fix_age,axis=1)
 test_data['Age'] = test_data[['Age','Pclass','Sex']].apply(fix_age,axis=1 )<data_type_conversions>
model = Sequential()
model.add(Dense(150, input_dim=len(cols), activation='relu'))
model.add(Dense(22, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['categorical_accuracy'])
 model.fit(x_train.as_matrix() , y_train.as_matrix() , validation_split=0.2, nb_epoch=150, batch_size=10)
 x_test = df_test[cols]
x_test = x_test.fillna(0)
 p_test = model.predict(x_test.as_matrix())
 for id, p in zip(ids, p_test):
id_preds[id] = [0,0] + list(p )
train_data["Fare"] = train_data["Fare"].fillna(train_data["Fare"].median())
 test_data["Fare"] = test_data["Fare"].fillna(test_data["Fare"].median() )<data_type_conversions>
model.inputs
train_data['Sex'] = train_data['Sex'].fillna('male')
 test_data['Sex'] = test_data['Sex'].fillna('male' )<feature_engineering>
model.outputs
def add_family(blob):
temp = blob.split(' ')[0]
temp = temp[:len(temp)-1]
return temp<feature_engineering>
fraction = 1
id_preds_combined = {}
for uid, p in id_preds.items() :
id_preds_combined[uid] = fraction*np.asarray(id_preds[uid])
 id_preds = id_preds_combined
train_data['FamilyName'] = train_data['Name'].apply(add_family )<feature_engineering>
usecols = ['ncodpers', 'ind_ahor_fin_ult1', 'ind_aval_fin_ult1', 'ind_cco_fin_ult1',
'ind_cder_fin_ult1', 'ind_cno_fin_ult1', 'ind_ctju_fin_ult1',
'ind_ctma_fin_ult1', 'ind_ctop_fin_ult1', 'ind_ctpp_fin_ult1',
'ind_deco_fin_ult1', 'ind_deme_fin_ult1', 'ind_dela_fin_ult1',
'ind_ecue_fin_ult1', 'ind_fond_fin_ult1', 'ind_hip_fin_ult1',
'ind_plan_fin_ult1', 'ind_pres_fin_ult1', 'ind_reca_fin_ult1',
'ind_tjcr_fin_ult1', 'ind_valo_fin_ult1', 'ind_viv_fin_ult1',
'ind_nomina_ult1', 'ind_nom_pens_ult1', 'ind_recibo_ult1']
test_data['FamilyName'] = test_data['Name'].apply(add_family )<count_values>
df_recent = pd.read_csv('.. /input/train-ver2/train_ver2.csv',usecols=usecols)
 df_recent=df_recent[df_recent['ncodpers'].isin(customer)]
df_recent.fillna(0, inplace=True )
train_data['FamilyName'].value_counts()<define_variables>
sample = pd.read_csv('.. /input/sample-submission/sample_submission.csv' )
def apply_name_suffix(blob):
temp = blob.split(' ')[1]
return temp<feature_engineering>
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p))
train_data['NameSuffix'] = train_data['Name'].apply(apply_name_suffix )<feature_engineering>
model.save('kerass.model' )
test_data['NameSuffix'] = test_data['Name'].apply(apply_name_suffix )<define_variables>
sample['added_products'] = test_preds
sample.to_csv('Keras1.csv', index=False)
 print(datetime.datetime.now() -start_time )