Kaggle competition: Santander Product Recommendation
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=4,
learning_rate=0.05, n_estimators=1024,
max_bin = 40, bagging_fraction = 0.9,
bagging_freq = 20, feature_fraction = 0.1,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf = 2, min_sum_hessian_in_leaf = 1, random_state=5)
<choose_model_class>
start_time = datetime.datetime.now()
demographic_cols = ['fecha_dato',
'ncodpers','ind_empleado','pais_residencia','sexo','age','fecha_alta','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov"]
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
train_cols = demographic_cols + product_col
def baseline_model() :
model = Sequential()
model.add(Dense(64, input_dim=len(X), kernel_initializer='normal', activation='relu'))
model.add(Dense(8, input_dim=64, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal'))
opt = optimizers.Adam(learning_rate = 0.005)
 model.compile(loss='mean_squared_error', optimizer=opt)
 return model
nn = KerasRegressor(build_fn=baseline_model, epochs=3000, batch_size=len(train), verbose=0)
<set_options>
df_train = pd.read_csv('.. /input/juneextra/juneExtra.csv', usecols=train_cols )
rf = RandomForestRegressor(n_estimators=128, min_samples_split=4, min_samples_leaf = 2,
random_state=42)
 rmse_cv(rf )<train_model>
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )
class Meta_Regressor(BaseEstimator):
def __init__(self, base_models, meta_models):
self.base_models = base_models
self.meta_models = meta_models
def fit(self, X, y):
self.base_models_ = [[] for _ in self.base_models]
self.meta_models_ = clone(self.meta_models)
 Kf = KFold(n_splits=5, shuffle=True, random_state=5)
 out_fold_pred = np.zeros(( len(X), len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_idx, val_idx in Kf.split(X):
model = clone(self.base_models[i])
 model.fit(X[train_idx], y[train_idx])
 pred = model.predict(X[val_idx])
 out_fold_pred[val_idx, i] = pred
self.base_models_[i].append(model)
 self.meta_models_.fit(X=out_fold_pred, y=y)
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in sub_models] ).mean(axis=1)
 for sub_models in self.base_models_])
 scores = self.meta_models_.predict(meta_features)
 return scores<train_model>
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']
class Weighted_Ensemble(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(model)for model in self.models]
for model in self.models_:
model.fit(X, y)
 def predict(self, x):
results = np.zeros(len(x))
scores = [model.predict(x)for model in self.models_]
return scores<split>
df_train.isnull().sum()
meta_regressor = Meta_Regressor(base_models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf], meta_models = EN)
 weighted_ensemble = Weighted_Ensemble(models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf, EN, meta_regressor])
 train_x, test_x, train_y, test_y = train_test_split(train[X].values, train[y].values, test_size=0.3, random_state=4)
 weighted_ensemble.fit(train_x, train_y)
 scores = weighted_ensemble.predict(test_x )<compute_test_metric>
df_train.tiprel_1mes.value_counts()
weights = [0.1, 0.2, 0.2, 0.06, 0.09, 0.01, 0.04, 0.2, 0.1]
score = np.zeros(len(scores[0]))
for i, model in enumerate(scores):
score += scores[i] * weights[i]
rmsle(score, test_y )<choose_model_class>
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )
meta_regressor = Meta_Regressor(base_models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf], meta_models = EN)
 weighted_ensemble = Weighted_Ensemble(models= [KRR, GBoost, model_xgb, EN])
 weighted_ensemble.fit(train[X].values, train[y].values)
 weights = [0.2, 0.2, 0.05,0.55]
scores = weighted_ensemble.predict(test[X].values)
 score = np.zeros(len(scores[0]))
for i, model in enumerate(scores):
score += scores[i] * weights[i]
pred = np.expm1(score )<save_to_csv>
df_train["renta"] = pd.to_numeric(df_train["renta"], errors="coerce")
 df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce" )
submission = pd.DataFrame({
'Id': range(1461, 2920),
'SalePrice': pred
})
 submission.to_csv("submission.csv", index=False )<define_variables>
for cod in unique_prov:
df_test.loc[df_test['cod_prov']==cod,['renta']] = df_test.loc[df_test['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df_train.loc[df_train['cod_prov']==cod,['renta']] = df_train.loc[df_train['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
random_state_split = 42
Dropout_num = 0
learning_rate = 6e-6
valid = 0.15
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False<set_options>
df_test.renta.fillna(df_test["renta"].median() , inplace=True )
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
warnings.simplefilter('ignore' )<load_from_csv>
df_train.isnull().sum()
tweet= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
df_test.isnull().sum()
<define_variables>
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )
Real_len = tweet[tweet['target'] == 1].shape[0]
Not_len = tweet[tweet['target'] == 0].shape[0]<feature_engineering>
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']
tweet['length'] = tweet['text'].apply(length )<string_transform>
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 print(use_col)
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
print(row_index)
 df_train.loc[row_index,col] = np.nan
print(df_train.loc[row_index,col])
 for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan
def create_corpus(target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus<string_transform>
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )
def create_corpus_df(tweet, target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus<sort_values>
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )
corpus=create_corpus(0)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:10]<prepare_x_and_y>
models = {}
model_preds = {}
id_preds1 = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = LogisticRegression()
clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds1[id].append(p)
 print(roc_auc_score(y_train, p_train2))
plt.figure(figsize=(16,5))
counter=Counter(corpus)
 most=counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
if(word not in stop):
x.append(word)
 y.append(count )<feature_engineering>
models = {}
model_preds = {}
id_preds3 = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = ensemble.RandomForestClassifier(n_estimators=60, n_jobs=-1,max_depth=10, min_samples_split=10, verbose=1)
 clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds3[id].append(p)
 print(roc_auc_score(y_train, p_train2))
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]<concatenate>
fraction1 = 0.4
fraction3 = 0.6
id_preds_combined = {}
for uid, p in id_preds1.items() :
id_preds_combined[uid] = fraction1*np.asarray(id_preds1[uid])+ fraction3*np.asarray(id_preds3[uid])
 id_preds = id_preds_combined
df=pd.concat([tweet,test])
 df.shape<train_model>
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
example="New competition launched :https://www.kaggle.com/c/nlp-getting-started"<drop_column>
sample = pd.read_csv('.. /input/santander-product-recommendation/sample_submission.csv' )
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 remove_URL(example )<feature_engineering>
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p))
df['text']=df['text'].apply(lambda x : remove_URL(x))<drop_column>
sample['added_products'] = test_preds
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 print(remove_html(example))<feature_engineering>
sample.to_csv('LR-RF_sub.csv', index=False)
 print(datetime.datetime.now() -start_time)