Kaggle competition: Costa Rican Household Poverty Level Prediction
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
train_df["activation_month"] = train_df["activation_date"].dt.month
test_df["activation_month"] = test_df["activation_date"].dt.month
train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))
train_df["description"].fillna("NA", inplace=True)
 test_df["description"].fillna("NA", inplace=True)
 train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))
train_df['param123'] = train_df['param_1'].fillna('')+ " " + train_df['param_2'].fillna('')+ " " + train_df['param_3'].fillna('')
 test_df['param123'] = test_df['param_1'].fillna('')+ " " + test_df['param_2'].fillna('')+ " " + test_df['param_3'].fillna('')
 enc = train_df.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0] ).astype(np.float32 ).reset_index()
enc.columns = ['category_name' ,'image_top_1_impute']
train_df = pd.merge(train_df, enc, how='left', on='category_name')
 test_df = pd.merge(test_df, enc, how='left', on='category_name')
 del enc
gc.collect()
train_df['image_top_1'].fillna(train_df['image_top_1_impute'], inplace=True)
 test_df['image_top_1'].fillna(test_df['image_top_1_impute'], inplace=True)
 test_df['image'] = test_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['image'] = train_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['city'] = train_df['city'] + "_" + train_df['region']
test_df['city'] = test_df['city'] + "_" + test_df['region']
train_df['price'].fillna(0, inplace=True)
 test_df['price'].fillna(0, inplace=True)
 train_df['price'] = np.log1p(train_df['price'])
 test_df['price'] = np.log1p(test_df['price'])
 price_mean = train_df['price'].mean()
price_std = train_df['price'].std()
train_df['price'] =(train_df['price'] - price_mean)/ price_std
test_df['price'] =(test_df['price'] - price_mean)/ price_std
cat_cols = ['category_name', 'image_top_1']
num_cols = ['price', 'deal_probability']
for c in cat_cols:
for c2 in num_cols:
enc = train_df.groupby(c)[c2].agg(['median'] ).astype(np.float32 ).reset_index()
enc.columns = ['_'.join([str(c), str(c2), str(c3)])if c3 != c else c for c3 in enc.columns]
train_df = pd.merge(train_df, enc, how='left', on=c)
 test_df = pd.merge(test_df, enc, how='left', on=c)
 del cat_cols, num_cols, c, c2, enc
gc.collect()<string_transform>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
def cleanName(text):
try:
textProc = text.lower()
textProc = re.sub('[!@
textProc = " ".join(textProc.split())
 return textProc
except:
return "name error"<data_type_conversions>
df_all = df.append(test)
 df_all.shape
textfeats = ["description", "title"]
train_df['desc_punc'] = train_df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df['desc_punc'] = test_df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df['title'] = train_df['title'].apply(lambda x: cleanName(x))
train_df["description"] = train_df["description"].apply(lambda x: cleanName(x))
test_df['title'] = test_df['title'].apply(lambda x: cleanName(x))
test_df["description"] = test_df["description"].apply(lambda x: cleanName(x))
for cols in textfeats:
train_df[cols] = train_df[cols].astype(str)
 train_df[cols] = train_df[cols].astype(str ).fillna('missing')
 train_df[cols] = train_df[cols].str.lower()
train_df[cols + '_num_words'] = train_df[cols].apply(lambda comment: len(comment.split()))
train_df[cols + '_num_unique_words'] = train_df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 train_df[cols + '_words_vs_unique'] = train_df[cols+'_num_unique_words'] / train_df[cols+'_num_words'] * 100
train_df[cols + '_num_letters'] = train_df[cols].apply(lambda comment: len(comment))
test_df[cols] = test_df[cols].astype(str)
 test_df[cols] = test_df[cols].astype(str ).fillna('missing')
 test_df[cols] = test_df[cols].str.lower()
test_df[cols + '_num_words'] = test_df[cols].apply(lambda comment: len(comment.split()))
test_df[cols + '_num_unique_words'] = test_df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 test_df[cols + '_words_vs_unique'] = test_df[cols+'_num_unique_words'] / test_df[cols+'_num_words'] * 100
test_df[cols + '_num_letters'] = test_df[cols].apply(lambda comment: len(comment))
train_df['title_desc_len_ratio'] = train_df['title_num_letters']/train_df['description_num_letters']
test_df['title_desc_len_ratio'] = test_df['title_num_letters']/test_df['description_num_letters']
print(train_df.head())
 print(test_df.head() )<categorify>
df_all['dependency'].value_counts()
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param123"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "activation_date", "image", "param_2", "param_3"
, "image_top_1_impute"]
train_X = train_df.drop(cols_to_drop + ["deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values
del train_df, test_df
gc.collect()<feature_engineering>
df_all[df_all['idhogar'] == '2b58d945f'][['hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total']]
russian_stop = set(stopwords.words('russian'))
tfidf_para = {
"stop_words": russian_stop,
"analyzer": 'word',
"token_pattern": r'\w{1,}',
"sublinear_tf": True,
"dtype": np.float32,
"norm": 'l2',
"smooth_idf":False
}
def get_col(col_name): return lambda x: x[col_name]
vectorizer = FeatureUnion([ 
('description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=17000,
**tfidf_para,
preprocessor=get_col('description'))), 
('title',CountVectorizer(
 ngram_range=(1, 2),
stop_words = russian_stop,
preprocessor=get_col('title')))
 ])
 df = pd.concat([train_X,test_X],axis=0)
 vectorizer.fit(df.to_dict('records'))
ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
df.drop(textfeats, axis=1,inplace=True )<concatenate>
df_all['edjefa'].value_counts()
sparse_m = csr_matrix(df.values)
 complete = hstack([sparse_m,ready_df])
 tfvocab = df.columns.tolist() + tfvocab
complete = complete.tocsr()
print(complete.shape)
 print(ready_df.shape)
 del sparse_m, ready_df, df
gc.collect()<split>
df_all['edjefe'].value_counts()
train_X = complete[0:traindex.shape[0],]
test_X = complete[traindex.shape[0]:,]
del complete
gc.collect()
train_X.shape<split>
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )
dev_X = train_X[:-200000,:]
val_X = train_X[-200000:,:]
dev_y = train_y[:-200000]
val_y = train_y[-200000:]
print(dev_X.shape, val_X.shape, test_X.shape)
 del train_X, train_y
gc.collect()<train_model>
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 300,
"learning_rate" : 0.01,
"bagging_fraction" : 0.75,
"feature_fraction" : 0.5,
"bagging_freq" : 2,
"bagging_seed" : 2018,
"verbosity" : -1,
"max_depth": 18,
"min_child_samples":100
}
lgtrain = lgb.Dataset(train_X, label=train_y, feature_name=tfvocab)
 lgval = lgb.Dataset(val_X, label=val_y, feature_name=tfvocab)
 del train_X, val_X; gc.collect()
evals_result = {}
model = lgb.train(params, lgtrain, 10000, valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 del lgtrain, lgval; gc.collect()
return pred_test_y, model, evals_result<train_model>
df_all.isnull().sum()
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X )<save_to_csv>
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()
pred_test[pred_test>1] = 1
pred_test[pred_test<0] = 0
sub_df = pd.DataFrame({"item_id":test_id})
 sub_df["deal_probability"] = pred_test
sub_df.to_csv("baseline_lgb.csv", index=False )<load_from_csv>
df_all['v18q'].value_counts()
print("
Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 testdex = testing.index
y = training.deal_probability.copy()
training.drop("deal_probability",axis=1, inplace=True)
 print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))<feature_engineering>
df_all.isnull().sum().sort_values()
df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
print("Feature Engineering")
 df["price"] = np.log(df["price"]+0.001)
 df["price"].fillna(-999,inplace=True)
 df["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 df["Weekday"] = df['activation_date'].dt.weekday
df["Week of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day
print('Create combined param len')
 df['param_combined'] = df.apply(lambda row:' '.join([str(row['param_1']),str(row['param_2']),str(row['param_3'])]),axis = 1)
 df['param_combined'].fillna(' ',inplace = True)
 df['param_combined_len'] = df['param_combined'].apply(lambda x:len(x.split()))
df.drop(["activation_date","image",'param_combined','Week of Year','Day of Month'],axis=1,inplace=True)
 print("
Encode Variables")
 categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type","image_top_1",'param_combined_len']
messy_categorical = ["param_1","param_2","param_3","title","description"]
print("Encoding :",categorical + messy_categorical)
<categorify>
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )
lbl = preprocessing.LabelEncoder()
for col in categorical + messy_categorical:
df[col] = lbl.fit_transform(df[col].astype(str))
print("
Catboost Modeling Stage")
 X = df.loc[traindex,:].copy()
print("Training Set shape",X.shape)
 test = df.loc[testdex,:].copy()
print("Submission Set Shape: {} Rows, {} Columns".format(*test.shape))
del df
gc.collect()<split>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
X_train, X_valid, y_train, y_valid = train_test_split(
 X, y, test_size=0.10, random_state=23)
 def column_index(df, query_cols):
cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols,query_cols,sorter=sidx)]
categorical_features_pos = column_index(X,categorical + messy_categorical)
<train_model>
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42 )
print("Train CatBoost Decision Tree")
 modelstart= time.time()
cb_model = CatBoostRegressor(iterations=700,
learning_rate=0.02,
depth=12,
eval_metric='RMSE',
random_seed = 23,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 75,
od_wait=100)
 cb_model.fit(X_train, y_train,
eval_set=(X_valid,y_valid),
cat_features=categorical_features_pos,
use_best_model=True,
verbose=True )<save_to_csv>
rf.fit(train[feats], train['Target'] )
print("Model Evaluation Stage")
 print(cb_model.get_params())
 print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, cb_model.predict(X_valid))))
catpred = cb_model.predict(test)
 catsub = pd.DataFrame(catpred,columns=["deal_probability"],index=testdex)
 catsub['deal_probability'].clip(0.0, 1.0, inplace=True)
 catsub.to_csv("submission.csv",index=True,header=True)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))<set_options>
test['Target'] = rf.predict(test[feats] ).astype(int )
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999<load_from_csv>
test['Target'].value_counts(normalize=True )
<filter><EOS> 
test[['Id', 'Target']].to_csv('submission.csv', index=False )