Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<feature_engineering>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import chi2, mutual_info_classif
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostClassifier
submission1['toxic'] = submission1['toxic']*0.1 + submission2['toxic']*0.9<save_to_csv>
sc = StandardScaler()
le = LabelEncoder()
onehot = OneHotEncoder(sparse=False )
submission1.to_csv('submission.csv', index=False )<define_variables>
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()
sub_path = ".. /input/blending"
all_files = os.listdir(sub_path)
 all_files<load_from_csv>
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "jigsaw" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
all_df = train_df.append(test_df, sort=False)
 all_df.shape
concat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
all_df.drop(['rez_esc', 'v18q1', 'v2a1'], axis=1, inplace=True)
 all_df.fillna({'SQBmeaned': all_df['SQBmeaned'].median() , 'meaneduc': all_df['meaneduc'].median() }, inplace=True )
cutoff_lo = 0.7
cutoff_hi = 0.3<save_to_csv>
float_cols = [col for col in all_df.columns if all_df[col].dtype=='float64']
int_cols = [col for col in all_df.columns if all_df[col].dtype=='int64']
object_cols = [col for col in all_df.columns if all_df[col].dtype=='object']
concat_sub['toxic'] = concat_sub['jigsaw_mean']
concat_sub[['toxic']].to_csv('submission2.csv',
index=False, float_format='%.6f' )<save_to_csv>
log_meaneduc = np.log1p(all_df['meaneduc'])
 log_overcrowding = np.log1p(all_df['overcrowding'])
 log_SQBovercrowding = np.log1p(all_df['SQBovercrowding'])
 log_SQBdependency = np.log1p(all_df['SQBdependency'])
 log_SQBmeaned = np.log1p(all_df['SQBmeaned'])
 temp_df = pd.DataFrame({'log_meaneduc': log_meaneduc, 'log_overcrowding': log_overcrowding, 'log_SQBovercrowding': log_SQBovercrowding, 'log_SQBdependency': log_SQBdependency, 'log_SQBmeaned': log_SQBmeaned})
 temp_df.head()
concat_sub['toxic'] = concat_sub['jigsaw_median']
concat_sub[['toxic']].to_csv('submission1.csv',
index=False, float_format='%.6f' )<save_to_csv>
all_df[['meaneduc', 'overcrowding', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned']] = temp_df[['log_meaneduc', 'log_overcrowding', 'log_SQBovercrowding', 'log_SQBdependency', 'log_SQBmeaned']]
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission3.csv',
index=False, float_format='%.6f' )<feature_engineering>
le = LabelEncoder()
for col in object_cols:
all_df[col] = le.fit_transform(all_df[col].values )
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_mean']))
concat_sub[['toxic']].to_csv('submission4.csv',
index=False, float_format='%.6f' )<feature_engineering>
dup_cols = [col for col in all_df.columns if col[:3] == 'SQB']
all_df.drop(dup_cols, axis=1, inplace=True)
 all_df.shape
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission5.csv',
index=False, float_format='%.6f' )<load_from_csv>
all_df.drop('agesq', axis=1, inplace=True)
 all_df['edjefe'] =(all_df['edjefe'] == all_df['edjefe'].max())* 1
all_df['edjefa'] =(all_df['edjefa'] == all_df['edjefa'].max())* 1
all_df.shape
mnist_test = pd.read_csv(".. /input/mnist-digit-recognizer/mnist_test.csv")
 mnist_train = pd.read_csv(".. /input/mnist-digit-recognizer/mnist_train.csv")
 sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<feature_engineering>
all_df['age'] = sc.fit_transform(all_df['age'].values.reshape(( -1, 1)))
 all_df['idhogar'] = sc.fit_transform(all_df['idhogar'].values.reshape(( -1, 1)) )
test['dataset'] = 'test'<feature_engineering>
cat_cols = [col for col in int_cols if col not in [col for col in int_cols if col[:3]=='SQB']]
cat_cols.remove('agesq')
 cat_cols.remove('age' )
train['dataset'] = 'train'<concatenate>
onehot_cols = [col for col in cat_cols if len(all_df[col].unique())> 2]
onehot_arr = onehot.fit_transform(all_df[onehot_cols].values)
 onehot_arr.shape
dataset = pd.concat([train.drop('label', axis=1), test] ).reset_index()<concatenate>
all_df.drop(onehot_cols, axis=1, inplace=True)
 all_df.shape
mnist = pd.concat([mnist_train, mnist_test] ).reset_index(drop=True)
 labels = mnist['label'].values
mnist.drop('label', axis=1, inplace=True)
 mnist.columns = cols<sort_values>
train_df['Target'] = train_df['Target'].apply(lambda x: x-1 )
idx_mnist = mnist.sort_values(by=list(mnist.columns)).index
dataset_from = dataset.sort_values(by=list(mnist.columns)) ['dataset'].values
original_idx = dataset.sort_values(by=list(mnist.columns)) ['index'].values<feature_engineering>
tr_cols = [col for col in train_df.columns if col not in ['Id', 'Target']]
X_train = train_df[tr_cols].values
X_train = np.concatenate(( X_train, onehot_arr[:9557, :]), axis=1)
 y_train = train_df['Target'].values
skf = StratifiedKFold(n_splits=5, shuffle=True )
for i in range(len(idx_mnist)) :
if dataset_from[i] == 'test':
sample_submission.loc[original_idx[i], 'Label'] = labels[idx_mnist[i]]<save_to_csv>
class_weights = compute_class_weight('balanced', np.sort(train_df['Target'].unique()), train_df['Target'].values)
 n_rounds = 1000
learning_rate = 0.2
max_depth = 5
l2_reg = 2
n_classes = 4
early_stopping_rounds = 20
sample_submission.to_csv('submission.csv', index=False )<load_from_csv>
test_arr = test_df[[col for col in test_df.columns if col not in ['Id', 'Target']]].values
test_arr = np.concatenate(( test_arr, onehot_arr[9557:, :]), axis=1 )
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 print(train.shape)
 train.head()<load_from_csv>
xgb_params = [('eta', learning_rate),('max_depth', max_depth),('colsample_bytree', 0.8),('lambda', l2_reg),('objective', 'multi:softprob'),('num_class', n_classes),('eval_metric', 'mlogloss'),('silent', 1)]
for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
dtrain = xgb.DMatrix(X_tr, y_tr, weight=[class_weights[int(y_tr[i])] for i in range(y_tr.shape[0])])
 dval = xgb.DMatrix(X_val, y_val, weight=[class_weights[int(y_val[i])] for i in range(y_val.shape[0])])
 eval_list = [(dtrain, 'train'),(dval, 'validation')]
bst = xgb.train(xgb_params, dtrain, n_rounds, eval_list, early_stopping_rounds=early_stopping_rounds, verbose_eval=True )
test= pd.read_csv(".. /input/digit-recognizer/test.csv")
 print(test.shape)
 test.head()<prepare_x_and_y>
dtest = xgb.DMatrix(test_arr, weight=class_weights)
 xgb_preds = bst.predict(dtest)
 xgb_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 xgb_sample_subm['Target'] = np.argmax(xgb_preds, axis=1 ).astype(int)
 xgb_sample_subm['Target'] = xgb_sample_subm['Target'].apply(lambda x: x+1)
 xgb_sample_subm.to_csv('xgb_preds.csv', index=False)
 sns.countplot(xgb_sample_subm['Target'])
 plt.show()
Y_train = train["label"]
X_train = train.drop(columns = ["label"],axis = 1 )<train_model>
lgb_params = {'objective': 'multiclass', 'num_class': n_classes, 'learning_rate': learning_rate, 'num_leaves': 31, 'num_thread': 4, 'max_depth': max_depth, 'feature_fraction': 0.8, 'lambda_l2': l2_reg}
for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
dtrain = lgb.Dataset(X_tr, y_tr, weight=[class_weights[int(y_tr[i])] for i in range(y_tr.shape[0])])
 dval = lgb.Dataset(X_val, y_val, weight=[class_weights[int(y_val[i])] for i in range(y_val.shape[0])])
 bst = lgb.train(lgb_params, dtrain, n_rounds, valid_sets = [dval, dtrain], valid_names = ['validation', 'train'], early_stopping_rounds = early_stopping_rounds, verbose_eval=False )
X_train = X_train / 255.0
test = test / 255.0
print("x_train shape: ",X_train.shape)
 print("test shape: ",test.shape )<categorify>
lgb_preds = bst.predict(test_arr)
 lgb_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 lgb_sample_subm['Target'] = np.argmax(lgb_preds, axis=1 ).astype(int)
 lgb_sample_subm['Target'] = lgb_sample_subm['Target'].apply(lambda x: x+1)
 lgb_sample_subm.to_csv('lgb_preds.csv', index=False)
 sns.countplot(lgb_sample_subm['Target'])
 plt.show()
Y_train = to_categorical(Y_train, num_classes = 10 )<split>
cat_model = CatBoostClassifier(iterations=n_rounds, learning_rate=learning_rate, depth=max_depth, loss_function='MultiClass', classes_count=n_classes, logging_level='Silent', l2_leaf_reg=l2_reg, thread_count=4, class_weights=class_weights)
 for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
cat_model.fit(X_tr, y_tr, use_best_model=True, eval_set=(X_val, y_val), early_stopping_rounds=early_stopping_rounds )
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1,random_state = 2)
 print("x_train shape",X_train.shape)
 print("x_test shape",X_val.shape)
 print("y_train shape",Y_train.shape)
 print("y_test shape",Y_val.shape )<choose_model_class>
cat_preds = cat_model.predict_proba(test_arr)
 cat_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 cat_sample_subm['Target'] = np.argmax(cat_preds, axis=1 ).astype(int)
 cat_sample_subm['Target'] = cat_sample_subm['Target'].apply(lambda x: x+1)
 cat_sample_subm.to_csv('cat_preds.csv', index=False)
 sns.countplot(cat_sample_subm['Target'])
 plt.show()
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3,3),activation = 'relu',
kernel_initializer='he_normal', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3,3),
activation = 'relu', kernel_initializer='he_normal'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size =(2,2)))
 model.add(Dropout(0.4))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same',
activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same',
activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size =(2,2), strides =(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation = 'softmax'))<choose_model_class>
avg_preds =(xgb_preds + cat_preds + lgb_preds)/ 3.0
avg_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 avg_sample_subm['Target'] = np.argmax(avg_preds, axis=1 ).astype(int)
 avg_sample_subm['Target'] = avg_sample_subm['Target'].apply(lambda x: x+1)
 avg_sample_subm.to_csv('avg_preds.csv', index=False)
 sns.countplot(avg_sample_subm['Target'])
 plt.show()
<choose_model_class><EOS>