Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<compute_train_metric>
%matplotlib inline
def lb_dist(model):
model.fit(x_train, y_train)
 print("Train Acc :: ", accuracy_score(y_train, model.predict(x_train)))
 print("Valid Acc :: ", accuracy_score(y_val, model.predict(x_val)))
 print("CV Accuracy :: ", cross_val_score(rand, train, label['surface'], cv=5 ).mean())
 return model<choose_model_class>
pd.set_option("display.max_rows", None)
 testDf = pd.read_csv('.. /input/test.csv')
 trainDf = pd.read_csv('.. /input/train.csv')
 print("Training dataset basic information")
 print("Rows: {}".format(len(trainDf)))
 print("Columns: {}".format(len(trainDf.columns)))
 trainDf.head()
rand = RandomForestClassifier(n_estimators=500, random_state=13)
 rand = lb_dist(rand )<compute_train_metric>
testDf['Target'] = np.nan
data = trainDf.append(testDf, ignore_index = True )
print("Accuracy Score :: ", accuracy_score(label['surface'], rand.predict(train)))
 plot_confusion_matrix(label['surface'], rand.predict(train), classes=le.classes_ )<define_variables>
def findColumnsWithNan(df):
cols = df.columns[df.isna().any() ]
print("Number of columns with Nan: {}".format(len(cols)))
 print("Column names: {}".format(cols))
print("-" * 80)
 for col in cols:
print("Column: [{}] missing {} values.".format(col, len(df[df[col].isna() == True])))
 print("Analysis of training dataset...")
 findColumnsWithNan(trainDf )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=20)
 predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( train.shape[0]))
score = 0<predict_on_test>
data.loc[(data['tipovivi1'] == 1)&(data['v2a1'].isna()), 'v2a1'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['v2a1'].isna() ])) )
for times,(trn_idx, val_idx)in enumerate(folds.split(train.values, label['surface'].values)) :
model = RandomForestClassifier(n_estimators=500, random_state=13)
 model.fit(train.iloc[trn_idx], label['surface'][trn_idx])
 measured[val_idx] = model.predict(train.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(train.iloc[val_idx], label['surface'][val_idx])
 print("Fold: {} score: {}".format(times, model.score(train.iloc[val_idx], label['surface'][val_idx])))
 gc.collect() 
<save_to_csv>
data.loc[data['v18q1'].isna() , 'v18q1'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['v18q1'].isna() ])) )
sub['surface'] = le.inverse_transform(predicted.argmax(axis=1))
sub.to_csv('rand_sub_10.csv', index=False)
 sub.head()<count_values>
data.loc[(data['age'] < 7)&(data['rez_esc'].isna()), 'rez_esc'] = 0
data.loc[(data['age'] > 19)&(data['rez_esc'].isna()), 'rez_esc'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['rez_esc'].isna() ])) )
sub.surface.value_counts()<import_modules>
data.loc[data['age'] < 19 & data['meaneduc'].isna() , 'meaneduc'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['meaneduc'].isna() ])) )
import tensorflow as tf
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense, LSTM, Flatten, Dropout, MaxPooling2D, CuDNNLSTM
from tensorflow.keras.utils import to_categorical
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif
from tsfresh import extract_features
from tsfresh.utilities.dataframe_functions import impute
import pickle
from functools import reduce<save_to_csv>
data.drop('SQBmeaned', inplace=True, axis=1)
 print("Total number of columns left: {}".format(len(data.columns)) )
if __name__ == '__main__':
df_train=pd.read_csv('.. /input/X_train.csv')
 df_train.drop(['row_id'],axis=1,inplace=True)
 df_train=extract_features(df_train,column_id = "series_id", chunksize=7, default_fc_parameters=None, column_sort = "measurement_number", impute_function=impute, n_jobs=7)
 df_train.to_csv('train_feature.csv',index=False)
 del df_train
df_test=pd.read_csv('.. /input/X_test.csv')
 df_test.drop(['row_id'],axis=1,inplace=True)
 df_test=extract_features(df_test,column_id = "series_id", chunksize=7, default_fc_parameters=None, column_sort = "measurement_number", impute_function=impute, n_jobs=7)
 df_test.to_csv('test_feature.csv',index=False)
 del df_test<load_from_csv>
for cols in data.columns[1:]:
if cols in ['idhogar', 'dependency', 'edjefe', 'edjefa']:
continue
percentile75 = np.percentile(data[cols].fillna(0), 75)
 percentile25 = np.percentile(data[cols].fillna(0), 25)
 threshold =(percentile75 - percentile25)* 1.5
lower, upper =(percentile25 - threshold),(percentile75 + threshold)
 outliers = data.loc[(data[cols] < lower)&(data[cols] > upper)]
if len(outliers)> 0:
print('Feature: {}.Identified outliers: {}'.format(cols, len(outliers)) )
<set_options>
for col in ['dependency', 'edjefe', 'edjefa']:
data.loc[data[col] == 'yes', col] = 1.0
data.loc[data[col] == 'no', col] = 0.0
data[col] = pd.to_numeric(data[col] )
warnings.filterwarnings('ignore' )<load_from_csv>
def featuresToDrop(corrMatrix):

 upper = corrMatrix.where(np.triu(np.ones(corrMatrix.shape), k=1 ).astype(np.bool))
return [column for column in upper.columns if any(upper[column] > 0.95)]
toDrop = featuresToDrop(corrMat)
 data.drop(toDrop, inplace=True, axis=1)
 print("Correlated features which are dropped: {}".format(toDrop))
%%time
X_train = pd.read_csv(os.path.join(PATH, 'X_train.csv'))
X_test = pd.read_csv(os.path.join(PATH, 'X_test.csv'))
y_train = pd.read_csv(os.path.join(PATH, 'y_train.csv'))<train_model>
features = list(data.drop(columns = ['Id', 'idhogar', 'Target'] ).columns)
 aggDf = data.drop(columns='Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std'])
 new_col = []
for c in aggDf.columns.levels[0]:
for stat in aggDf.columns.levels[1]:
new_col.append('{}-{}'.format(c, stat))
aggDf.columns = new_col
toDrop = featuresToDrop(aggDf.corr())
 aggDf.drop(toDrop, inplace=True, axis=1)
 data = data.merge(aggDf, on='idhogar', how ='left')
 print('Training feature shape: ', data.shape )
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))<count_missing_values>
data['phones-per-capita'] = data['qmobilephone'] / data['tamviv']
data['tablets-per-capita'] = data['v18q1'] / data['tamviv']
data['rooms-per-capita'] = data['rooms'] / data['tamviv']
data['rent-per-capita'] = data['v2a1'] / data['tamviv']
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
trainTarget = np.array(list(data[data['Target'].notnull() ]['Target'].astype(np.uint8)))
 submission = data.loc[data['Target'].isnull() , 'Id'].to_frame()
trainData = data[data['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 testData = data[data['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 imputer = SimpleImputer(missing_values=np.nan, strategy='median')
 trainData = imputer.fit_transform(trainData)
 testData = imputer.transform(testData)
 scaler = MinMaxScaler()
trainData = scaler.fit_transform(trainData)
 testData = scaler.transform(testData )
<count_missing_values><EOS> 
model = lightgbm.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 predicts_result = []
for idx,(train_index, test_index)in enumerate(kf.split(trainData, trainTarget)) :
print("Fold: {}".format(idx))
X_train, X_val = trainData[train_index], trainData[test_index]
y_train, y_val = trainTarget[train_index], trainTarget[test_index]
model.fit(X_train, y_train, verbose=100)
 predicts_result.append(model.predict(testData))
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission.csv', index=False)
 print("Completed!" )