Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<count_missing_values>
sns.set()
missing_data(y_train )<normalization>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z<feature_engineering>
train_df.isnull().sum() [train_df.isnull().sum() > 0]
def perform_euler_factors_calculation(df):
df['total_angular_velocity'] = np.sqrt(np.square(df['angular_velocity_X'])+ np.square(df['angular_velocity_Y'])+ np.square(df['angular_velocity_Z']))
df['total_linear_acceleration'] = np.sqrt(np.square(df['linear_acceleration_X'])+ np.square(df['linear_acceleration_Y'])+ np.square(df['linear_acceleration_Z']))
df['total_xyz'] = np.sqrt(np.square(df['orientation_X'])+ np.square(df['orientation_Y'])+
np.square(df['orientation_Z']))
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
x, y, z, w = df['orientation_X'].tolist() , df['orientation_Y'].tolist() , df['orientation_Z'].tolist() , df['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 df['euler_x'] = nx
df['euler_y'] = ny
df['euler_z'] = nz
df['total_angle'] = np.sqrt(np.square(df['euler_x'])+ np.square(df['euler_y'])+ np.square(df['euler_z']))
df['angle_vs_acc'] = df['total_angle'] / df['total_linear_acceleration']
df['angle_vs_vel'] = df['total_angle'] / df['total_angular_velocity']
return df<feature_engineering>
import missingno as msno
def perform_feature_engineering(df):
df_out = pd.DataFrame()
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def mean_abs_change(x):
return np.mean(np.abs(np.diff(x)))
 for col in df.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
df_out[col + '_mean'] = df.groupby(['series_id'])[col].mean()
df_out[col + '_min'] = df.groupby(['series_id'])[col].min()
df_out[col + '_max'] = df.groupby(['series_id'])[col].max()
df_out[col + '_std'] = df.groupby(['series_id'])[col].std()
df_out[col + '_mad'] = df.groupby(['series_id'])[col].mad()
df_out[col + '_med'] = df.groupby(['series_id'])[col].median()
df_out[col + '_skew'] = df.groupby(['series_id'])[col].skew()
df_out[col + '_range'] = df_out[col + '_max'] - df_out[col + '_min']
df_out[col + '_max_to_min'] = df_out[col + '_max'] / df_out[col + '_min']
df_out[col + '_mean_abs_change'] = df.groupby('series_id')[col].apply(mean_abs_change)
 df_out[col + '_mean_change_of_abs_change'] = df.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df_out[col + '_abs_max'] = df.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 df_out[col + '_abs_min'] = df.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 df_out[col + '_abs_mean'] = df.groupby('series_id')[col].apply(lambda x: np.mean(np.abs(x)))
 df_out[col + '_abs_std'] = df.groupby('series_id')[col].apply(lambda x: np.std(np.abs(x)))
 df_out[col + '_abs_avg'] =(df_out[col + '_abs_min'] + df_out[col + '_abs_max'])/2
df_out[col + '_abs_range'] = df_out[col + '_abs_max'] - df_out[col + '_abs_min']
return df_out<feature_engineering>
train_df.loc[(pd.isnull(train_df['v2a1'])& train_df['tipovivi1'] == 1), 'v2a1'] = 0
%%time
X_train = perform_euler_factors_calculation(X_train )<compute_test_metric>
train_df = train_df.loc[pd.notnull(train_df['v2a1'])]
%%time
X_test = perform_euler_factors_calculation(X_test )<feature_engineering>
train_df['v18q1'].dropna().value_counts()
%%time
X_train = perform_feature_engineering(X_train[features] )<feature_engineering>
train_df.loc[(pd.isnull(train_df['v18q1'])) , 'v18q1'] = 0
%%time
X_test = perform_feature_engineering(X_test[features] )<train_model>
train_df['rez_esc'].dropna().value_counts()
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))<sort_values>
train_df.loc[pd.isnull(train_df['rez_esc']), 'rez_esc'] = 0
%%time
correlations = X_train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]<count_unique_values>
train_df.loc[pd.isnull(train_df['meaneduc']),('edjefa', 'edjefe', 'escolari', 'meaneduc')]
n_top_corr = correlations[correlations[0]==1.0].shape[0]
print("There are {} different features pairs with correlation factor 1.0.".format(n_top_corr))<drop_column>
train_df.loc[pd.isnull(train_df['meaneduc']), 'meaneduc'] = train_df.loc[pd.isnull(train_df['meaneduc']), 'escolari']
drop_features = list(correlations.head(n_top_corr)['level_0'].unique())
 X_train = X_train.drop(drop_features,axis=1)
 X_test = X_test.drop(drop_features,axis=1 )<train_model>
train_df.loc[pd.isnull(train_df['SQBmeaned']), 'SQBmeaned'] = train_df.loc[pd.isnull(train_df['SQBmeaned']), 'meaneduc']**2
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))<categorify>
len(train_df.loc[train_df['age'] == 0].index )
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )<categorify>
train_df = train_df.loc[train_df['age']!=0]
X_train.fillna(0, inplace = True)
 X_train.replace(-np.inf, 0, inplace = True)
 X_train.replace(np.inf, 0, inplace = True)
 X_test.fillna(0, inplace = True)
 X_test.replace(-np.inf, 0, inplace = True)
 X_test.replace(np.inf, 0, inplace = True )<normalization>
train_df.groupby('dependency' ).size()
scaler = StandardScaler()
X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train))
X_test_scaled = pd.DataFrame(scaler.transform(X_test))
print("Scaled !" )<choose_model_class>
mode = train_df.loc[(train_df['dependency'] != 'yes')&(train_df['dependency'] != 'no'), 'dependency'].astype(float ).mode()
mode
folds = StratifiedKFold(n_splits=49, shuffle=True, random_state=2018 )<predict_on_test>
def mutate_columns(df):
df.loc[df['dependency']=='no', 'dependency'] = np.sqrt(df['SQBdependency'])
 df.loc[df['dependency']=='yes', 'dependency'] = np.sqrt(df['SQBdependency'])
 df['dependency'] = df['dependency'].astype('float16')
 df.loc[df['edjefe']=='no', 'edjefe'] = 0
df.loc[df['edjefe']=='yes', 'edjefe'] = 4
df['edjefe'] = df['edjefe'].astype('uint8')
 df.loc[df['edjefa']=='no', 'edjefa'] = 0
df.loc[df['edjefa']=='yes', 'edjefa'] = 4
df['edjefa'] = df['edjefa'].astype('uint8' )
sub_preds_rf = np.zeros(( X_test_scaled.shape[0], 9))
oof_preds_rf = np.zeros(( X_train_scaled.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train_scaled, y_train['surface'])) :
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train_scaled.iloc[trn_idx], y_train['surface'][trn_idx])
 oof_preds_rf[val_idx] = clf.predict(X_train_scaled.iloc[val_idx])
 sub_preds_rf += clf.predict_proba(X_test_scaled)/ folds.n_splits
score += clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])
 print('Fold: {} score: {}'.format(fold_,clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])))
 print('Avg Accuracy', score / folds.n_splits )<split>
tdf = train_df[['Target']]
n_train_df = train_df
for col in ['v18q', 'refrig', 'computer', 'television', 'mobilephone', 'v14a']:
n_train_df[col] = n_train_df[col].astype('category')
 dfcat = pd.get_dummies(n_train_df[[ 'v18q', 'refrig', 'computer', 'television', 'mobilephone', 'v14a']])
 df_ = pd.concat([dfcat, tdf], axis=1 )
USE_LGB = True
if(USE_LGB):
sub_preds_lgb = np.zeros(( X_test.shape[0], 9))
oof_preds_lgb = np.zeros(( X_train.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train['surface'])) :
train_x, train_y = X_train.iloc[trn_idx], y_train['surface'][trn_idx]
valid_x, valid_y = X_train.iloc[val_idx], y_train['surface'][val_idx]
clf = LGBMClassifier(
 nthread=-1,
n_estimators=2000,
learning_rate=0.01,
boosting_type='gbdt',
is_unbalance=True,
objective='multiclass',
numclass=9,
silent=-1,
verbose=-1,
feval=None)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
verbose= 1000, early_stopping_rounds= 200)
 oof_preds_lgb[val_idx] = clf.predict(valid_x)
 sub_preds_lgb += clf.predict_proba(X_test)/ folds.n_splits
score += clf.score(valid_x, valid_y)
 print('Fold: {} score: {}'.format(fold_,clf.score(valid_x, valid_y)))
 print('Avg Accuracy', score / folds.n_splits )<save_to_csv>
df_ = df_.groupby(['Target'] ).sum()
df_.reset_index(inplace = True )
submission = pd.read_csv(os.path.join(PATH,'sample_submission.csv'))
submission['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
submission.to_csv('submission_rf.csv', index=False)
 submission.head(10 )<save_to_csv>
df_q = train_df[['Target', 'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3']]
df_q.loc[df_q['epared1'] == 1, 'wall'] = 'Bad'
df_q.loc[df_q['epared2'] == 1, 'wall'] = 'Regular'
df_q.loc[df_q['epared3'] == 1, 'wall'] = 'Good'
df_q.loc[df_q['etecho1'] == 1, 'roof'] = 'Bad'
df_q.loc[df_q['etecho2'] == 1, 'roof'] = 'Regular'
df_q.loc[df_q['etecho3'] == 1, 'roof'] = 'Good'
df_q.loc[df_q['eviv1'] == 1, 'floor'] = 'Bad'
df_q.loc[df_q['eviv2'] == 1, 'floor'] = 'Regular'
df_q.loc[df_q['eviv3'] == 1, 'floor'] = 'Good'
df_q = df_q[['Target', 'wall', 'roof', 'floor']]
USE_LGB = True
if(USE_LGB):
submission['surface'] = le.inverse_transform(sub_preds_lgb.argmax(axis=1))
submission.to_csv('submission_lgb.csv', index=False)
 submission.head(10 )<import_modules>
def feature_engineer(x):
x['escolari_age'] = x['escolari'] / x['age']
x['refrig'] = x['refrig'].astype(int)
 x['computer'] = x['computer'].astype(int)
 x['television'] = x['television'].astype(int)
 x['mobilephone'] = x['mobilephone'].astype(int)
 x['v14a'] = x['v14a'].astype(int)
 x['v18q'] = x['v18q'].astype(int)
 x['epared1'] = x['epared1'].astype(int)
 x['epared2'] = x['epared2'].astype(int)
 x['epared3'] = x['epared3'].astype(int)
 x['etecho1'] = x['etecho1'].astype(int)
 x['etecho2'] = x['etecho2'].astype(int)
 x['etecho3'] = x['etecho3'].astype(int)
 x['eviv1'] = x['eviv1'].astype(int)
 x['eviv2'] = x['eviv2'].astype(int)
 x['eviv3'] = x['eviv3'].astype(int)
 x['abastaguadentro'] = x['abastaguadentro'].astype(int)
 x['abastaguafuera'] = x['abastaguafuera'].astype(int)
 x['abastaguano'] = x['abastaguano'].astype(int)
 x['abastaguano'] = x['abastaguano'].astype(int)
 x[['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']] = x[['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']].apply(pd.to_numeric)
 x['appliances'] =(x['refrig'] + x['computer'] + x['television'])
 x['rent_by_hhsize'] = x['v2a1'] / x['hhsize']
x['rent_by_people'] = x['v2a1'] / x['r4t3']
x['rent_by_rooms'] = x['v2a1'] / x['rooms']
x['rent_by_living'] = x['v2a1'] / x['tamviv']
x['rent_by_minor'] = x['v2a1'] / x['hogar_nin']
x['rent_by_adult'] = x['v2a1'] / x['hogar_adul']
x['rent_by_dep'] = x['v2a1'] / x['dependency']
x['rent_by_head_educ'] = x['v2a1'] /(x['edjefe'] + x['edjefa'])
 x['rent_by_educ'] = x['v2a1'] / x['meaneduc']
x['rent_by_numPhone'] = x['v2a1'] / x['qmobilephone']
x['rent_by_gadgets'] = x['v2a1'] /(x['computer'] + x['mobilephone'] + x['v18q'])
 x['rent_by_num_gadgets'] = x['v2a1'] /(x['v18q1'] + x['qmobilephone'])
 x['rent_by_appliances'] = x['v2a1'] / x['appliances']
x['under12'] = x['r4t1']/x['r4t3']
x['under12_male'] = x['r4h1']/x['r4t3']
x['under12_female'] = x['r4m1']/x['r4t3']
x['Proportion_male'] = x['r4h3']/x['r4t3']
x['Proportion_female'] = x['r4m3']/x['r4t3']
x['tablet_density'] = x['v18q1'] / x['r4t3']
x['phone_density'] = x['qmobilephone'] / x['r4t3']
x['wall_qual'] = x['epared3'] - x['epared1']
x['roof_qual'] = x['etecho3'] - x['etecho1']
x['floor_qual'] = x['eviv3'] - x['eviv1']
x['water_qual'] = x['abastaguadentro'] - x['abastaguano']
x['house_qual'] = x['wall_qual'] + x['roof_qual'] + x['floor_qual']
x['person_per_room'] = x['hhsize'] / x['rooms']
x['person_per_appliances'] = x['hhsize'] / x['appliances']
x['educ_qual'] =(1 * x['instlevel1'])+(2 * x['instlevel2'])+(3 * x['instlevel3'])+(4 * x['instlevel4'])+(5 * x['instlevel5'])+(6 * x['instlevel6'])+(7 * x['instlevel7'])+(8 * x['instlevel8'])+(9 * x['instlevel9'])
 x['educ_by_individual'] = x['escolari']/x['r4t3']
x['educ_by_adult'] = x['escolari']/(x['r4t3'] - x['r4t1'])
 x['educ_by_child'] = x['escolari']/x['r4t1']
x['max_educ'] = np.max(x[['edjefa','edjefe']])
 def reverse_label_encoding(row, df):
for c in df.columns:
if row[c] == 1:
return int(c[-1])
 def rate_sanitary(row, df):
c = df.columns.tolist() [0]
if row[c] == 'sanitario2':
return 3
elif row[c] == 'sanitario3':
return 2
elif row[c] == 'sanitario5':
return 1
else:
return 0
def rate_cooking(row, df):
c = df.columns.tolist() [0]
if row[c] == 'energcocinar2':
return 3
elif row[c] == 'energcocinar3':
return 2
elif row[c] == 'energcocinar4':
return 1
else:
return 0
def rate_rubbish(row, df):
c = df.columns.tolist() [0]
if row[c] == 'elimbasu1':
return 1
elif row[c] == 'elimbasu2':
return 2
else:
return 0
x['sanitary'] = x.apply(lambda q: reverse_label_encoding(q, x[['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']]), axis=1)
 x['cooking'] = x.apply(lambda q: reverse_label_encoding(q, x[['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4']]), axis=1)
 x['rubbish'] = x.apply(lambda q: reverse_label_encoding(q, x[['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6']]), axis=1)
 x['region'] = x.apply(lambda q: reverse_label_encoding(q, x[['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']]), axis=1)
 x['sanitary_i'] = x.apply(lambda q: rate_sanitary(q, x[['sanitary']]), axis = 1)
 x['cooking_i'] = x.apply(lambda q: rate_cooking(q, x[['cooking']]), axis = 1)
 x['rubbish_i'] = x.apply(lambda q: rate_rubbish(q, x[['rubbish']]), axis = 1)
 x['zone'] = x['area1'] - x['area2']
x.replace([np.inf, -np.inf], 0, inplace = True)
 x.fillna(0, inplace = True )
import os
import time
import numpy as np
import pandas as pd
from seaborn import countplot,lineplot, barplot
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score, confusion_matrix
from bayes_opt import BayesianOptimization
import lightgbm as lgb
import xgboost as xgb
import catboost as cb<load_from_csv>
feature_engineer(train_df)
 feature_engineer(test_df )
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )<count_values>
def agg_features(y):
mean_list = []
count_list = ['escolari_age', 'phone_density', 'rez_esc', 'dis', 'male', 'female','v2a1','house_qual', 'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total', 'escolari','estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7','parentesco1', 'parentesco2', 'parentesco3',
'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12']
for group in ['idhogar']:
for item in count_list:
for agg_m in ['mean','std','min','max','sum']:
id_agg = y[item].groupby(y[group] ).agg(agg_m)
 new_col = item + '_' + agg_m + '_' + group
y[new_col] = id_agg
drop_ = ['sanitary', 'cooking', 'rubbish', 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6', 'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6', 'tamhog', 'tamviv', 'hhsize', 'v18q',
'v14a', 'agesq','mobilephone', 'female', 'estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7', 'parentesco2', 'parentesco3',
'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12', 'lugar1', 'lugar2', 'lugar3', 'lugar4',
'lugar5', 'lugar6', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned',
'r4t1', 'r4t2', 'r4t3']
y.drop(( drop_), inplace = True, axis = 1)
 y.replace([np.inf, -np.inf], 0, inplace = True)
 y.fillna(0, inplace = True)
 return y
len(tr.measurement_number.value_counts() )<normalization>
train_df = agg_features(train_df)
 test_df = agg_features(test_df )
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(actual):
new = pd.DataFrame()
actual['total_angular_velocity'] =(actual['angular_velocity_X'] ** 2 + actual['angular_velocity_Y'] ** 2 + actual['angular_velocity_Z'] ** 2)** 0.5
actual['total_linear_acceleration'] =(actual['linear_acceleration_X'] ** 2 + actual['linear_acceleration_Y'] ** 2 + actual['linear_acceleration_Z'] ** 2)** 0.5
actual['acc_vs_vel'] = actual['total_linear_acceleration'] / actual['total_angular_velocity']
x, y, z, w = actual['orientation_X'].tolist() , actual['orientation_Y'].tolist() , actual['orientation_Z'].tolist() , actual['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
actual['total_angle'] =(actual['euler_x'] ** 2 + actual['euler_y'] ** 2 + actual['euler_z'] ** 2)** 5
actual['angle_vs_acc'] = actual['total_angle'] / actual['total_linear_acceleration']
actual['angle_vs_vel'] = actual['total_angle'] / actual['total_angular_velocity']
def f1(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def f2(x):
return np.mean(np.abs(np.diff(x)))
 for col in actual.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
new[col + '_mean'] = actual.groupby(['series_id'])[col].mean()
new[col + '_min'] = actual.groupby(['series_id'])[col].min()
new[col + '_max'] = actual.groupby(['series_id'])[col].max()
new[col + '_std'] = actual.groupby(['series_id'])[col].std()
new[col + '_max_to_min'] = new[col + '_max'] / new[col + '_min']
new[col + '_mean_abs_change'] = actual.groupby('series_id')[col].apply(f2)
 new[col + '_mean_change_of_abs_change'] = actual.groupby('series_id')[col].apply(f1)
 new[col + '_abs_max'] = actual.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 new[col + '_abs_min'] = actual.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 return new<categorify>
train_df = train_df.loc[train_df['parentesco1'] == 1]
train_df.fillna(value=0, inplace=True )
le = LabelEncoder()
target['surface'] = le.fit_transform(target['surface'] )<data_type_conversions>
y = train_df[['Target']]
x = train_df.drop(['Target','Id','idhogar'], axis = 1 )
tr.fillna(0, inplace = True)
 te.fillna(0, inplace = True )<categorify>
train_df.columns.tolist()
tr.replace(-np.inf, 0, inplace = True)
 tr.replace(np.inf, 0, inplace = True)
 te.replace(-np.inf, 0, inplace = True)
 te.replace(np.inf, 0, inplace = True )<predict_on_test>
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=546789)
 sub_preds_rf = np.zeros(( te.shape[0], 9))
oof_preds_rf = np.zeros(( tr.shape[0]))
score = 0
for i,(train_index, test_index)in enumerate(folds.split(tr, target['surface'])) :
print('-'*20, i, '-'*20)
 clf = RandomForestClassifier(n_estimators = 200, n_jobs = -1)
 clf.fit(tr.iloc[train_index], target['surface'][train_index])
 oof_preds_rf[test_index] = clf.predict(tr.iloc[test_index])
 sub_preds_rf += clf.predict_proba(te)/ folds.n_splits
score += clf.score(tr.iloc[test_index], target['surface'][test_index])
 print('score ', clf.score(tr.iloc[test_index], target['surface'][test_index]))
importances = clf.feature_importances_
indices = np.argsort(importances)
 features = tr.columns
hm = 30
plt.figure(figsize=(7, 10))
plt.title('Feature Importances')
 plt.barh(range(len(indices[:hm])) , importances[indices][:hm], color='b', align='center')
 plt.yticks(range(len(indices[:hm])) , [features[i] for i in indices])
 plt.xlabel('Relative Importance')
 plt.show()
print('Avg Accuracy', score / folds.n_splits )<save_to_csv>
clf = RandomForestClassifier()
clf.fit(x, y)
 imp = clf.feature_importances_
name = np.array(x.columns.values.tolist())
 df_imp = pd.DataFrame({'feature':name, 'importance':imp})
 df_imp = df_imp.sort_values(by='importance', ascending=False )
ss['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
ss.to_csv('rf.csv', index=False)
 ss.head(10 )<set_options>
def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list
sns.set() 
<load_from_csv>
important_cols = df_imp.loc[(df_imp['importance']>0),'feature']
%%time
X_train_df = pd.read_csv(PATH+"X_train.csv")
 X_test_df = pd.read_csv(PATH+"X_test.csv")
 Y_train_df = pd.read_csv(PATH+"y_train.csv")
 sub = pd.read_csv(PATH+"sample_submission.csv" )<count_missing_values>
from sklearn.preprocessing import StandardScaler
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
features = [c for c in x_.columns if c not in ['Target']]
target = train_df['Target']
missing_data(X_train_df )<count_missing_values>
from sklearn.model_selection import train_test_split
missing_data(X_test_df )<count_missing_values>
from sklearn.model_selection import train_test_split
missing_data(Y_train_df )<compute_test_metric>
smote_tomek = SMOTETomek(random_state=0)
 X_resampled, y_resampled = smote_tomek.fit_sample(x_, target )
def _kurtosis(x):
return kurtosis(x)
 def CPT5(x):
den = len(x)*np.exp(np.std(x))
return sum(np.exp(x)) /den
def skewness(x):
return skew(x)
 def SSC(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
xn_i1 = x[0:len(x)-2]
ans = np.heaviside(( xn-xn_i1)*(xn-xn_i2),0)
 return sum(ans[1:])
 def wave_length(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
return sum(abs(xn_i2-xn))
def norm_entropy(x):
tresh = 3
return sum(np.power(abs(x),tresh))
def SRAV(x):
SRA = sum(np.sqrt(abs(x)))
 return np.power(SRA/len(x),2)
 def mean_abs(x):
return sum(abs(x)) /len(x)
 def zero_crossing(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
return sum(np.heaviside(-xn*xn_i2,0)) 
<create_dataframe>
def feature_extraction(raw_frame):
frame = pd.DataFrame()
raw_frame['angular_velocity'] = raw_frame['angular_velocity_X'] + raw_frame['angular_velocity_Y'] + raw_frame['angular_velocity_Z']
raw_frame['linear_acceleration'] = raw_frame['linear_acceleration_X'] + raw_frame['linear_acceleration_Y'] + raw_frame['linear_acceleration_Z']
raw_frame['velocity_to_acceleration'] = raw_frame['angular_velocity'] / raw_frame['linear_acceleration']
for col in raw_frame.columns[3:]:
frame[col + '_mean'] = raw_frame.groupby(['series_id'])[col].mean()
frame[col + '_CPT5'] = raw_frame.groupby(['series_id'])[col].apply(CPT5)
 frame[col + '_SSC'] = raw_frame.groupby(['series_id'])[col].apply(SSC)
 frame[col + '_skewness'] = raw_frame.groupby(['series_id'])[col].apply(skewness)
 frame[col + '_wave_lenght'] = raw_frame.groupby(['series_id'])[col].apply(wave_length)
 frame[col + '_norm_entropy'] = raw_frame.groupby(['series_id'])[col].apply(norm_entropy)
 frame[col + '_SRAV'] = raw_frame.groupby(['series_id'])[col].apply(SRAV)
 frame[col + '_kurtosis'] = raw_frame.groupby(['series_id'])[col].apply(_kurtosis)
 frame[col + '_mean_abs'] = raw_frame.groupby(['series_id'])[col].apply(mean_abs)
 frame[col + '_zero_crossing'] = raw_frame.groupby(['series_id'])[col].apply(zero_crossing)
 return frame<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(X_resampled, y_resampled, test_size=0.1, random_state=1)
train_df = feature_extraction(X_train_df)
 test_df = feature_extraction(X_test_df)
<categorify>
pd.options.display.width =300
X_resampled = pd.DataFrame(X_resampled)
 X_resampled.columns = features
y_resampled = pd.DataFrame(y_resampled)
 y_resampled.columns = ['Target']
X_train_df = pd.DataFrame(X_train)
 X_train_df.columns = features
train_df.fillna(0, inplace = True)
 train_df.replace(-np.inf, 0, inplace = True)
 train_df.replace(np.inf, 0, inplace = True)
 test_df.fillna(0, inplace = True)
 test_df.replace(-np.inf, 0, inplace = True)
 test_df.replace(np.inf, 0, inplace = True )<normalization>
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight
from sklearn.metrics import f1_score
from sklearn.ensemble import VotingClassifier
scaler = StandardScaler()
X_train_scaled = pd.DataFrame(scaler.fit_transform(train_df))
X_test_scaled = pd.DataFrame(scaler.transform(test_df))<categorify>
skf = StratifiedKFold(n_splits=5, shuffle=True )
le = LabelEncoder()
target = le.fit_transform(Y_train_df['surface'] )<save_to_csv>
lgbm = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.88, min_child_samples = 90, num_leaves = 16, subsample = 0.94 )
sub['surface'] = le.inverse_transform(predictions.argmax(axis=1))
sub.to_csv('submission_lgbm.csv', index=False )<split>
predicts_lgb = []
for train_index, test_index in skf.split(X_resampled, y_resampled):
X_t, X_v = X_resampled.iloc[train_index], X_resampled.iloc[test_index]
y_t, y_v = y_resampled.iloc[train_index], y_resampled.iloc[test_index]
lgbm.fit(X_t, y_t)
 predicts_lgb.append(lgbm.predict(test_df[features]))
X_train, X_test, Y_train, Y_test = train_test_split(X_train_scaled, target, test_size=0.2, random_state=23, stratify=target )<train_model>
optimal_k = 120
rf_acc = []
rf_clf = RandomForestClassifier(n_estimators=optimal_k, random_state=0)
 rf_clf.fit(X_train, Y_train)
 Y_pred = rf_clf.predict(X_test)
 acc = accuracy_score(Y_pred,Y_test)
 print('Acc_Test:')
 print(float("%0.3f" %(100*acc)) )<train_model>
submission = pd.DataFrame()
submission['Id'] = test_df['Id']
submission['Target'] = np.array(predicts_lgb ).mean(axis=0 ).round().astype(int)
rf_clf.fit(X_train_scaled, target )<save_to_csv>
submission.to_csv('submissions.csv', index=False )