Kaggle competition: Costa Rican Household Poverty Level Prediction
%%time
train_data = pd.read_csv('.. /input/titanic/train.csv')
 train_data.head()<load_from_csv>
data=pd.read_csv('.. /input/train.csv' )
test_data = pd.read_csv('.. /input/titanic/test.csv')
 test_data.head()<load_from_csv>
data[(data['dependency']=='no')&(data['SQBdependency']!=0)]
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission.head()<feature_engineering>
data[(data['dependency']=='yes')&(data['SQBdependency']!=1)]
def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
def create_extra_features(data):
data['Ticket_type'] = data['Ticket'].map(lambda x: x[0:3])
 data['Name_Words_Count'] = data['Name'].map(lambda x: len(x.split()))
data['Has_Cabin'] = data["Cabin"].map(lambda x: 1 - int(type(x)== float))
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['CategoricalFare'] = pd.qcut(data['Fare'], 5 ).astype(str)
 data['CategoricalAge'] = pd.cut(data['Age'], 5 ).astype(str)
 data['Title'] = data['Name'].apply(get_title ).replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs')
 data['Title'] = data['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5} ).fillna(0)
 return data
train_data = create_extra_features(train_data)
 test_data = create_extra_features(test_data )<split>
data[(data['dependency']=='3')&(data['SQBdependency']!=9)]
tr_data, te_data = train_test_split(train_data,
test_size=TEST_SIZE,
stratify=train_data['Survived'],
random_state=RANDOM_STATE)
 print('Data splitted.Parts sizes: tr_data = {}, te_data = {}'.format(tr_data.shape, te_data.shape))<compute_test_metric>
data['dependency']=np.sqrt(data['SQBdependency'] )
%%time
def acc_score(y_true, y_pred, **kwargs):
return accuracy_score(y_true,(y_pred > 0.5 ).astype(int), **kwargs)
 task = Task('binary', metric = acc_score )<define_variables>
data[['edjefe', 'edjefa', 'SQBedjefe']][data['edjefe']=='yes']
%%time
roles = {
'target': 'Survived',
'drop': ['PassengerId', 'Name','Ticket'],
}<predict_on_test>
data[(data['edjefe']=='yes')&(data['edjefa']!='no')]
%%time
automl = TabularAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(tr_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))<compute_test_metric>
data[(data['edjefa']=='yes')&(data['parentesco1']==1)][['edjefe', 'edjefa', 'parentesco1', 'escolari']]
%%time
test_pred = automl.predict(te_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(tr_data['Survived'].values, oof_pred.data[:, 0])))
 print('TEST score: {}'.format(acc_score(te_data['Survived'].values, test_pred.data[:, 0])) )<train_model>
data[data['edjefe']=='yes'][['edjefe', 'edjefa','age', 'escolari', 'parentesco1','male', 'female', 'idhogar']]
%%time
automl = TabularUtilizedAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(tr_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))<compute_test_metric>
data[(data['edjefe']=='no')&(data['edjefa']=='no')][['edjefe', 'edjefa', 'age', 'escolari', 'female', 'male', 'Id', 'parentesco1', 'idhogar']]
%%time
test_pred = automl.predict(te_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(tr_data['Survived'].values, oof_pred.data[:, 0])))
 print('TEST score: {}'.format(acc_score(te_data['Survived'].values, test_pred.data[:, 0])) )<train_model>
data[(data['edjefe']=='yes')& data['parentesco1']==1][['escolari']]
%%time
automl = TabularUtilizedAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
general_params = {'use_algos': [['linear_l2', 'lgb', 'lgb_tuned']]},
reader_params = {'n_jobs': N_THREADS})
 oof_pred = automl.fit_predict(train_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred[:10], oof_pred.shape))<predict_on_test>
conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data[['edjefe', 'edjefa', 'edjefx']][:15]
%%time
test_pred = automl.predict(test_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred[:10], test_pred.shape))
print('Check scores...')
 print('OOF score: {}'.format(acc_score(train_data['Survived'].values, oof_pred.data[:, 0])) )<save_to_csv>
data.columns[data.isna().sum() !=0]
submission['Survived'] =(test_pred.data[:, 0] > 0.5 ).astype(int)
 submission.to_csv('automl_utilized_600.csv', index = False )<import_modules>
data[data['meaneduc'].isnull() ]
from fbprophet import Prophet<load_from_csv>
data[data['meaneduc'].isnull() ][['Id','idhogar','edjefe','edjefa', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'age', 'escolari']]
train = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/train_1.csv.zip" )<define_variables>
print(len(data[data['idhogar']==data.iloc[1291]['idhogar']]))
print(len(data[data['idhogar']==data.iloc[1840]['idhogar']]))
print(len(data[data['idhogar']==data.iloc[2049]['idhogar']]))
all_pages = train['Page']<define_variables>
meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
first_page = all_pages[0]
first_page<create_dataframe>
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
df = pd.DataFrame(train_allT, columns = ['Date',first_page])
 df = df.rename(columns={'Date':'ds', first_page:'y'} )<train_model>
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
m = Prophet()
m.fit(df )<prepare_output>
data['v2a1'].isnull().sum()
future = m.make_future_dataframe(periods=60)
 future.head()<predict_on_test>
norent=data[data['v2a1'].isnull() ]
print("Owns his house:", norent[norent['tipovivi1']==1]['Id'].count())
 print("Owns his house paying installments", norent[norent['tipovivi2']==1]['Id'].count())
 print("Rented ", norent[norent['tipovivi3']==1]['Id'].count())
 print("Precarious ", norent[norent['tipovivi4']==1]['Id'].count())
 print("Other ", norent[norent['tipovivi5']==1]['Id'].count())
 print("Total ", 6860 )
forecast = m.predict(future)
 forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()<prepare_output>
data['v2a1']=data['v2a1'].fillna(0 )
result = forecast['yhat']<split>
data['v18q1'].isna().sum()
sub_result = result[-60:]<load_from_csv>
tabletnan=data[data['v18q1'].isnull() ]
tabletnan[tabletnan['v18q']==0]['Id'].count()
key = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/key_1.csv.zip" )<filter>
data['v18q1']=data['v18q1'].fillna(0 )
listOfIds = key.index[key['Page'].str.contains(first_page)].values<load_from_csv>
data['rez_esc'].isnull().sum()
sub = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/sample_submission_1.csv.zip" )<feature_engineering>
data[data['rez_esc']>1][['age', 'escolari', 'rez_esc']][:20]
sub['Visits'].loc[listOfIds.min() :listOfIds.max() ] = sub_result.values.round() 
<predict_on_test>
rez_esc_nan=data[data['rez_esc'].isnull() ]
rez_esc_nan[(rez_esc_nan['age']<18)& rez_esc_nan['escolari']>0][['age', 'escolari']]
def my_function() :
for page in all_pages:
df = pd.DataFrame(train_allT, columns = ['Date',page])
 df = df.rename(columns={'Date':'ds', page:'y'})
 m = Prophet()
m.fit(df)
 future = m.make_future_dataframe(periods=60)
 forecast = m.predict(future)
 result = forecast['yhat']
sub_result = result[-60:]
listOfIds = key.index[key['Page'].str.contains(page)].values
sub['Visits'].loc[listOfIds.min() :listOfIds.max() ] = sub_result.values.round() 
<load_from_csv>
data['rez_esc']=data['rez_esc'].fillna(0 )
print('Reading data...')
 key_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/key_2.csv.zip')
 train_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/train_2.csv.zip')
 ss_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/sample_submission_2.csv.zip')
<define_variables>
d={}
weird=[]
for row in data.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target
print('Preprocessing...')
 print('Processing...')
 ids = key_1.Id.values
pages = key_1.Page.values 
<define_variables>
len(set(weird))
print('key_1...')
 d_pages = {}
for id, page in zip(ids, pages):
d_pages[id] = page[:-11] 
<save_to_csv>
data[data['idhogar']==weird[2]][['idhogar','parentesco1', 'Target']]
for i, ss_id in enumerate(ss_ids):
ss_visits[i] = d_visits[d_pages[ss_id]]
print('Saving submission...')
 subm = pd.DataFrame({'Id': ss_ids, 'Visits': ss_visits})
 subm.to_csv('submission.csv', index=False )<set_options>
for i in set(weird):
hhold=data[data['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
data.at[idx, 'Target']=target
warnings.filterwarnings('ignore' )<data_type_conversions>
data[data['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]
print('Pre-processing and feature engineering train data...')
 train_flattened = pd.melt(train[list(train.columns[-49:])+['Page']], id_vars='Page', var_name='date', value_name='Visits')
 train_flattened['date'] = train_flattened['date'].astype('datetime64[ns]')
 train_flattened['weekend'] =(( train_flattened.date.dt.dayofweek)// 5 == 1 ).astype(float)
<create_dataframe>
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data
times_series_means = pd.DataFrame(mean_group ).reset_index(drop=False )<rename_columns>
%matplotlib inline
df_date_index = times_series_means[['date','Visits']].set_index('date' )<import_modules>
data_undersampled=data.drop(data.query('Target == 4' ).sample(frac=.75 ).index )
result = adfuller(df_date_index)
 print(result[0])
 print(result[1] )<train_on_grid>
X=data_undersampled.drop(['Id', 'idhogar', 'Target', 'edjefe', 'edjefa'], axis=1)
 y=data_undersampled['Target']
order_aic_bic=[]
for p in range(3):
for q in range(3):
try:
model = SARIMAX(df_date_index, order=(p,0,q), seasonal_order=(1,2,0,7))
results = model.fit()
order_aic_bic.append(( p,q, results.aic, results.bic))
print(p,q,results.aic, results.bic)
 except:
print(p, q, None, None )<install_modules>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )
!pip install pmdarima<import_modules>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
import pmdarima as pm<train_model>
clf = RandomForestClassifier()
params={'n_estimators': list(range(40,61, 1)) }
gs = GridSearchCV(clf, params, cv=5 )
model = SARIMAX(df_date_index,
order=(2,0,1),
seasonal_order=(1,1,1,7),
trend='c')
 results = model.fit()<prepare_output>
gs.fit(X_train, y_train )
forecast_object = results.get_forecast(steps=90)
 mean = forecast_object.predicted_mean
conf_int = forecast_object.conf_int()
dates = mean.index<data_type_conversions>
preds=gs.predict(X_test )
df_date_index.index = pd.to_datetime(df_date_index.index )<predict_on_test>
print(classification_report(y_test, preds))
print(mean.iloc[-1])
 print(conf_int.iloc[-1] )<compute_test_metric>
print(confusion_matrix(y_test, preds))
y_forecasted = pred.predicted_mean
y_truth = df_date_index['2016-10-01':]
mse =(( y_forecasted - y_truth)** 2 ).mean()
print('The Mean Squared Error of our forecasts is {}'.format(round(mse, 2)))
<define_variables>
print(gs.best_params_)
 print(gs.best_score_)
 print(gs.best_estimator_ )
N_FEATURES = 22
THRESHOLD = 73<load_from_csv>
cvres = gs.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(mean_score), params )
train_df = pd.read_csv("/kaggle/input/train.zip")
 train_df[train_df.columns[1:]] = train_df[train_df.columns[1:]].astype(np.float32)
 train_df.shape<filter>
test_data=pd.read_csv('.. /input/test.csv' )
good_ids = set(train_df.loc[train_df['Ref'].notna() , 'Id'])
 train_df = train_df[train_df['Id'].isin(good_ids)]
train_df.shape<groupby>
ids=test_data['Id']
test_data.drop(['Id', 'idhogar'], axis=1, inplace=True )
train_groups = train_df.groupby("Id")
 train_size = len(train_groups )<prepare_x_and_y>
test_predictions=gs.predict(test_data )
X_train = np.zeros(( train_size, MAX_SEQ_LEN, N_FEATURES), dtype=np.float32)
 y_train = np.zeros(train_size, dtype=np.float32)
 i = 0
for _, group in train_groups:
X = group.values
seq_len = X.shape[0]
X_train[i,:seq_len,:] = X[:,1:23]
y_train[i] = X[0,23]
i += 1
del X
del train_groups
X_train.shape, y_train.shape<load_from_csv>
submit=pd.DataFrame({'Id': ids, 'Target': test_predictions} )
<groupby><EOS> 
submit.to_csv('submit.csv', index=False )