Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<prepare_x_and_y>
%matplotlib inline
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_valid = olid_valid.toxic.values
y_olid_test =(olid_test_label.label == 'OFF' ).astype(int ).values<define_variables>
train = pd.read_csv('.. /input/train.csv')
 print('Shape of train set {}'.format(train.shape))
test = pd.read_csv('.. /input/test.csv')
 print('Shape of test set {}'.format(test.shape))
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<categorify>
def onehot_encode(cat_feat, data):

 encoded = pd.get_dummies(data[cat_feat], prefix=cat_feat, dummy_na=True)
 res = pd.concat([data.drop(columns=[cat_feat]), encoded], axis='columns')
 return res
def add_quality(df, componente='pared', component='wall'):
for i in [1,2,3]:
i_quality =(df['e{}{}'.format(componente, i)] == 1)
 df.loc[i_quality, '{}_quality'.format(component)] = i
return df
def to_english(df, sp_pre='pared', eng_pre='wall_', translate=None):

 for sp in translate.keys() :
spanish_name = sp_pre + '{}'.format(sp)
 english_name = eng_pre + '{}'.format(translate[sp])
 df.rename(columns={spanish_name: english_name}, inplace=True)
 return df
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_valid_encode, y_olid_valid)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<choose_model_class>
def mk_derived_feats(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 return df
def mk_agg_feats(df):
aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['sum', 'mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'sum']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def drop_redundant(df):
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['hhsize', 'area2'], axis=1, inplace=True)
 return df
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
test['Target'] = np.nan
data_all = pd.concat([train, test])
 n_house = data_all['idhogar'].nunique()
print('
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead.from_pretrained('bert-base-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary() 
<set_options>
yes_no_map = {'no': 0, 'yes': 1}
data_all['dependency'] = data_all['dependency'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefe'] = data_all['edjefe'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefa'] = data_all['edjefa'].replace(yes_no_map ).astype(np.float32)
 data_all['idhogar'] = LabelEncoder().fit_transform(data_all['idhogar'])
 data_all = mk_derived_feats(data_all)
 data_all = mk_agg_feats(data_all)
 data_all = drop_redundant(data_all )
for layer in model.layers[:-1]:
layer.trainable = False<train_model>
fe_feats = [ff for ff in data_all.columns if ff.startswith('fe_')]
agg_feats = [ff for ff in data_all.columns if ff.startswith('agg')]
basic_feats = ['dependency']
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
too_high_edu_levels = [ff for ff in agg_feats if('instlevel9' in ff)or('instlevel8' in ff)]
irrelevant_members = [ff for ff in agg_feats if('parentesco9' in ff)or('parentesco5' in ff)
 or('parentesco11' in ff)or('parentesco6' in ff)]
to_drop = too_high_edu_levels + irrelevant_members
to_drop
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
for ff in to_drop:
agg_feats.remove(ff )
sub['toxic'] = model.predict(test_dataset, verbose=0)
 sub.to_csv('submission.csv', index=False )<import_modules>
is_head =(data_all.parentesco1 == 1)
 head_df = data_all.loc[is_head, :]
from sklearn.metrics import roc_auc_score<save_to_csv>
head_df.loc[head_df['male'] == 1, 'head_gender'] = 'male'
head_df.loc[head_df['female'] == 1, 'head_gender'] = 'female'
head_df = onehot_encode('head_gender', head_df)
 head_gender_feats = [cc for cc in head_df.columns if 'head_gender' in cc]
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )<compute_train_metric>
for i in range(1, 10):
head_df.loc[head_df['instlevel{}'.format(i)] == 1, 'head_edu_level'] = i
head_df = head_df.rename(columns={'escolari': 'head_school_years'} )
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]<compute_test_metric>
cols = ['idhogar', 'head_school_years', 'head_edu_level'] + head_gender_feats
data_all = pd.merge(data_all, head_df[cols], how='left', on='idhogar' )
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
<load_pretrained>
house_head_feats = ['head_school_years', 'head_edu_level'] + head_gender_feats
model.save_weights("/kaggle/working/ckpt.h5")
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
data_all = add_quality(data_all, componente='pared', component='wall')
 data_all = add_quality(data_all, componente='techo', component='roof')
 data_all = add_quality(data_all, componente='viv', component='floor')
 print(data_all.shape )
with strategy.scope() :
model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
<load_pretrained>
translate = {'blolad': 'block',
'zocalo': 'socket',
'preb': 'cement',
'des': 'waste',
'mad': 'wood',
'zinc': 'zink',
'fibras': 'natural_fibers',
'other': 'other'}
data_all = to_english(data_all, sp_pre='pared', eng_pre='wall_',
translate=translate)
 wall_feats = [cc for cc in data_all.columns if 'wall_' in cc]
translate = {
'moscer': 'mosaic',
'cemento': 'cement',
'other': 'other',
'natur': 'natural',
'notiene': 'no_floor',
'madera': 'wood'
}
data_all = to_english(data_all, sp_pre='piso', eng_pre='floor_', translate=translate)
 floor_feats = [cc for cc in data_all.columns if 'floor_' in cc]
translate = {
'zinc': 'zinc',
'entrepiso': 'fiber cement',
'cane': 'natural fibers',
'otro': 'other'
}
data_all = to_english(data_all, sp_pre='techo', eng_pre='roof_', translate=translate)
 roof_feats = [cc for cc in data_all.columns if 'roof_' in cc]
material_feats = roof_feats + wall_feats + floor_feats
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
translate = {
'guadentro': 'inside_house',
'guafuera': 'outside_house',
'guano': 'no'
}
data_all = to_english(data_all, sp_pre='abasta', eng_pre='water_provision_',
translate=translate)
 water_feats = [cc for cc in data_all.columns if 'water_provision_' in cc]
translate = {
'public': 'public',
'planpri': 'private_plan',
'noelec': 'no',
'coopele': 'cooperate'
}
data_all = to_english(data_all, sp_pre='', eng_pre='electric_', translate=translate)
 elec_feats = [cc for cc in data_all.columns if 'electric_' in cc]
translate = {
'cinar1': 'no',
'cinar2': 'electricity',
'cinar3': 'gas',
'cinar4': 'charcoal'
}
data_all = to_english(data_all, sp_pre='energco', eng_pre='energy_', translate=translate)
 energy_feats = [cc for cc in data_all.columns if 'energy_' in cc]
translate = {
'1': 'no',
'2': 'sewer',
'3': 'septic_tank',
'5': 'black hole',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='sanitario', eng_pre='toilet_', translate=translate)
 toilet_feats = [cc for cc in data_all.columns if 'toilet_' in cc]
translate = {
'1': 'tanker truck',
'2': 'buried',
'3': 'burning',
'4': 'throw empty place',
'5': 'throw to river',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='elimbasu', eng_pre='rubbish_', translate=translate)
 rubbish_feats = [cc for cc in data_all.columns if 'rubbish_' in cc]
facility_feats = water_feats + elec_feats + energy_feats + toilet_feats + rubbish_feats
n_steps = olid_train_2k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
<load_pretrained>
translate = {
'1': 'own_fully_paid',
'2': 'own_pay_installment',
'3': 'rented',
'4': 'precarious',
'5': 'other'
}
data_all = to_english(data_all, sp_pre='tipovivi', eng_pre='living_type_',
translate=translate)
 live_feats = [cc for cc in data_all.columns if 'living_type_' in cc]
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
head_df = data_all.query('parentesco1 == 1')
 train = head_df.loc[head_df['Target'].notnull() , :]
test = head_df.loc[head_df['Target'].isnull() , :]
n_steps = olid_train_5k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)) )<load_pretrained>
features = basic_feats + house_head_feats + material_feats + facility_feats + live_feats + fe_feats + agg_feats
print('
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
X, y = train[['idhogar', 'Id'] + features], train['Target']
X_train, X_valid, y_train, y_valid = train_test_split(X[features], y,
test_size=0.1, random_state=0 )
n_steps = olid_train.shape[0] // BATCH_SIZE
model.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))
<load_from_csv>
gbm = lgb.LGBMClassifier(n_jobs=4, random_state=0, class_weight='balanced')
 param_grid = {'num_leaves': np.arange(10, 50, 10),
'learning_rate': np.arange(0.05, 0.2, 0.05),
'n_estimators': np.arange(10, 50, 5)}
scoring = {'f1_macro': get_scorer('f1_macro')}
metric = 'f1_macro'
data = {k.split('/')[-1][:-4]:k for k in glob.glob('/kaggle/input/**/**.csv')}
train = pd.read_csv(data['jigsaw-toxic-comment-train'], usecols=['id', 'comment_text', 'toxic'])
 val = pd.read_csv(data['validation'], usecols=['comment_text', 'toxic'])
 test = pd.read_csv(data['test'], usecols=['id', 'content'])
 test.columns = ['id', 'comment_text']
test['toxic'] = 0.5
sub2 = pd.read_csv('.. /input/submit/submission14.csv' )<feature_engineering>
gs = GridSearchCV(gbm,
param_grid=param_grid,
scoring=scoring,
cv=5,
refit=metric,
verbose=True,
n_jobs=4, 
)
 gs.fit(X_train, y_train)
 best_estimator = gs.best_estimator_
%%time
def f_experience(c, s):
it = {'memory':10,
'influence':0.5,
'inference':0.5,
'interest':0.9,
'sentiment':1e-10,
'harmony':0.5}
exp = {}
for i in range(len(c)) :
words = set([w for w in str(c[i] ).lower().split(' ')])
 for w in words:
try:
exp[w]['influence'] = exp[w]['influence'][1:] + [s[i]]
exp[w]['inference'] += 1
exp[w]['interest'] = exp[w]['interest'][1:] + [(exp[w]['interest'][it['memory']-1] +(s[i] * it['interest'])) /2]
exp[w]['sentiment'] += s[i]
except:
m = [0.for m_ in range(it['memory'])]
exp[w] = {}
exp[w]['influence'] = m[1:] + [s[i]]
exp[w]['inference'] = 1
exp[w]['interest'] = m[1:] + [s[i] * it['interest'] / 2]
exp[w]['sentiment'] = s[i]
for w in exp:
exp[w]['sentiment'] /= exp[w]['inference'] + it['sentiment']
exp[w]['inference'] /= len(c)* it['inference']
return exp
exp = f_experience(train['comment_text'].values, train['toxic'].values )<feature_engineering>
y_pred = best_estimator.predict(X_valid)
 macro_f1 = f1_score(y_valid, y_pred, average='macro')
 print(macro_f1 )
%%time
def features(df):
df['len'] = df['comment_text'].map(len)
 df['wlen'] = df['comment_text'].map(lambda x: len(str(x ).split(' ')))
 df['influence_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['influence_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['sentiment_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
df['sentiment_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
return df
val = features(val)
 test= features(test )<train_on_grid>
X_test = test[['Id', 'idhogar'] + features]
y_pred = best_estimator.predict(X_test[features])
 pred_for_heads = pd.DataFrame({'Id': X_test['Id'], 'idhogar': X_test['idhogar'],
'Target': y_pred})
 pred_for_heads['Target'] = pred_for_heads['Target'].apply(int )
col = [c for c in val if c not in ['id', 'comment_text', 'toxic']]
x1, x2, y1, y2 = model_selection.train_test_split(val[col], val['toxic'], test_size=0.3, random_state=20)
 model = ensemble.ExtraTreesClassifier(n_estimators=1000, max_depth=7, n_jobs=-1, random_state=20)
 model.fit(x1, y1)
 print(metrics.roc_auc_score(y2, model.predict_proba(x2)[:,1].clip(0.,1.)))
model.fit(val[col], val['toxic'])
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
sub1 = test[['id', 'toxic']]<save_to_csv>
test_all = data_all.loc[data_all['Target'].isnull() , ['Id', 'idhogar']]
submit = pd.merge(test_all, pred_for_heads[['idhogar', 'Target']], how='left' )
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub3 = pd.merge(sub1, sub2, how='left', on='id')
 sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic2'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.5)+(sub3['toxic'] * 0.5)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )<prepare_output>
sum(submit['Target'].isnull() )
test = pd.DataFrame(['Howling with Wolf on Lügenpresse'], columns=['comment_text'])
 test['id'] = test.index
test= features(test)
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
test[['id', 'comment_text', 'toxic']].head()<import_modules>
submit.fillna(1, inplace=True)
 submit['Target'] = submit['Target'].astype(np.int16)
 submit.head()
<load_from_csv><EOS> 
submit[['Id', 'Target']].to_csv('submisssion.csv', index=False )