Kaggle competition: 2019  ML competition with KISTI
<SOS> metric: accuracy Kaggle data source: 2019-ml-competition-with-kisti<prepare_x_and_y>
plt.style.use('seaborn')
 sns.set(font_scale=2.5)
 warnings.filterwarnings('ignore')
 %matplotlib inline
%%time
x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_valid=valid.toxic.values 
<create_dataframe>
train_data = pd.read_csv('.. /input/train.csv')
 train_data.head()
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<prepare_x_and_y>
test_data = pd.read_csv('.. /input/test.csv')
 test_data.head()
def get_fold_data(train_ind,valid_ind):
print("Getting fold data...")
 train_x=np.vstack([x_train,x_valid[train_ind]])
 train_y=np.hstack([y_train,y_valid[train_ind]])
 valid_x= x_valid[valid_ind]
valid_y =y_valid[valid_ind]
train_data =(
 tf.data.Dataset 
.from_tensor_slices(( train_x, train_y)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO))
valid_data=(
 tf.data.Dataset 
.from_tensor_slices(( valid_x, valid_y)) 
.shuffle(2048)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 return train_data,valid_data<compute_train_metric>
train_data.isnull().sum()
def focal_loss(gamma=1.5, alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed 
<split>
for col in train_data.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(train_data[col].isnull().sum() / train_data[col].shape[0]))
print(msg )
pred_test=np.zeros(( test.shape[0],1))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_valid,valid.lang.values)) :
if fold < 0:
print("fold",fold+1)
 K.clear_session()
tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data,valid_data=get_fold_data(train_ind,valid_ind)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base_{fold+1}.h5", monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps=(x_train.shape[0]+len(train_ind)) //BATCH_SIZE
print(n_steps)
 print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=1)
 print("Loading model...")
 model.load_weights(f"roberta_base_{fold+1}.h5")
 print("fold {} validation auc {}".format(fold+1,np.mean(train_history.history['val_auc'])))
 print("fold {} validation auc {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 val_score.append(np.mean(train_history.history['val_auc']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/3 
<save_to_csv>
for col in test_data.columns:
msg = 'column: {:>10}\t Percent of NaN value: {:.2f}%'.format(col, 100 *(test_data[col].isnull().sum() / test_data[col].shape[0]))
print(msg )
sub['toxic'] = pred_test
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
train_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=True ).count()
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV<load_from_csv>
train_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=True ).sum()
df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 id_train = df_train['PassengerId']
id_test = df_test['PassengerId']
df = pd.concat([df_train, df_test])
 df = df.set_index('PassengerId')
 df.info()
df<categorify>
train_data[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
def category_to_one_hot(df, name_column):
one_hot = pd.get_dummies(df[name_column], prefix=name_column, prefix_sep='-')
 df = pd.concat([df, one_hot], axis=1)
 df = df.drop(name_column, axis=1)
 return df, one_hot
def exclude_low_category(df, name_column, threshold_using_category):
df_value_counts = df[name_column].value_counts()
excluding_categories = df_value_counts.loc[df_value_counts < threshold_using_category].index
for c in excluding_categories:
df.loc[df[name_column] == c, name_column] = None
return df<categorify>
train_data['FamilySize'] = train_data['SibSp'] + train_data['Parch'] + 1
test_data['FamilySize'] = test_data['SibSp'] + test_data['Parch'] + 1
name_column = 'Pclass'
print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<count_values>
test_data.loc[test_data.Fare.isnull() , 'Fare'] = test_data['Fare'].mean()
train_data['Fare'] = train_data['Fare'].map(lambda i: np.log(i)if i > 0 else 0)
 test_data['Fare'] = test_data['Fare'].map(lambda i: np.log(i)if i > 0 else 0 )
name_column = 'Name'
df[name_column] = df[name_column].str.extract(', (.*?\.) ')
 print(df[name_column].value_counts(dropna=False))
df = exclude_low_category(df, name_column, 10)
 print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<count_values>
def get_one_hot(array):
return np.array(( array['Pclass'] == 1, array['Pclass'] == 2,
array['Pclass'] == 3, array['Sex'] == 'male',
array['Sex'] == 'female', array['SibSp'],
array['Parch'], array['Fare'],
array['Embarked'] == 'C', array['Embarked'] == 'Q',
array['Embarked'] == 'S')).swapaxes(0, 1 ).astype('float32' )
name_column = 'Sex'
print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<feature_engineering>
x_train = train_data[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']]
x_train.head()
name_column = 'Ticket'
df[name_column] = df[name_column].replace('(|^)[0-9]*', '', regex=True)
 df[name_column] = df[name_column].replace('(/|\.).*', '', regex=True)
 df.loc[df[name_column] == '', name_column] = None
print(df[name_column].value_counts(dropna=False))
df = exclude_low_category(df, name_column, 10)
 print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<categorify>
x_train = get_one_hot(x_train)
 x_train[:10]
name_column = 'Cabin'
df['Cabin'] = df['Cabin'].replace('.*', '', regex=True)
 df['Cabin'] = df['Cabin'].replace('[0-9]*', '', regex=True)
 print(df[name_column].value_counts(dropna=False))
df = exclude_low_category(df, name_column, 10)
 print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<categorify>
y_train = np.array(train_data['Survived'])
 y_train[:10]
name_column = 'Embarked'
print(df[name_column].value_counts(dropna=False))
df, one_hot = category_to_one_hot(df, name_column)
 one_hot<feature_engineering>
x_test = get_one_hot(x_test)
 x_test[:10]
def standardize_df(df, name_column):
x = np.array(df.loc[df[name_column].notnull() , name_column])
 standardized_x =(x - x.mean())/ x.std(ddof=1)
 df.loc[df[name_column].notnull() , name_column] = standardized_x
df.loc[df[name_column].isnull() , name_column] = 888
return df<drop_column>
train_data.drop(['PassengerId','Name','SibSp','Parch','Ticket','Cabin'], axis=1, inplace=True)
 test_data.drop(['PassengerId','Name','SibSp','Parch','Ticket','Cabin'], axis=1, inplace=True )
name_column = 'Age'
df = standardize_df(df, name_column)
 df[name_column]<drop_column>
from mlxtend.classifier import StackingCVClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.impute import SimpleImputer
name_column = 'SibSp'
df = standardize_df(df, name_column)
 df[name_column]<drop_column>
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
 imp = imp.fit(x_train)
 x_train_imp = imp.transform(x_train )
name_column = 'Parch'
df = standardize_df(df, name_column)
 df[name_column]<drop_column>
clf1 = RandomForestClassifier()
clf2 = GradientBoostingClassifier()
lr = LogisticRegression()
sclf = StackingCVClassifier(classifiers=[clf1, clf2], meta_classifier=lr )
name_column = 'Fare'
df = standardize_df(df, name_column)
 df[name_column]<prepare_x_and_y>
param_test = {'randomforestclassifier__n_estimators': [10, 120],
'randomforestclassifier__max_depth': [2, 15],
'gradientboostingclassifier__n_estimators': [10, 120],
'gradientboostingclassifier__max_depth': [2, 15],
'gradientboostingclassifier__learning_rate' : [0.01, 0.1],
'meta_classifier__C': [0.1, 10.0]}
name_class = 'Survived'
y_train = np.array(df_train[name_class])
 X_train = np.array(df_train.drop(name_class, axis=1))
X_test = np.array(df_test.drop(name_class, axis=1))<train_on_grid>
sclf.fit(x_train_imp, y_train )
param_grid = {
'max_depth': [5, 10, 15],
'min_samples_split': [10, 20, 30],
'n_estimators': [100, 200, 300],
'min_samples_leaf': [5, 10, 15],
'n_jobs': [4],
"bootstrap": [True],
"criterion": ["entropy"]
}
grid = GridSearchCV(estimator=RandomForestClassifier(random_state=0),
param_grid=param_grid,
scoring="accuracy",
cv=10)
 grid.fit(X_train, y_train)
 print(f"Best Score: {grid.best_score_}, Param: {grid.best_params_}" )<predict_on_test>
X_test_imp = imp.transform(x_test )
best_model = grid.best_estimator_
y_pred = best_model.predict(X_test)
 y_pred = np.array(y_pred, dtype=np.int )<save_to_csv>
submission = pd.read_csv('.. /input/sample_submission.csv' )
submission_df = pd.DataFrame({'PassengerId': id_test, 'Survived': y_pred})
 submission_df.to_csv('submission.csv', index=False )<import_modules>
submission = pd.DataFrame(data, columns=['PassengerId', 'Survived'])
 submission.set_index('PassengerId', inplace=True)
 submission.head()
import numpy as np
import pandas as pd<load_from_csv>
prediction = sclf.predict(x_test)
 submission['Survived'] = prediction
prediction
<import_modules><EOS> 
submission.to_csv('./submission.csv',index= False )