Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<load_from_csv>
!pip install -U vega_datasets notebook vega
%%time
file_folder = '.. /input/champs-scalar-coupling'
print("Load Giba's features...")
 train = pd.read_csv('.. /input/giba-molecular-features/train_giba.csv/train_giba.csv',
usecols=index_columns+giba_columns+['scalar_coupling_constant'])
 y = train['scalar_coupling_constant']
train = reduce_mem_usage(train,verbose=True)
 test = pd.read_csv('.. /input/giba-molecular-features/test_giba.csv/test_giba.csv',
usecols=index_columns+giba_columns)
 test = reduce_mem_usage(test,verbose=True)
 print("Load QM9 features...")
 data_qm9 = pd.read_pickle('.. /input/quantum-machine-9-qm9/data.covs.pickle')
 data_qm9 = data_qm9.drop(columns = ['type', 'linear', 'atom_index_0', 'atom_index_1',
'scalar_coupling_constant', 'U', 'G', 'H',
'mulliken_mean', 'r2', 'U0'], axis=1)
 data_qm9 = reduce_mem_usage(data_qm9,verbose=False)
 train = pd.merge(train, data_qm9, how='left', on=['molecule_name','id'])
 test = pd.merge(test, data_qm9, how='left', on=['molecule_name','id'])
 del data_qm9
gc.collect()
print("Encoding label features...
 ")
 for f in label_columns:
if f in train.columns:
lbl = LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
train = train[index_columns+giba_columns+qm9_columns]
test = test[index_columns+giba_columns+qm9_columns]<compute_train_metric>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
alt.renderers.enable('notebook')
 %env JOBLIB_TEMP_FOLDER=/tmp
coef = [0.25, 0.5, 0.2, 0.05]
def custom_objective(y_true, y_pred):
c = 0.5
residual = y_pred - y_true
grad = c * residual /(np.abs(residual)+ c)
 hess = c ** 2 /(np.abs(residual)+ c)** 2
h = 1.2
scale = 1 +(residual / h)** 2
scale_sqrt = np.sqrt(scale)
 grad_huber = residual / scale_sqrt
hess_huber = 1 / scale / scale_sqrt
grad_rmse = residual
hess_rmse = 1.0
grad_mae = np.array(residual)
 grad_mae[grad_mae > 0] = 1.
 grad_mae[grad_mae <= 0] = -1.
 hess_mae = 1.0
return coef[0] * grad + coef[1] * grad_huber + coef[2] * grad_rmse + coef[3] * grad_mae, \
coef[0] * hess + coef[1] * hess_huber + coef[2] * hess_rmse + coef[3] * hess_mae
params = {
'num_leaves': 400,
'objective': custom_objective,
'max_depth': 9,
'learning_rate': 0.1,
'boosting_type': 'gbdt',
'metric': 'mae',
'verbosity': -1,
'lambda_l1': 2,
'lambda_l2': 0.2,
'feature_fraction': 0.6,
}<create_dataframe>
def prepare_altair() :

 vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + v5.SCHEMA_VERSION
vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'
vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION
vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'
noext = "?noext"
paths = {
'vega': vega_url + noext,
'vega-lib': vega_lib_url + noext,
'vega-lite': vega_lite_url + noext,
'vega-embed': vega_embed_url + noext
}
workaround = f
return workaround
def add_autoincrement(render_func):
cache = {}
def wrapped(chart, id="vega-chart", autoincrement=True):
if autoincrement:
if id in cache:
counter = 1 + cache[id]
cache[id] = counter
else:
cache[id] = 0
actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])
 else:
if id not in cache:
cache[id] = 0
actual_id = id
return render_func(chart, id=actual_id)
 return wrapped
@add_autoincrement
def render(chart, id="vega-chart"):

 chart_str =
return HTML(
 chart_str.format(
 id=id,
chart=json.dumps(chart)if isinstance(chart, dict)else chart.to_json(indent=None)
)
)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
c_prec = df[col].apply(lambda x: np.finfo(x ).precision ).max()
if c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max and c_prec == np.finfo(np.float32 ).precision:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
@jit
def fast_auc(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(y_true, y_pred):

 return 'auc', fast_auc(y_true, y_pred), True
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
def train_model_regression(X, X_test, y, params, folds=None, model_type='lgb', eval_metric='mae', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3):

 columns = X.columns if columns is None else columns
X_test = X_test[columns]
splits = folds.split(X)if splits is None else splits
n_splits = folds.n_splits if splits is None else n_folds
metrics_dict = {'mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE',
'sklearn_scoring_function': metrics.mean_absolute_error},
'group_mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE',
'scoring_function': group_mean_log_mae},
'mse': {'lgb_metric_name': 'mse',
'catboost_metric_name': 'MSE',
'sklearn_scoring_function': metrics.mean_squared_error}
}
result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(splits):
if verbose:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = n_estimators, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=metrics_dict[eval_metric]['catboost_metric_name'])
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 if eval_metric != 'group_mae':
scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
else:
scores.append(metrics_dict[eval_metric]['scoring_function'](y_valid, y_pred_valid, X_valid['type']))
prediction += y_pred
if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
return result_dict
def train_model_classification(X, X_test, y, params, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3, averaging='usual', n_jobs=-1):

 columns = X.columns if columns is None else columns
n_splits = folds.n_splits if splits is None else n_folds
X_test = X_test[columns]
metrics_dict = {'auc': {'lgb_metric_name': eval_auc,
'catboost_metric_name': 'AUC',
'sklearn_scoring_function': metrics.roc_auc_score},
}
result_dict = {}
if averaging == 'usual':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
elif averaging == 'rank':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMClassifier(**params, n_estimators=n_estimators, n_jobs = n_jobs)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test, num_iteration=model.best_iteration_)[:, 1]
if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict_proba(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=n_estimators, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=Logloss)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if averaging == 'usual':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += y_pred.reshape(-1, 1)
 elif averaging == 'rank':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += pd.Series(y_pred ).rank().values.reshape(-1, 1)
 if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['top_columns'] = cols
return result_dict
workaround = prepare_altair()
HTML("".join((
"<script>",
workaround,
"</script>", 
)) )
X_short = pd.DataFrame({'ind': list(train.index),
'type': train['type'].values,
'oof': [0] * len(train),
'target': y.values})
 X_short_test = pd.DataFrame({'ind': list(test.index),
'type': test['type'].values,
'prediction': [0] * len(test)} )<prepare_x_and_y>
folder_path = '.. /input/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )
%%time
CV_score = 0
folds = GroupKFold(n_splits=3)
 n_estimators = [6000, 2500, 3500, 3000, 3000, 5000, 3000, 3000]
for t in train['type'].unique() :
type_ = lbl.inverse_transform([t])[0]
print(f'Training of type {t}: {type_}.
 ')
 X_t = train.loc[train['type'] == t]
X_test_t = test.loc[test['type'] == t]
y_t = X_short.loc[X_short['type'] == t, 'target']
scaler = StandardScaler()
X_t[diff_columns] = scaler.fit_transform(X_t[diff_columns].fillna(-999))
X_test_t[diff_columns] = scaler.transform(X_test_t[diff_columns].fillna(-999))
molecules_id = X_t.molecule_name
result_dict_lgb = train_lgb_regression_group(X=X_t.drop(columns=['molecule_name','id']),
X_test=X_test_t.drop(columns=['molecule_name','id']),
y=y_t, params=params,
folds=folds, groups=molecules_id,
eval_metric='group_mae',
plot_feature_importance=False,
verbose=3000, early_stopping_rounds=200,
n_estimators=n_estimators[t])
 X_short.loc[X_short['type'] == t, 'oof'] = result_dict_lgb['oof']
X_short_test.loc[X_short_test['type'] == t, 'prediction'] = result_dict_lgb['prediction']
CV_score += np.array(result_dict_lgb['scores'] ).mean() /8<save_to_csv>
del train_identity, train_transaction, test_identity, test_transaction
sub = pd.read_csv(f'{file_folder}/sample_submission.csv')
 sub['scalar_coupling_constant'] = X_short_test['prediction']
today = str(datetime.date.today())
 sub.to_csv(f'LGB_{today}_{CV_score:.4f}.csv', index=False )<save_to_csv>
print(f'There are {train.isnull().any().sum() } columns in train dataset with missing values.' )
sub1['scalar_coupling_constant'] = 0.6*sub1['scalar_coupling_constant'] + 0.4*sub2['scalar_coupling_constant']
sub1.to_csv('weighted-avg-blend-lgb-keras-1.csv', index=False)
 sub1['scalar_coupling_constant'].describe()<set_options>
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
one_value_cols_test = [col for col in test.columns if test[col].nunique() <= 1]
one_value_cols == one_value_cols_test
%matplotlib inline 
<define_variables>
print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.')
 print(f'There are {len(one_value_cols_test)} columns in test dataset with one unique value.' )
NUM_EPOCHS = 10
NUM_WORDS = 50000
MODEL_NAME = 'toxicity'
BEST_WEIGHTS =('{}_glove_best_weights.hdf5'.format(MODEL_NAME))
MODEL_FILE =('{}_glove_model.h5'.format(MODEL_NAME))<categorify>
train['id_03'].value_counts(dropna=False, normalize=True ).head()
def process_text(text):
text = text.encode('ascii', errors='ignore' ).decode()
text = re.sub(r'http\S+', ' ', text)
 text = re.sub(r'
text = re.sub(r'@[A-Za-z0-9]+', ' ', text)
 text = re.sub(r"([A-Za-z]+)'s", r"\1 is", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"won't", "will not ", text)
 text = re.sub(r"isn't", "is not ", text)
 text = re.sub(r"can't", "can not ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub('\W', ' ', text)
 text = re.sub(r'\d+', ' ', text)
 text = re.sub('\s+', ' ', text)
 text = text.strip()
return text<load_from_csv>
train['id_11'].value_counts(dropna=False, normalize=True ).head()
def load_dataset() :
df1 = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 print(df1.shape)
 df2 = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 df3 = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test_labels.csv')
 df4 = pd.merge(df2, df3, on='id')
 print(df4.shape)
 df = pd.concat([df1, df4], ignore_index=True)
 df['text'] = df['comment_text'].apply(lambda x: process_text(x))
tokenizer = Tokenizer(num_words = NUM_WORDS, split = ' ')
 tokenizer.fit_on_texts(df['text'].values)
 X = tokenizer.texts_to_sequences(df['text'].values)
 X = pad_sequences(X, maxlen=50)
 df['X'] = pd.Series(list(X))
df.loc[df.toxic == -1, 'toxic'] = 0
df.loc[df.severe_toxic == -1, 'severe_toxic'] = 0
df.loc[df.obscene == -1, 'obscene'] = 0
df.loc[df.threat == -1, 'threat'] = 0
df.loc[df.insult == -1, 'insult'] = 0
df.loc[df.identity_hate == -1, 'identity_hate'] = 0
df_train = df.iloc[: df1.shape[0]]
df_test= df.iloc[df1.shape[0] :]
df_train, df_val = train_test_split(df_train, test_size=0.02, random_state=1)
 return df_train, df_val, df_test, tokenizer
df_train, df_val, df_test, tokenizer = load_dataset()
X_train = np.asarray(df_train['X'].tolist())
 X_val = np.asarray(df_val['X'].tolist())
 X_test = np.asarray(df_test['X'].tolist())
 Y_train = np.asarray(df_train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values)
 Y_val = np.asarray(df_val[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values)
 Y_test = np.asarray(df_test[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values)
 print("Train shape: {} - {}".format(X_train.shape, Y_train.shape))
print("Val shape: {} - {}".format(X_val.shape, Y_val.shape))
print("Test shape: {} - {}".format(X_test.shape, Y_test.shape)) 
<feature_engineering>
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('std' )
def load_vectors(file, total_num):
word_vec = {}
with open(file, encoding='utf-8', mode = 'r')as f:
count = 0
for line in f:
values = line.strip().split()
curr_word = values[0]
try:
word_vec[curr_word] = np.array(values[1:], dtype=np.float64)
 count += 1
if count % 1000 == 0:
print('Processed {0} of {1}'.format(count, total_num), end='\r')
 {1!r}"
except ValueError:
pass
print('Processed {0} of {1}'.format(count, total_num))
return word_vec
word_vec = load_vectors('.. /input/glove840b300dtxt/glove.840B.300d.txt', 2195997 )<choose_model_class>
train[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = train['P_emaildomain'].str.split('.', expand=True)
 train[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = train['R_emaildomain'].str.split('.', expand=True)
 test[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = test['P_emaildomain'].str.split('.', expand=True)
 test[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = test['R_emaildomain'].str.split('.', expand=True )
def pretrained_embedding_layer(word_vec, word_index):
vocab_len = len(word_index)+ 1
emb_dim = word_vec["the"].shape[0]
embedding_layer = layers.Embedding(vocab_len, emb_dim, trainable=False)
 embedding_layer.build(( None,))
emb_matrix = np.zeros(( vocab_len, emb_dim))
for word, index in word_index.items() :
vec = word_vec.get(word)
 if vec is not None:
emb_matrix[index, :] = vec
embedding_layer.set_weights([emb_matrix])
 return embedding_layer
def create_model(input_shape, out_shape, activation, word_vec, word_index):
inputs = layers.Input(input_shape, dtype="int32")
 embedding_layer = pretrained_embedding_layer(word_vec, word_index)
 X = embedding_layer(inputs)
 X = layers.SpatialDropout1D(0.3 )(X)
 X = layers.Bidirectional(layers.LSTM(256, return_sequences=True, dropout=0.2, recurrent_dropout=0.2), merge_mode='concat' )(X)
 X = layers.Dropout(0.3 )(X)
 X = layers.Bidirectional(layers.LSTM(256, return_sequences=False, dropout=0.2, recurrent_dropout=0.2), merge_mode='concat' )(X)
 X = layers.Dropout(0.3 )(X)
 X = layers.Dense(512, activation='relu' )(X)
 X = layers.Dropout(0.3 )(X)
 outputs = layers.Dense(out_shape, activation=activation )(X)
 model = Model(inputs=inputs, outputs=outputs)
 return model
model = create_model(( X_train.shape[1],), 6, 'sigmoid', word_vec, tokenizer.word_index)
 model.summary()<choose_model_class>
many_null_cols = [col for col in train.columns if train[col].isnull().sum() / train.shape[0] > 0.9]
many_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.9]
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6, beta_1=0.9, beta_2=0.999), metrics=['accuracy'] )<compute_train_metric>
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
big_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
class ROC_AUC(Callback):
def __init__(self, X, Y, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X, self.Y = X, Y
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
Y_pred = self.model.predict(self.X, verbose=0)
 score = roc_auc_score(self.Y, Y_pred)
 print("ROC AUC score: {:.6f}".format(score))<train_model>
cols_to_drop = list(set(many_null_cols + many_null_cols_test + big_top_value_cols + big_top_value_cols_test + one_value_cols+ one_value_cols_test))
cols_to_drop.remove('isFraud')
 len(cols_to_drop )
monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=10, verbose=1, mode='auto')
 checkpoint = ModelCheckpoint(filepath=BEST_WEIGHTS, verbose=1, save_best_only=True)
 tensorboard = TensorBoard(log_dir='logs/{} - {}'.format(MODEL_NAME, time()))
roc_auc = ROC_AUC(X_val, Y_val)
 hist = model.fit(X_train,
Y_train,
batch_size = 32,
epochs = NUM_EPOCHS,
verbose=1,
callbacks=[monitor, checkpoint, tensorboard, roc_auc],
validation_data=(X_val, Y_val),
validation_split=0.02,
shuffle=True )<load_pretrained>
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
model.load_weights(BEST_WEIGHTS)
 model.save(MODEL_FILE)
<compute_test_metric>
cat_cols = ['id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',
'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',
'R_emaildomain', 'card1', 'card2', 'card3', 'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9',
'P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3', 'R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']
for col in cat_cols:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
score = model.evaluate(X_test, Y_test)
 print("Test Loss = " + str(score[0]))
print("Test Accuracy = " + str(score[1]))<predict_on_test>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT', 'TransactionID'], axis=1)
 del train
test = test[["TransactionDT", 'TransactionID']]
Y_test_pred = model.predict(X_test, batch_size=32, verbose=1 )<compute_test_metric>
def clean_inf_nan(df):
return df.replace([np.inf, -np.inf], np.nan)
 X = clean_inf_nan(X)
 X_test = clean_inf_nan(X_test )
def analyze(Y, Y_pred, classes, activation="softmax", threshold=None):
if activation == "sigmoid":
Y_cls = Y
Y_pred_cls =(Y_pred > threshold ).astype(float)
 elif activation == "softmax":
Y_cls = np.argmax(Y, axis=1)
 Y_pred_cls = np.argmax(Y_pred, axis=1)
 accuracy = accuracy_score(Y_cls, Y_pred_cls)
 print("Accuracy Score: {}
".format(accuracy))
rmse = np.sqrt(mean_squared_error(Y, Y_pred))
print("RMSE Score: {}
".format(rmse))
print("Confusion Matrix:")
 cm = confusion_matrix(Y_cls, Y_pred_cls)
 print(cm)
 plt.matshow(cm)
 num_classes = len(classes)
 plt.colorbar()
tick_marks = np.arange(num_classes)
 plt.xticks(tick_marks, range(num_classes))
plt.yticks(tick_marks, range(num_classes))
plt.xlabel('Predicted')
 plt.ylabel('True')
 print("Classification Report:")
 print(classification_report(Y_cls, Y_pred_cls, target_names=classes))
analyze(Y_test[:, 0], Y_test_pred[:, 0], ['True', 'False'], "sigmoid", threshold0 )<save_to_csv>
gc.collect()
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = Y_test_pred
submission.to_csv('submission.csv', index=False )<import_modules>
n_fold = 5
folds = TimeSeriesSplit(n_splits=n_fold)
 folds = KFold(n_splits=5 )
import numpy as np
import pandas as pd
from pathlib import Path
from typing import *
import torch
import torch.optim as optim<import_modules>
params = {'num_leaves': 256,
'min_child_samples': 79,
'objective': 'binary',
'max_depth': 13,
'learning_rate': 0.03,
"boosting_type": "gbdt",
"subsample_freq": 3,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3,
'reg_lambda': 0.3,
'colsample_bytree': 0.9,
}
result_dict_lgb = train_model_classification(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb', eval_metric='auc', plot_feature_importance=True,
verbose=500, early_stopping_rounds=200, n_estimators=5000, averaging='usual', n_jobs=-1 )
from fastai import *
from fastai.vision import *
from fastai.text import *
from fastai.callbacks import *<install_modules>
sub['isFraud'] = result_dict_lgb['prediction']
sub.to_csv('submission.csv', index=False )
%%bash
pip install pytorch-pretrained-bert<feature_engineering>
pd.DataFrame(result_dict_lgb['oof'] ).to_csv('lgb_oof.csv', index=False )