Kaggle competition: Titanic - Machine Learning from Disaster
dataset_train = lgb.Dataset(X_train_tfidf, y_train)
 dataset_valid = lgb.Dataset(X_valid_tfidf, y_valid )<train_model>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 combine = [train, test]
booster = lgb.train(
 params,
dataset_train,
feature_name=(
 [f"feat_{i}" for i in range(1, 94)]
+ [f"tfidf_{i}" for i in range(1, 94)] 
),
num_boost_round=500,
valid_sets=dataset_valid,
early_stopping_rounds=20,
)<find_best_params>
train["Title"] = train["Name"].str.extract('([A-Za-z]+)\.', expand=False)
 test["Title"] = test["Name"].str.extract('([A-Za-z]+)\.', expand=False )
best_iteration = booster.best_iteration
print(best_iteration )<load_from_csv>
train['Title'] = train['Title'].replace([ 'Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona', 'Countess', 'Lady', 'Sir'], 'Rare')
 train['Title'] = train['Title'].replace('Mlle', 'Miss')
 train['Title'] = train['Title'].replace('Ms', 'Miss')
 train['Title'] = train['Title'].replace('Mme', 'Mrs' )
df_test = pd.read_csv(
 "/kaggle/input/otto-group-product-classification-challenge/test.csv",
dtype=dtypes
).set_index("id" )<data_type_conversions>
test['Title'] = test['Title'].replace([ 'Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona', 'Countess', 'Lady', 'Sir'], 'Rare')
 test['Title'] = test['Title'].replace('Mlle', 'Miss')
 test['Title'] = test['Title'].replace('Ms', 'Miss')
 test['Title'] = test['Title'].replace('Mme', 'Mrs' )
tfidf = TfidfTransformer()
tfidf_feature_train_all = tfidf.fit_transform(df_train[feature_columns] ).toarray().astype("float32")
 X_train_all_tfidf = np.hstack(( df_train[feature_columns].values, tfidf_feature_train_all))
dataset_train_all = lgb.Dataset(X_train_all_tfidf, df_train[target_column] )<train_model>
train[["Title","Age"]].groupby("Title" ).mean()
booster = lgb.train(
 params,
dataset_train_all,
feature_name=(
 [f"feat_{i}" for i in range(1, 94)]
+ [f"tfidf_{i}" for i in range(1, 94)] 
),
num_boost_round=best_iteration,
)<concatenate>
for i in train["Title"]:
if i=="Master":
train["Age"]=train["Age"].fillna(4)
 elif i=="Miss":
train["Age"]=train["Age"].fillna(22)
 elif i=="Mr":
train["Age"]=train["Age"].fillna(32)
 elif i=="Mrs":
train["Age"]= train["Age"].fillna(36)
 elif i=="Major":
train["Age"]= train["Age"].fillna(46)
 else:
train["Age"]=train["Age"].fillna(41 )
tfidf_feature_test = tfidf.transform(df_test ).toarray()
X_test_tfidf = np.hstack(( df_test[feature_columns].values, tfidf_feature_test))<predict_on_test>
train.isnull().sum()
pred = booster.predict(X_test_tfidf )<prepare_output>
train.isnull().sum()
for idx, col in order_to_class.items() :
df_test[col] = pred[:,idx]<save_to_csv>
test[["Title","Age"]].groupby("Title" ).mean()
df_test[[f"Class_{i}" for i in range(1, 10)]].to_csv('submission.csv', index=True )<define_variables>
for i in train["Title"]:
if i=="Master":
test["Age"]=test["Age"].fillna(7)
 elif i=="Miss":
test["Age"]=test["Age"].fillna(22)
 elif i=="Mr":
test["Age"]=test["Age"].fillna(32)
 elif i=="Mrs":
test["Age"]= test["Age"].fillna(38)
 elif i=="Major":
test["Age"]= test["Age"].fillna(44)
 else:
test["Age"]=test["Age"].fillna(41 )
INPUT_PATH = Path(".. /input/otto-group-product-classification-challenge/")
 STACKING_LEVEL0_INPUT_PATH = Path(".. /input/otto-stacking-level0/")
 XGB_PATH = Path(".. /input/otto-simple-xgb/")
 LGB_PATH = Path(".. /input/otto-simple-lgb/")
 NN_PATH = Path(".. /input/otto-neuralnetwork/" )<load_from_csv>
test.isnull().sum()
train = pd.read_csv(INPUT_PATH / "train.csv")
 test = pd.read_csv(INPUT_PATH / "test.csv")
 train.shape, test.shape<data_type_conversions>
test.isnull().sum()
drop_cols = ["id"]
target_col = "target"
target = train[target_col]
feat_cols = [col for col in train.columns if col not in drop_cols + [target_col]]
train[target_col] = train[target_col].str.replace('Class_', '')
 train[target_col] = train[target_col].astype(int)- 1
target = train[target_col]
train.drop(columns=drop_cols + [target_col], inplace=True)
 test.drop(columns=drop_cols, inplace=True )<load_pretrained>
test[["Pclass","Fare"]].groupby("Pclass" ).mean()
keras_oof = np.load(NN_PATH / "keras_oof.npy")
 lgb_oof = np.load(LGB_PATH / "lgb_oof.npy")
 xgb_oof = np.load(XGB_PATH / "xgb_oof.npy")
 keras_oof.shape, lgb_oof.shape, xgb_oof.shape<load_from_csv>
test["Fare"] = test["Fare"].fillna(12 )
keras_test = pd.read_csv(NN_PATH / "submit.csv" ).drop(columns=["id"])
 lgb_test = pd.read_csv(LGB_PATH / "submit.csv" ).drop(columns=["id"])
 xgb_test = pd.read_csv(XGB_PATH / "submit_xgboost.csv" ).drop(columns=["id"])
 keras_test.shape, lgb_test.shape, xgb_test.shape<concatenate>
test.isnull().sum()
train = np.concatenate([keras_oof, lgb_oof, xgb_oof], axis=1)
 test = np.concatenate([keras_test, lgb_test, xgb_test], axis=1)
 train.shape, test.shape<init_hyperparams>
test.isnull().sum()
NFOLDS = 5
RANDOM_STATE = 871972
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE)
 params = {
'metric':'multi_logloss',
'objective': 'multiclass',
'num_class': 9,
'verbosity': 1,
}<train_model>
train.isnull().sum()
y_pred_test = np.zeros(( len(test), 9))
oof = np.zeros(( len(train), 9))
score = 0
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y=target)) :
print('Fold', fold_n)
 X_train, X_valid = train[train_index], train[valid_index]
y_train, y_valid = target.loc[train_index].astype(int), target.loc[valid_index].astype(int)
 train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_boost_round=30000,
valid_sets=[train_data, valid_data],
verbose_eval=300,early_stopping_rounds=300)
 y_pred_valid = lgb_model.predict(X_valid,
num_iteration=lgb_model.best_iteration)
 oof[valid_index] = y_pred_valid
score += log_loss(y_valid, y_pred_valid)
 y_pred_test += lgb_model.predict(test, num_iteration=lgb_model.best_iteration)/NFOLDS
print('valid logloss average:', score/NFOLDS, log_loss(target, oof))<save_to_csv>
train.isnull().sum()
sample_submit = pd.read_csv(INPUT_PATH / "sampleSubmission.csv")
 submit = pd.concat([sample_submit[['id']], pd.DataFrame(y_pred_test)], axis=1)
 submit.columns = sample_submit.columns
submit.to_csv('submit.csv', index=False )<import_modules>
train["Embarked"].value_counts()
from typing import Any, Dict
import umap
import numpy as np
from scipy.sparse.csgraph import connected_components
from sklearn.decomposition import PCA
import os
import pandas as pd
import matplotlib.pyplot as plt
import warnings
import tensorflow as tf
from sklearn.decomposition import PCA
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import Reshape, LayerNormalization, PReLU
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import AveragePooling1D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.constraints import max_norm
from tensorflow.keras import regularizers
import os
import lightgbm as lgb
import matplotlib.pyplot as plt
from datetime import datetime
from sklearn.model_selection import KFold
import xgboost as xgb<load_from_csv>
train["Embarked"]=train["Embarked"].fillna("S" )
train = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/train.csv")
 test = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/test.csv")
 ss = pd.read_csv("/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv" )<create_dataframe>
from sklearn import preprocessing
def docking(train: pd.DataFrame, test: pd.DataFrame)-> [pd.DataFrame, pd.Series]:
print("docking")
 dict_a: Dict
dict_a = {"Class_1": 0,
"Class_2": 1,
"Class_3": 2,
"Class_4": 3,
"Class_5": 4,
"Class_6": 5,
"Class_7": 6,
"Class_8": 7,
"Class_9": 8,
}
target = train["target"].map(dict_a)
 con_df = pd.concat([train, test], sort=False)

con_df = con_df.drop(["id", "target"], axis=1)
 con_df.columns = con_df.columns.map(lambda x: int(x[5:]))
con_df = np.log1p(con_df)
 return con_df, target
def split_data(df, df_pca, df_features, target, join_pca: bool):
if join_pca:
df = pd.concat([df.reset_index(drop=True), df_pca.reset_index(drop=True)], axis=1)
 df = pd.concat([df.reset_index(drop=True), df_features.reset_index(drop=True)], axis=1)
 train_df = df[:len(target)]
test_df = df[len(target):]
return train_df, test_df
def make_features(df: pd.DataFrame):
memo = pd.DataFrame()
memo["count_zero"] = df[df == 0].count(axis=1)
 return memo 
<create_dataframe>
def do_pca(df: pd.DataFrame, target: pd.Series):
n = 10
pca = PCA(n_components=n)
 pca.fit(df[:len(target)])
 df_pca = pca.transform(df)
 n_name = [f"pca{i}" for i in range(n)]
df_pca = pd.DataFrame(df_pca, columns=n_name)
 return df_pca<split>
dumytitle=pd.get_dummies(train['Title'] )
df, target = docking(train, test)
 df_pca = do_pca(df, target)
 df_features = make_features(df)
 df_train, df_test = split_data(df, df_pca, df_features, target, join_pca=True )<train_model>
train=pd.concat([train,dumytitle], axis=1)
 train.head()
def nn_train_model(
 df: pd.DataFrame, target: pd.DataFrame, test: pd.DataFrame 
):
n_splits = 5
num_class = 9
epochs = 10
lr_init = 0.01
bs = 256
num_features = df.shape[1]
folds = KFold(n_splits=n_splits, random_state=71, shuffle=True)
 model = tf.keras.models.Sequential([
Input(shape=(num_features,)) ,
Dense(128, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(128, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(64, kernel_initializer='glorot_uniform', activation="relu"),
BatchNormalization() ,
Dropout(0.25),
Dense(num_class, activation="softmax")
 ])
 optimizer = tf.keras.optimizers.Adam(lr=lr_init, decay=0.0001)

callbacks = []
model.compile(optimizer=optimizer, loss=tf.keras.losses.SparseCategoricalCrossentropy() , metrics=['accuracy'])
 preds = np.zeros(( test.shape[0], num_class))
for trn_idx, val_idx in folds.split(df, target):
train_x = df.iloc[trn_idx, :].values
val_x = df.iloc[val_idx, :].values
train_y = target[trn_idx].values
val_y = target[val_idx].values
model.fit(train_x, train_y, validation_data=(val_x, val_y), epochs=epochs, verbose=2, batch_size=bs,
callbacks=callbacks)
 preds += model.predict(test.values)/ n_splits
return preds<train_model>
dumytitle2=pd.get_dummies(test['Title'] )
pred1 = nn_train_model(df_train, target, df_test )<save_to_csv>
test=pd.concat([test,dumytitle2], axis=1)
 test.head(1 )
def make_submit_file(pred: np.ndarray, ss: pd.DataFrame)-> None:
save_path = "submission.csv"
ss.iloc[:, 1:] = pred
ss.to_csv(save_path, index=None )<install_modules>
train_name=train["Name"]
for i in train['Name']:
train['Name']= train['Name'].replace(i,len(i))
!pip install uc==2.2.0<load_from_csv>
test_name=test["Name"]
for i in test['Name']:
test['Name']= test['Name'].replace(i,len(i))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sample = pd.read_csv('.. /input/sampleSubmission.csv')
 dict_mapping = {"Class_1":1,"Class_2":2,"Class_3":3,"Class_4":4,"Class_5":5,"Class_6":6,"Class_7":7,"Class_8":8,"Class_9":9}
labels = train[["target"]]['target'].apply(lambda x: dict_mapping[x]-1)
 train = train.drop(["id", "target"], axis=1)
 test = test.drop("id", axis=1)
 feature_names = list(train.columns[0:])
 train_x = np.array(train, np.float64)
 train_y = np.array(labels, np.float64)
 test_x = np.array(test, np.float64)
 feature_range=(-1, 1)
 scaler = preprocessing.MinMaxScaler(feature_range=feature_range, copy=False ).fit(train_x)
 scaler.transform(train_x)
 scaler.transform(test_x)
 np.clip(test_x, out=test_x, a_min=feature_range[0], a_max=feature_range[1])
 print("train_x", train_x.shape, train_x.min() , train_x.max())
 print("train_y", train_y.shape, train_y.min() , train_y.max())
 print("test_x", test_x.shape, test_x.min() , test_x.max())
 epoch_decay = 4
epoch_train = epoch_decay * 16
mlp = MLP(
 layer_size = [train_x.shape[1], 100, 100, 100, 100, 100, 100, 9],
activation = 'am2',
op='fc',
rate_init = 0.05,
rate_decay = 0.8,
bias_rate = [],
regularization = 0,
loss_type = "softmax",
verbose=1,
importance_out=True,
importance_mul = 0.0001,
epoch_train = epoch_train,
epoch_decay = epoch_decay,
exf = [
{'Tins': [0], 'Touts': [1]},
{'Tins': [1], 'Touts': [2]},
{'Tins': [0], 'Touts': [3]},
{'Tins': [3], 'Touts': [4]},
{'Tins': [4], 'Touts': [5], 'Op': 'sigmoid'},
{'Tins': [2, 5], 'Touts': [6], 'Op': 'mul'},
{'Tins': [6], 'Touts': [7]},
] 
)
 mlp.fit(train_x, train_y)
 plot_importance(mlp.feature_importances_, 20, feature_names=feature_names)
 pred = mlp.predict_proba(test_x)
 submission = pd.DataFrame(pred, index=sample.id.values, columns=sample.columns[1:])
 submission.to_csv('submission.csv', index_label='id' )<set_options>
bins = [0,25,40, np.inf]
mylabels = ['s_name', 'm_name', 'l_name',]
train["Name_len"] = pd.cut(train["Name"], bins, labels = mylabels)
 test["Name_len"] = pd.cut(test["Name"], bins, labels = mylabels )
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
train["Name_len"].value_counts()
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str})
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str})
 train.shape, test.shape<prepare_x_and_y>
Name_mapping = {'s_name': 1, 'm_name': 2 , 'l_name': 3}
train['Name_len'] = train['Name_len'].map(Name_mapping)
 test['Name_len'] = test['Name_len'].map(Name_mapping )
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<data_type_conversions>
%matplotlib inline
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
train['target'] = y_reg
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day
df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
y_reg = train['target']
del train['target']<define_variables>
train[["AgeGroup","Survived"]].groupby("AgeGroup" ).mean()
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
age_mapping = {'Baby': 1, 'Child': 2, 'Teenager': 3, 'Student': 4, 'Young Adult':5 , 'Adult': 6, 'Senior':7}
train['AgeGroup'] = train['AgeGroup'].map(age_mapping)
 test['AgeGroup'] = test['AgeGroup'].map(age_mapping )
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<prepare_x_and_y>
train["FamilySize"] =train["SibSp"]+train["Parch"]+1
train["FamilySize"].mean()
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
test["FamilySize"] =test["SibSp"]+test["Parch"]+1
test["FamilySize"].mean()
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
train['Single'] = train['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 train['SmallFam'] = train['FamilySize'].map(lambda s: 1 if s == 2 else 0)
 train['MedFam'] = train['FamilySize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 train['LargeFam'] = train['FamilySize'].map(lambda s: 1 if s >= 5 else 0 )
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
test['Single'] = test['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 test['SmallFam'] = test['FamilySize'].map(lambda s: 1 if s == 2 else 0)
 test['MedFam'] = test['FamilySize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 test['LargeFam'] = test['FamilySize'].map(lambda s: 1 if s >= 5 else 0 )
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
train = pd.get_dummies(train, columns = ["Embarked"], prefix="Em" )
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
test = pd.get_dummies(test, columns = ["Embarked"], prefix="Em" )
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
train["Pclass"] = train["Pclass"].astype("category")
 train = pd.get_dummies(train, columns = ["Pclass"],prefix="Pc" )
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
test["Pclass"] = test["Pclass"].astype("category")
 test = pd.get_dummies(test, columns = ["Pclass"],prefix="Pc" )
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<normalization>
train['Ticket1']=train['Ticket'].str[:1]
df = pd.concat([full_data,sub_full_data],sort=False)
 df = df.reset_index(drop=False)
 for c in df.columns[1:]:
if(( df[c].min() >=0)&(df[c].max() >=10)) :
df[c] = np.log1p(df[c])
 elif(( df[c].min() <0)&(( df[c].max() -df[c].min())>=10)) :
df.loc[df[c]!=0,c] = np.sign(df.loc[df[c]!=0,c])*np.log(np.abs(df.loc[df[c]!=0,c]))
for c in df.columns[1:]:
ss = StandardScaler()
df.loc[~np.isfinite(df[c]),c] = np.nan
df.loc[~df[c].isnull() ,c] = ss.fit_transform(df.loc[~df[c].isnull() ,c].values.reshape(-1,1))
df.fillna(-99999,inplace=True)
 gp_trn_users = df[:full_data.shape[0]].copy().set_index('fullVisitorId')
 gp_trn_users['target'] = np.log1p(trn_user_target['target'].values)
 gp_sub_users = df[full_data.shape[0]:].copy().set_index('fullVisitorId')
 newcols = [x.replace('.','_')for x in gp_trn_users.columns]
gp_trn_users.columns = newcols
newcols = [x.replace('.','_')for x in gp_sub_users.columns]
gp_sub_users.columns = newcols<feature_engineering>
test['Ticket1']=test['Ticket'].str[:1]
def GP1(data):
return(12.500000 +
0.100000*np.tanh(((((((((((-3.0)+(data["t_sum_log"])))-(((( 12.95331287384033203)) / 2.0)))) *(( 4.50129508972167969)))) +(data["t_sum_act"])))* 2.0)) +
0.100000*np.tanh(((((((((((((data["t_sum_log"])-(( 9.0)))) * 2.0)) +(data["channelGrouping"])))* 2.0)) +(2.0)))* 2.0)) +
0.100000*np.tanh(((( 10.0)) *(((data["t_sum_log"])-(((((10.0)) +(((( 4.60818958282470703)) +(((data["t_sum_log"])-(data["t_sum_act"])))))) /2.0)))))) +
0.100000*np.tanh(((((((((((data["t_sum_act"])-(( 8.0)))) * 2.0)) -(((3.0)* 2.0)))) +(data["t_sum_log"])))* 2.0)) +
0.100000*np.tanh(((((((((((((((( data["t_sum_act"])+(2.0)) /2.0)) +(data["t_sum_log"])))-(( 12.46262645721435547)))) * 2.0)) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((( 10.0)) *(((((data["t_sum_log"])+(((((data["t_mean"])-(3.0)))-(( 12.30329895019531250)))))) * 2.0)))) * 2.0)) +
0.100000*np.tanh(((( 11.12195396423339844)) *(((((data["t_sum_log"])-(( 11.12195396423339844)))) +(((data["t_mean"])/ 2.0)))))) +
0.100000*np.tanh(((((data["pred_2"])+(data["pred_1"])))-(((((data["pred_3"])-(( 7.0)))) *(((data["t_mean"])-(( 7.0)))))))) +
0.100000*np.tanh(((((( 8.0)) -(data["next_session_2"])))+(((((data["t_sum_log"])-(np.maximum(((( 8.0))),(( data["next_session_2"])))))) *(( 8.41708469390869141)))))) +
0.100000*np.tanh(((((data["pred_3"])+(((((data["pred_3"])+(((((data["pred_2"])-(( 4.0)))) +(data["pred_4"])))))* 2.0)))) * 2.0)) +
0.100000*np.tanh(((((((((np.where(((data["t_sum_log"])-(( 8.0)))>0, data["t_sum_log"], data["t_sum_act"])) -(( 8.0)))) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["pred_3"])+(((((((data["next_session_1"])+(data["pred_0"])) /2.0)) +(data["pred_1"])) /2.0)))) -(( 5.55097246170043945)))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((( 14.49165344238281250)) *(((data["t_sum_act"])-(( 7.80256223678588867)))))) +(np.where(data["pred_5"] < -99998, data["t_sum_log"],(( data["t_sum_log"])* 2.0)))))+
0.100000*np.tanh(((((((data["t_sum_log"])-(( 6.95692586898803711)))) *(data["t_sum_act"])))+(((( 8.0)) *(((data["t_sum_act"])-(( 6.95692586898803711)))))))) +
0.100000*np.tanh(((((((((((((((data["t_sum_act"])-(( 5.79598665237426758)))) * 2.0)) +(-3.0)))* 2.0)) * 2.0)) * 2.0)) +(( 5.79598665237426758)))) +
0.100000*np.tanh(((data["pred_2"])+(((data["pred_4"])+(((data["pred_2"])+(((((data["t_median"])+(((-3.0)* 2.0)))) * 2.0)))))))) +
0.100000*np.tanh(((((((((((((data["t_sum_log"])-(( 7.0)))) -(((( data["t_sum_log"])>(data["t_sum_act"])) *1.))))* 2.0)) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((data["t_sum_log"])+(((((data["t_sum_log"])+(((((data["t_sum_act"])-(( 11.79764175415039062)))) * 2.0)))) * 2.0)))) *(( 11.79764175415039062)))) +
0.100000*np.tanh(((((((((((data["t_sum_log"])-(( 8.0)))) * 2.0)) * 2.0)) +(((( 6.84783124923706055)) -(data["visitNumber"])))))*(( 8.0)))) +
0.100000*np.tanh(((((( 7.0)) *(((data["t_sum_log"])+(((data["t_sum_act"])-(( 14.84496498107910156)))))))) +(-3.0)))+
0.100000*np.tanh(((( 7.0)) *(((((((data["t_sum_act"])-(( 7.0)))) * 2.0)) +(np.tanh(((( data["totals_pageviews"])+(data["pred_4"])))))))))+
0.100000*np.tanh(((((((((((data["t_sum_log"])-(( 8.0)))) * 2.0)) * 2.0)) +(((data["t_mean"])-(data["visitNumber"])))))* 2.0)) +
0.100000*np.tanh(((((((((np.minimum(((data["pred_0"])) ,(( data["pred_4"])))) +(((data["pred_3"])-(3.0)))))* 2.0)) * 2.0)) +(data["pred_3"])))+
0.100000*np.tanh(((((( 9.0)) *(((((((data["t_sum_log"])-(( 9.88647079467773438)))) -(-3.0)))* 2.0)))) * 2.0)) +
0.100000*np.tanh(((((data["t_sum_act"])+(((((((data["t_sum_act"])+(((((data["t_sum_act"])-(( 10.89448451995849609)))) * 2.0)))) * 2.0)) * 2.0)))) * 2.0)) +
0.100000*np.tanh(((((((((((data["t_sum_log"])-(( 9.0)))) * 2.0)) * 2.0)) +(( 2.0)))) +(((( 9.0)) -(data["visitNumber"])))))+
0.100000*np.tanh(((((((data["t_mean"])-(((( 9.0)) -(((data["t_sum_log"])-(( 7.0)))))))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((((( 7.24431943893432617)) *(((data["t_sum_log"])-(( 7.24431943893432617)))))) -(( 7.24431943893432617)))) +(-2.0)))* 2.0)) * 2.0)) +
0.100000*np.tanh(((((( 14.16644096374511719)) *(((data["t_sum_act"])-(((( 14.16644096374511719)) -(np.maximum(((( 8.0))),(( data["t_sum_log"])))))))))) -(( 8.0)))) +
0.100000*np.tanh(((((((((((data["t_mean"])+(((((data["t_sum_log"])-(((( 5.0)) * 2.0)))) * 2.0)))) * 2.0)) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((data["pred_4"])+(((( 14.16003227233886719)) *(((( 14.16003227233886719)) *(((np.maximum(((data["t_median"])) ,(( data["t_mean"])))) -(( 6.0)))))))))) +
0.100000*np.tanh(((data["pred_1"])+(((((( 8.92317485809326172)) *(((np.maximum(((data["t_sum_log"])) ,(( data["pred_1"])))) -(( 8.92317485809326172)))))) * 2.0)))) +
0.100000*np.tanh(((((( 10.0)) * 2.0)) *(((data["t_sum_log"])-(((((( 10.0)) -(data["t_mean"])))* 2.0)))))) +
0.100000*np.tanh(((((((((data["t_sum_log"])-(( 8.0)))) *(((( 9.0)) +(( 12.95555210113525391)))))) -(data["t_sum_log"])))*(( 9.0)))) +
0.100000*np.tanh(((((((data["pred_3"])+(((((data["t_sum_log"])-(( 8.74195766448974609)))) *(((data["visitNumber"])* 2.0)))))) * 2.0)) *(( 8.74195766448974609)))) +
0.100000*np.tanh(((((( 10.68931770324707031)) *(((((((((data["t_sum_log"])-(( 10.0)))) * 2.0)) +(data["t_mean"])))* 2.0)))) +(data["t_sum_log"])))+
0.100000*np.tanh(((((data["t_sum_log"])+(((((((data["t_sum_log"])-(np.maximum(((( 9.0))),(( data["visitNumber"])))))) * 2.0)) * 2.0)))) * 2.0)) +
0.100000*np.tanh(((( 10.33415031433105469)) *(((( 14.93418216705322266)) *(((((((data["t_sum_act"])-(( 9.26103305816650391)))) * 2.0)) +(data["t_sum_log"])))))))+
0.100000*np.tanh(((((np.maximum(((data["t_sum_log"])) ,(((( data["totals_hits"])+(data["pred_1"])))))) -(np.maximum(((( 9.0))),(( data["visitNumber"])))))) *(( 9.0)))) +
0.100000*np.tanh(((((((( data["pred_0"])+(data["t_sum_log"])) /2.0)) -(((( 14.61157703399658203)) -(data["t_sum_log"])))))+(data["pred_1"])))+
0.100000*np.tanh(((((((( 8.0)) -(data["t_mean"])))*(-3.0)))-(((( 7.0)) -(np.maximum(((data["pred_0"])) ,(( data["pred_3"])))))))) +
0.100000*np.tanh(((((( 12.26031208038330078)) *(((data["t_mean"])+(((data["t_sum_log"])-(( 12.26031208038330078)))))))) * 2.0)) +
0.100000*np.tanh(((( 9.30737781524658203)) *(((((data["t_sum_log"])-(( 9.30737781524658203)))) +(((((data["t_sum_act"])* 2.0)) -(( 9.30737781524658203)))))))) +
0.100000*np.tanh(((((((( 9.85758876800537109)) *(((data["t_sum_log"])-(( 8.81438827514648438)))))) -(( 8.0)))) *(( 8.81438827514648438)))) +
0.100000*np.tanh(((((((((data["pred_0"])-(((( 13.48505210876464844)) -(((data["pred_1"])+(data["t_sum_log"])))))))* 2.0)) -(2.0)))* 2.0)) +
0.100000*np.tanh(((((((((((( 9.53180027008056641)) *(((data["t_sum_log"])-(( 6.0)))))) * 2.0)) * 2.0)) +(((( 9.53180027008056641)) / 2.0)))) * 2.0)) +
0.100000*np.tanh(((data["pred_3"])+(((data["pred_3"])+(((data["pred_3"])+(((((((data["t_sum_log"])-(( 10.0)))) * 2.0)) * 2.0)))))))) +
0.100000*np.tanh(((((((((((((data["t_sum_act"])-(( 11.60270595550537109)))) +(data["t_sum_log"])))* 2.0)) *(( 11.60270595550537109)))) -(( 11.60270595550537109)))) * 2.0)) +
0.100000*np.tanh(((((((((((data["pred_0"])-(( 9.22513771057128906)))) *(( 9.22513771057128906)))) -(data["t_median"])))* 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["pred_3"])+(((((data["t_sum_log"])-(( 10.0)))) *(( 10.0)))))) * 2.0)) +(( 10.0)))) * 2.0)) +
0.100000*np.tanh(((((((data["t_sum_act"])-(( 6.00677633285522461)))) *(((( 6.00677633285522461)) -(((data["pred_3"])-(data["t_sum_act"])))))))+(data["t_sum_act"])))+
0.100000*np.tanh(((((data["pred_1"])+(((((data["pred_1"])+(((((data["t_mean"])-(( 14.10800457000732422)))) +(data["pred_0"])))))* 2.0)))) * 2.0)) +
0.100000*np.tanh(((((( 9.98417091369628906)) * 2.0)) *(((( 8.10735034942626953)) *(((( 9.98417091369628906)) *(((((data["pred_2"])-(( 4.0)))) * 2.0)))))))) +
0.100000*np.tanh(((((((((((((data["t_sum_log"])-(( 9.48249053955078125)))) +(-2.0)))+(data["t_sum_act"])))* 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((data["pred_2"])-(np.maximum(((data["next_session_2"])) ,(( np.maximum(((((data["visitNumber"])*(data["visitNumber"])))) ,(( data["visitNumber"])))))))))* 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_act"])+(((((((((data["t_sum_act"])-(( 6.93721342086791992)))) * 2.0)) * 2.0)) * 2.0)))) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((( 11.06935405731201172)) *(((((((data["t_sum_log"])-(( 11.06935405731201172)))) +(np.where(data["pred_1"]>0, 0.0, data["t_sum_log"])))) * 2.0)))) +
0.100000*np.tanh(((((np.where(((-3.0)+(data["pred_1"])) >0,(6.0), -3.0)) +(data["pred_3"])))-(data["visitNumber"])))+
0.100000*np.tanh(((((( 10.0)) *(((((( np.where(data["pred_1"]>0,(2.85201859474182129), data["pred_0"])) +(data["pred_1"])) /2.0)) +(-3.0)))))* 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_log"])+(((-3.0)* 2.0)))) * 2.0)) * 2.0)) *(((( 8.0)) -(data["t_sum_log"])))))+
0.100000*np.tanh(((((((((((((data["t_sum_act"])-(np.maximum(((( 6.0))),(( data["visitNumber"])))))) * 2.0)) * 2.0)) +(data["next_session_1"])))* 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_act"])-(((3.0)* 2.0)))) +(( 0.43446072936058044)))) *(( 9.0)))) * 2.0)) +
0.100000*np.tanh(((( 9.72239112854003906)) *(((np.where(data["pred_2"] < -99998, data["pred_1"], data["t_sum_log"])) -(np.maximum(((data["visitNumber"])) ,(((9.72239112854003906)))))))))+
0.100000*np.tanh(((((((((((((data["t_sum_log"])-(np.maximum(((( 6.0))),(( data["visitNumber"])))))) * 2.0)) * 2.0)) -(-2.0)))* 2.0)) * 2.0)) +
0.100000*np.tanh(np.minimum(((((( data["t_median"])<(( 7.85152149200439453)))*1.))) ,(((((( data["next_session_2"])+(((((data["pred_0"])-(( 7.85152149200439453)))) * 2.0)))) * 2.0)))))+
0.100000*np.tanh(((((((((((data["t_sum_log"])-(np.maximum(((data["visitNumber"])) ,(((( 3.0)* 2.0)))))))* 2.0)) * 2.0)) +(3.0)))* 2.0)) +
0.100000*np.tanh(((( 14.69243335723876953)) *(((((data["pred_3"])-(((data["pred_0"])+(((data["geoNetwork_country"])+(( 10.17979812622070312)))))))) +(data["pred_3"])))))+
0.100000*np.tanh(((((((((((((data["t_sum_act"])-(((( 10.99480342864990234)) / 2.0)))) * 2.0)) *(( 10.99802207946777344)))) +(data["t_sum_log"])))* 2.0)) * 2.0)) +
0.100000*np.tanh(((((((( 10.0)) *(((data["pred_3"])-(( 6.0)))))) +(3.0)))*(( 12.68457317352294922)))) +
0.100000*np.tanh(((((( 6.0)) -(((data["pred_0"])-(data["t_median"])))))*(((((data["pred_0"])-(( 7.13541793823242188)))) *(( 8.68632888793945312)))))) +
0.100000*np.tanh(((((((np.minimum(((((data["t_sum_log"])-(( 5.0))))),(((( data["next_session_2"])-(data["visitNumber"])))))) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_act"])* 2.0)) -(((data["pred_0"])-(data["pred_3"])))))-(( 10.0)))) *(( 10.0)))) +
0.100000*np.tanh(((( 13.06485366821289062)) *(((((((( 13.06485366821289062)) *(((((data["t_sum_log"])-(-3.0)))-(( 8.01604843139648438)))))) * 2.0)) * 2.0)))) +
0.100000*np.tanh(((((((data["t_sum_log"])-(np.maximum(((data["t_median"])) ,(( np.maximum(((data["visitNumber"])) ,(((((10.0)) -(data["t_sum_log"])))))))))))* 2.0)) * 2.0)) +
0.100000*np.tanh(((((np.where(data["pred_3"]<0,(( data["t_sum_log"])+(data["t_sum_log"])) , data["t_sum_log"])) -(( 10.0)))) *(( 5.26560354232788086)))) +
0.100000*np.tanh(((((((data["t_mean"])-(data["t_sum_log"])))+(((((data["t_sum_log"])-(( 10.0)))) *(( 11.74072933197021484)))))) * 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_act"])* 2.0)) -(( 10.21498489379882812)))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((( 10.75512409210205078)) *(((((data["t_sum_log"])-(np.tanh(( data["pred_2"])))))-(( 10.75512409210205078)))))) +(( 8.73960781097412109)))) * 2.0)) +
0.100000*np.tanh(((( 14.71356868743896484)) *(((((( 14.71356868743896484)) *(((data["pred_0"])+(((-3.0)* 2.0)))))) +(((-3.0)* 2.0)))))) +
0.100000*np.tanh(((( 9.81327152252197266)) *(((((((((7.16616725921630859)) >(data["t_mean"])) *1.))*(data["t_sum_act"])))+(((data["t_sum_act"])-(( 9.76804924011230469)))))))) +
0.100000*np.tanh(((((data["t_sum_act"])+(((data["t_sum_log"])-(np.maximum(((((((data["totals_hits"])* 2.0)) +(data["t_sum_log"])))) ,(((9.55896186828613281)))))))))* 2.0)) +
0.100000*np.tanh(((((((((((((data["t_sum_log"])* 2.0)) -(( 10.0)))) -(data["geoNetwork_country"])))* 2.0)) * 2.0)) -(data["visitNumber"])))+
0.100000*np.tanh(((((data["t_sum_log"])-(np.where(((data["pred_1"])-(2.0)) <0,(11.79721641540527344),(( data["visitNumber"])+(data["t_median"])))))) * 2.0)) +
0.100000*np.tanh(((((((((( 11.64218807220458984)) *(((data["pred_3"])-(( 5.93939924240112305)))))) * 2.0)) -(-3.0)))*(( 5.93939924240112305)))) +
0.100000*np.tanh(((( 4.71989488601684570)) *(((((((( 4.71989488601684570)) *(((((data["t_sum_log"])-(( 4.71989488601684570)))) * 2.0)))) +(data["pred_0"])))* 2.0)))) +
0.100000*np.tanh(((((((np.where(((data["t_sum_act"])-(( 7.0)))<0,(( data["t_sum_log"])* 2.0), data["t_sum_act"])) -(( 9.0)))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["t_sum_log"])-(( 10.06210994720458984)))) *(( 13.34921360015869141)))) -(data["t_sum_log"])))-(( 10.0)))) +
0.100000*np.tanh(((((((((data["t_sum_log"])* 2.0)) -(( 8.0)))) *(((( 7.0)) -(data["t_sum_log"])))))*(( 8.0)))) +
0.100000*np.tanh(np.minimum(((((((((np.minimum(((((((data["t_mean"])-(( 5.0)))) * 2.0))),(( data["trafficSource_source"])))) * 2.0)) * 2.0)) * 2.0))),(( data["geoNetwork_metro"])))) +
0.100000*np.tanh(np.minimum(((((((( 8.0)) *(((((data["t_sum_log"])* 2.0)) -(( 8.0)))))) * 2.0))),(((((7.0)) -(data["t_sum_log"])))))) +
0.100000*np.tanh(((((((-3.0)+(np.where(data["pred_2"] < -99998, np.where(data["pred_1"] < -99998, -3.0, data["pred_0"]), data["t_mean"])))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((data["pred_0"])-(2.0)))-(np.where(data["pred_1"]<0, data["totals_hits"], data["pred_0"])))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((((((data["pred_0"])* 2.0)) +(np.minimum(((data["pred_1"])) ,(( data["pred_1"])))))) -(( 8.0)))) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((-2.0)* 2.0)) +(data["t_sum_log"])))*(((( 6.0)) -(data["t_sum_log"])))))*(( 6.22247934341430664)))) +
0.100000*np.tanh(((((((data["t_sum_log"])-(np.maximum(((np.maximum(((data["pred_0"])) ,(((((( 1.0)* 2.0)) * 2.0)))))) ,(( data["visitNumber"])))))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((data["t_sum_log"])+(((np.where(data["pred_2"]>0, -3.0,(( data["t_sum_act"])-(( 6.0)))))* 2.0)))) * 2.0)) * 2.0)) +
0.100000*np.tanh(((( 14.27134418487548828)) *(((((( 11.12297344207763672)) *(((((data["geoNetwork_continent"])-(( 9.0)))) +(data["t_sum_log"])))))+(data["next_session_1"])))))+
0.100000*np.tanh(np.where(((data["pred_0"])+(((-2.0)* 2.0)))>0, data["pred_0"],(( np.minimum(((-3.0)) ,(( data["pred_3"])))) +(data["t_mean"])))) +
0.100000*np.tanh(((((((np.where(((-3.0)+(data["totals_hits"])) <0, data["t_sum_act"], 2.0)) -(( 4.0)))) * 2.0)) * 2.0)) +
0.100000*np.tanh(np.minimum(((((( 4.68725061416625977)) -(((data["totals_hits"])-(-2.0)))))) ,(( np.minimum(((data["t_median"])) ,(((( data["t_sum_log"])-(( 4.68725061416625977)))))))))) +
0.100000*np.tanh(np.minimum(((((data["pred_0"])*(( 12.75279808044433594))))),(((((( data["t_sum_act"])*(np.maximum(((data["t_sum_log"])) ,(( data["pred_0"])))))) -(( 12.75279808044433594)))))))+
0.100000*np.tanh(((( 13.62804603576660156)) *(((((((((4.70133781433105469)) -(( 9.57412528991699219)))) +(((((data["pred_1"])* 2.0)) * 2.0)))/2.0)) -(data["t_median"])))))+
0.100000*np.tanh(((((((np.minimum(((((((((data["t_sum_log"])+(-3.0)))* 2.0)) * 2.0))),(( data["totals_pageviews"])))) * 2.0)) +(-3.0)))* 2.0)) +
0.100000*np.tanh(np.minimum(((((( 5.0)) *(((data["t_sum_act"])-(((( 6.49275684356689453)) -(data["t_sum_act"])))))))) ,(((((5.0)) -(data["pred_0"])))))) +
0.100000*np.tanh(((((( 9.0)) *(((((data["t_median"])-(( 9.0)))) -(data["visitNumber"])))))+(((data["t_sum_log"])/ 2.0)))) +
0.100000*np.tanh(((((((((((((data["t_sum_log"])+(-3.0)))* 2.0)) +(((data["pred_0"])-(data["totals_hits"])))))* 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((data["t_median"])+(((((((((((np.maximum(((data["t_median"])) ,(( data["t_sum_log"])))) -(( 10.68563747406005859)))) * 2.0)) * 2.0)) * 2.0)) * 2.0)))) +
0.100000*np.tanh(((((((((-3.0)+(np.where(data["pred_12"] < -99998, np.maximum(((data["t_mean"])) ,(( data["pred_0"]))), data["pred_12"])))) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((((np.minimum(((data["pred_1"])) ,(((( np.minimum(((2.0)) ,(( data["pred_2"])))) -(data["pred_1"])))))) * 2.0)) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((np.minimum(((((np.where(data["geoNetwork_country"]<0, data["t_sum_log"], -3.0)) -(3.0)))) ,(((( data["next_session_1"])-(data["visitNumber"])))))) * 2.0)) +
0.100000*np.tanh(((((np.where(((-2.0)+(data["pred_1"])) >0, -3.0,(( -3.0)+(data["t_sum_log"])))) * 2.0)) * 2.0)) +
0.100000*np.tanh(np.minimum(((((( 5.0)) -(data["t_mean"])))) ,(((((( data["t_sum_log"])-(((( 5.0)) -(data["t_sum_log"])))))* 2.0)))))+
0.100000*np.tanh(((-2.0)+(((((((((data["pred_0"])+(-3.0)))* 2.0)) * 2.0)) +(((data["t_sum_log"])*(data["next_session_1"])))))))+
0.100000*np.tanh(((((((data["t_sum_log"])-(np.maximum(((2.0)) ,(((((( data["t_sum_log"])-(2.0)))* 2.0)))))))*(( 12.37948131561279297)))) * 2.0)) +
0.100000*np.tanh(((data["t_sum_act"])-(np.maximum(((((( 4.0)) *(((( 4.94181728363037109)) *(((( data["next_session_1"])>(data["next_session_2"])) *1.))))))) ,(((4.0)))))))+
0.100000*np.tanh(((((np.where(data["geoNetwork_continent"]>0, -3.0, np.where(data["pred_2"]>0, data["pred_2"],(( data["t_sum_log"])* 2.0)))) * 2.0)) -(( 6.66043043136596680)))) +
0.100000*np.tanh(((((data["t_sum_log"])-(np.maximum(((np.where(data["channelGrouping"]<0, data["totals_pageviews"], data["t_sum_log"]))),(( np.maximum(((data["pred_0"])) ,(( 3.0)))))))))* 2.0)) +
0.100000*np.tanh(((((((((np.minimum(((((data["t_sum_act"])+(-2.0)))) ,(((( data["geoNetwork_country"])*(data["pred_1"])))))) * 2.0)) * 2.0)) * 2.0)) * 2.0)) +
0.100000*np.tanh(((((((data["pred_13"])* 2.0)) *(((( 2.0)>(np.where(data["pred_13"] < -99998, data["t_sum_log"], data["pred_12"])))*1.))))*(( 14.28744792938232422)))) +
0.100000*np.tanh(np.where(data["pred_1"]>0,(( np.where(data["pred_0"]>0,(6.0), data["pred_1"])) -(data["t_median"])) ,(( data["t_median"])-(( 6.0)))))+
0.100000*np.tanh(((((((((np.minimum(((data["t_sum_log"])) ,(( data["totals_pageviews"])))) * 2.0)) +(((( -3.0)+(-2.0)) /2.0)))) *(( 7.0)))) * 2.0)) +
0.100000*np.tanh(((np.where(data["pred_13"] < -99998, data["t_mean"], data["pred_13"])) +(((((np.maximum(((data["pred_12"])) ,(( data["pred_13"])))) * 2.0)) +(-3.0)))))+
0.100000*np.tanh(np.where(data["pred_29"] < -99998,(((((((( -1.0)+(((data["t_sum_log"])* 2.0)))) * 2.0)) * 2.0)) +(-3.0)) , -3.0)) +
0.100000*np.tanh(((((((((data["pred_13"])* 2.0)) * 2.0)) * 2.0)) *(((( data["pred_200"])<(((data["t_sum_act"])*(((data["pred_13"])* 2.0)))))*1.))))+
0.099658*np.tanh(((np.tanh(( np.tanh(( np.where(((((( -3.0)/ 2.0)) +(data["pred_12"])) /2.0)>0,(3.79615640640258789),(( -3.0)/ 2.0)))))))* 2.0)) +
0.100000*np.tanh(((data["pred_254"])*(np.maximum(((((data["pred_254"])-(-3.0)))) ,(((((data["pred_29"])>(((data["pred_148"])-(data["pred_41"])))) *1.))))))) +
0.099951*np.tanh(((data["pred_132"])-(np.where(data["pred_29"]<0,(( data["pred_41"])-(((((( data["pred_132"])<(data["pred_29"])) *1.))* 2.0))),(4.58089542388916016)))))+
0.099756*np.tanh(np.where(data["pred_12"] < -99998, 0.0, np.where(data["pred_43"] < -99998, data["pred_12"], data["pred_254"])))+
0.099707*np.tanh(np.where(data["pred_29"]>0, data["pred_148"],(( np.minimum(((data["pred_29"])) ,(((((( data["pred_132"])-(data["pred_41"])))* 2.0)))))-(data["pred_254"])))) +
0.099316*np.tanh(((np.where(np.where(data["pred_132"] < -99998, data["pred_29"], data["pred_254"])< -99998, data["pred_29"],(( data["pred_254"])-(data["pred_41"])))) -(data["pred_29"])))+
0.099951*np.tanh(np.where(data["pred_29"]>0, -3.0, np.where(data["pred_132"]>0, -3.0,(( data["pred_132"])-(np.minimum(((-3.0)) ,(( data["pred_29"])))))))) +
0.100000*np.tanh(((((((np.where(data["pred_11"]<0, data["pred_254"],(( data["pred_11"])-(data["pred_29"])))) -(data["pred_29"])))* 2.0)) * 2.0)) +
0.099609*np.tanh(((((data["pred_29"])-(((data["pred_148"])-(np.maximum(((((data["pred_254"])-(data["pred_29"])))) ,(( data["pred_198"])))))))) *(data["pred_254"])))+
0.099805*np.tanh(((((( np.where(data["pred_132"] < -99998, data["pred_41"], 3.0)) +(data["pred_132"])) /2.0)) -(np.where(data["pred_29"]<0, data["pred_41"],(6.78871536254882812)))))+
0.099853*np.tanh(((((np.minimum(((((( data["pred_29"])>(data["pred_88"])) *1.))) ,(((((( data["pred_111"])-(data["pred_41"])))-(data["pred_29"])))))) * 2.0)) * 2.0)) +
0.099560*np.tanh(np.where(data["pred_29"]>0, -2.0, np.where(data["pred_41"]>0, -2.0,(( data["pred_153"])-(np.minimum(((data["pred_41"])) ,(( -3.0)))))))) +
0.099707*np.tanh(((np.where(data["pred_29"]>0, data["pred_251"],(((data["pred_29"])>(data["pred_136"])) *1.))) +(((data["pred_132"])-(data["pred_41"])))))+
0.099463*np.tanh(((((data["pred_29"])-(data["pred_136"])))*(((( 11.27190399169921875)) +(((data["pred_111"])+(( 12.76537227630615234)))))))) +
0.100000*np.tanh(np.maximum(((((data["pred_132"])-(((( data["pred_251"])+(data["pred_13"])) /2.0))))),(((((((((( data["pred_13"])* 2.0)) * 2.0)) * 2.0)) * 2.0)))))+
0.099805*np.tanh(np.where(data["pred_29"]>0, data["pred_251"],(( data["pred_132"])-(np.where(data["pred_148"] < -99998, data["pred_43"],(((( data["pred_251"])* 2.0)) * 2.0)))))) +
0.099658*np.tanh(np.where(data["pred_29"]>0, data["pred_254"], np.where(data["pred_153"] < -99998,(( data["pred_29"])-(data["pred_43"])) ,(( data["pred_29"])-(data["pred_153"])))))+
0.099805*np.tanh(np.maximum(((((((data["pred_254"])* 2.0)) * 2.0))),(((( np.minimum(((data["pred_41"])) ,(((((( data["pred_132"])/ 2.0)) / 2.0)))))-(data["pred_41"])))))) +
0.099756*np.tanh(np.where(data["pred_29"]>0, data["pred_254"],(( np.where(data["pred_41"] < -99998, data["pred_29"],(( data["pred_148"])-(-1.0)))) -(data["pred_41"])))) +
0.099707*np.tanh(((data["pred_148"])-(np.where(data["pred_29"]>0,(7.0), np.where(data["pred_148"]>0,(7.0), data["pred_41"])))))+
0.099805*np.tanh(np.where(np.maximum(((data["pred_111"])) ,(( data["pred_29"])))>0, -3.0,(( data["pred_111"])-(np.minimum(((data["pred_41"])) ,(( -3.0)))))))+
0.100000*np.tanh(np.where(data["pred_61"]>0,(9.0),(( data["pred_111"])-(data["pred_29"])))) +
0.099170*np.tanh(np.where(data["pred_29"]<0,(( np.where(data["pred_41"] < -99998, data["pred_29"], data["pred_132"])) -(((( data["pred_41"])+(data["pred_251"])) /2.0))), data["pred_254"])) +
0.099609*np.tanh(((((data["pred_29"])-(data["pred_153"])))*(((((((data["pred_29"])+(data["pred_41"])))-(data["pred_132"])))*(data["pred_61"])))))+
0.099609*np.tanh(np.where(data["pred_153"]>0, -3.0, np.where(data["pred_29"]>0, -3.0,(( data["pred_153"])-(np.minimum(((-3.0)) ,(( data["pred_43"])))))))) +
0.099609*np.tanh(((data["pred_29"])-(np.where(np.where(data["pred_29"]<0, data["pred_41"], data["pred_29"])< -99998, data["pred_61"],(( data["pred_61"])*(data["pred_200"])))))))
<compute_test_metric>
train['Ticket1']=train['Ticket1'].replace(['1', '2', '3', '4', '5', '6', '7', '8', '9'], 'N' )
print(mean_squared_error(np.log1p(trn_user_target['target']), GP1(gp_trn_users ).clip(0,maxval)) **.5)
<find_best_params>
train['Ticket1']=train['Ticket1'].replace('N', 'NumberTicket' )
a = GP1(gp_sub_users ).clip(0,maxval ).values<save_to_csv>
train['Ticket1']=train['Ticket1'].replace(['A','W', 'F', 'L'], 'OtherTicket' )
sub_full_data['PredictedLogRevenue'] = a
sub_full_data[['PredictedLogRevenue']].to_csv('gp1.csv', index=True)
<load_from_csv>
train['Ticket1']=train['Ticket1'].replace(['S','P', 'C'], ['S_Ticket', 'P_Ticket', 'C_Ticket'] )
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str)
 train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False)
 for df in [train, test]:
df["visitId"] = df["visitId"].apply(lambda x: x.split('.', 1)[0] ).astype(str)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId")
 for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 for df in [train_new, test_new]:
df["Sessions"] = df["Sessions"].fillna(0)
 df["Avg.Session Duration"] = df["Avg.Session Duration"].fillna(0)
 df["Bounce Rate"] = df["Bounce Rate"].fillna(0)
 df["Revenue"] = df["Revenue"].fillna(0)
 df["Transactions"] = df["Transactions"].fillna(0)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].fillna(0)
 df['trafficSource.adContent'].fillna('N/A', inplace=True)
 df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)
 df['trafficSource.referralPath'].fillna('N/A', inplace=True)
 df['trafficSource.keyword'].fillna('N/A', inplace=True)
 df['totals.bounces'].fillna(0.0, inplace=True)
 df['totals.newVisits'].fillna(0.0, inplace=True)
 df['totals.pageviews'].fillna(0.0, inplace=True)
 del train
del test
train = train_new
test = test_new
del train_new
del test_new
gc.collect()<data_type_conversions>
train["Ticket1"].value_counts()
for df in [train, test]:
df.rename({'fullVisitorId': 'id', 'totals.transactionRevenue': 'target'}, axis = 1, inplace = True)
 df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['weekday'] = df['date'].dt.dayofweek
df['hour'] = df['date'].dt.hour
df['monthday'] = df['date'].dt.day
df.sort_values(['id', 'date'], ascending = True, inplace = True)
 df['next_session'] =(df['date'] - df.groupby('id', sort = False)['date'].shift(1)).astype(np.int64)// 1e9 // 60 // 60
df['prev_session'] =(df['date'] - df.groupby('id', sort = False)['date'].shift(-1)).astype(np.int64)// 1e9 // 60 // 60
df.sort_index(inplace = True)
 train['target'].fillna(0, inplace = True)
 user_labels =(train.groupby('id', sort = False)['target'].max() > 0 ).astype(int)
 user_sums = np.log1p(np.array(train.groupby('id', sort = False)['target'].sum() ).tolist())
 user_ids = train['id'].unique()
session_sums = train['target'].copy()
del train['target']<define_variables>
test['Ticket1']=test['Ticket1'].replace(['1', '2', '3', '4', '5', '6', '7', '8', '9'], 'N' )
mobile_words = {'android', 'samsung', 'mini', 'iphone', 'in-app', 'playstation',
'mozilla', 'chrome', 'blackberry', 'nokia', 'browser', 'amazon',
'lunascape', 'netscape', 'konqueror', 'puffin', 'amazon'}
normal_browsers = {'chrome', 'safari', 'firefox', 'internet explorer', 'edge', 'opera',
'coc coc', 'maxthon', 'iron'}
key_sources = {'google', 'youtube', 'yahoo', 'facebook', 'reddit', 'bing', 'outlook', 'linkedin',
'pinterest', 'ask', 'siliconvalley', 'lunametrics', 'amazon', 'mysearch', 'qiita',
'messenger', 'twitter', 't.co', 'vk.com', 'search', 'edu', 'mail', 'ad', 'golang',
'direct', 'dealspotr', 'sashihara', 'phandroid', 'baidu', 'mdn', 'duckduckgo', 'seroundtable',
'metrics', 'sogou', 'businessinsider', 'github', 'gophergala', 'yandex', 'msn', 'dfa',
'feedly', 'arstechnica', 'squishable', 'flipboard', 't-online.de', 'sm.cn', 'wow', 'baidu',
'partners'}
def browser_mapping(x):
if x in normal_browsers:
return x
elif any([word in x for word in mobile_words]):
return 'mobile_browser'
elif '(not set)' in x:
return 'nan'
else:
return 'others'
def adcontents_mapping(x):
if 'google' in x:
return 'google'
elif '(not set)' in x or 'nan' in x:
return 'nan'
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
for word in key_sources:
if word in x:
return word
if '(not set)' in x or 'nan' in x:
return 'nan'
else:
return 'others'
for df in [train, test]:
df['device.browser'] = df['device.browser'].astype(str ).map(lambda x: browser_mapping(x.lower()))
df['trafficSource.adContent'] = df['trafficSource.adContent'].astype(str ).map(lambda x: browser_mapping(x.lower()))
df['trafficSource.source'] = df['trafficSource.source'].astype(str ).map(lambda x: source_mapping(x.lower()))
pairs = [('trafficSource.source', 'geoNetwork.country'),('trafficSource.campaign', 'trafficSource.medium'), 
('device.browser', 'device.deviceCategory'),('device.browser', 'device.operatingSystem'), 
('device.browser', 'channelGrouping'),('device.deviceCategory', 'channelGrouping'), 
('device.operatingSystem', 'channelGrouping'), 
('trafficSource.adContent', 'source_country'),('trafficSource.medium', 'source_country')]
def get_second_part(word):
return re.sub('.*\.', '', word)
 for df in [train, test]:
for first, second in pairs:
df[get_second_part(first)+ '_' + get_second_part(second)] = df[first] + '_' + df[second]
for first in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro',
'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for second in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[get_second_part(first)+ "_" + get_second_part(second)] = df[first] + "_" + df[second]<feature_engineering>
test['Ticket1']=test['Ticket1'].replace(['S','P', 'C', 'N'], ['S_Ticket', 'P_Ticket', 'C_Ticket','NumberTicket'] )
excluded_cols = {'date', 'id', 'visitId', 'visitStartTime', 'sessionId'}
cat_cols = [col for col in train.columns if col not in excluded_cols and train[col].dtype == 'object']
for col in cat_cols:
train[col], indexer = pd.factorize(train[col])
 test[col] = indexer.get_indexer(test[col] )<prepare_x_and_y>
test['Ticket1']=test['Ticket1'].replace(['A','W', 'F', 'L'], 'OtherTicket' )
n_splits = 5
splits = get_folds(df = train, n_splits = n_splits)
 train_cols = [col for col in train.columns if col not in excluded_cols]
oof_preds = np.zeros(train.shape[0])
 test_preds = np.zeros(test.shape[0])
 val_scores = []
for i in range(n_splits):
tr_idx, val_idx = splits[i]
print("Fold:", i + 1, end = '.')
 train_X, train_y = train[train_cols].iloc[tr_idx].values, np.log1p(session_sums[tr_idx])
 val_X, val_y = train[train_cols].iloc[val_idx].values, np.log1p(session_sums[val_idx])
 gbm = LGBMRegressor(num_leaves = 31, learning_rate = 0.03, n_estimators = 1000, subsample=.9,
colsample_bytree=.9 , random_state = 1)
 gbm.fit(train_X, train_y, eval_set =(val_X, val_y),early_stopping_rounds = 150, verbose = False,
eval_metric = 'rmse')
 val_pred = gbm.predict(val_X)
 oof_preds[val_idx] = val_pred
val_scores.append(np.sqrt(mean_squared_error(val_pred, val_y)))
 print('Score:', val_scores[-1])
 test_pred = gbm.predict(test[train_cols])
 test_pred[test_pred < 0] = 0
test_preds += np.expm1(test_pred)/ n_splits
oof_preds[oof_preds < 0] = 0
print(np.sqrt(mean_squared_error(np.log1p(session_sums), oof_preds)))
 train['preds'] = np.expm1(oof_preds)
 train['log_preds'] = oof_preds
test['preds'] = test_preds
test['log_preds'] = np.log1p(test_preds )<merge>
test["Ticket1"].value_counts()
stats = ['max', 'mean', 'median', 'std', 'size', 'sum']
user_train = train[train_cols + ['id']].groupby('id', sort = False ).mean()
user_test = test[train_cols + ['id']].groupby('id', sort = False ).mean()
train_preds = train.groupby('id', sort = False ).agg({'preds': stats, 'log_preds': ['sum']} ).fillna(0)
 train_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']
for col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:
train_preds[col] = np.log1p(train_preds[col])
 user_train = user_train.merge(train_preds, left_index = True, right_index = True)
 test_preds = test.groupby('id', sort = False ).agg({'preds': stats, 'log_preds': ['sum']} ).fillna(0)
 test_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']
for col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:
test_preds[col] = np.log1p(test_preds[col])
 user_test = user_test.merge(test_preds, left_index = True, right_index = True )<feature_engineering>
dumytitletr1=pd.get_dummies(train['Ticket1'])
 train=pd.concat([train,dumytitletr1], axis=1 )
time_min = train['visitStartTime'].min()
time_max = train['visitStartTime'].max()
for df in [train, test]:
df['visitStartTime'] -= time_min
df['visitStartTime'] /=(time_max - time_min)
 aggregations = ['min', 'max', 'std']
times_train = train.groupby('id', sort = False)['visitStartTime'].agg(aggregations ).fillna(0)
 times_train.columns = ['times_' + word for word in aggregations]
times_train['times_diff'] = times_train['times_max'] - times_train['times_min']
times_train['times_diff_n'] = times_train['times_diff'] / user_train['pred_size']
times_train.drop(['times_min', 'times_max'], axis = 1, inplace = True)
 times_test = test.groupby('id', sort = False)['visitStartTime'].agg(aggregations ).fillna(0)
 times_test.columns = ['times_' + word for word in aggregations]
times_test['times_diff'] = times_test['times_max'] - times_test['times_min']
times_test['times_diff_n'] = times_test['times_diff'] / user_test['pred_size']
times_test.drop(['times_min', 'times_max'], axis = 1, inplace = True)
 train['date'] = train['date'].astype(str ).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))
 test['date'] = test['date'].astype(str ).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))
 def analyze_dates(user):
features = []
dates = sorted(user['date'].values)
 n = user.shape[0]
diff =(dates[-1] - dates[0] ).days/360
features += [diff, diff/n]
features += [Counter(dates ).most_common() [0][1]]
return features
dates_train = np.array(train.groupby('id', sort = False ).apply(lambda x: analyze_dates(x)).tolist())
 dates_test = np.array(test.groupby('id', sort = False ).apply(lambda x: analyze_dates(x)).tolist() )<init_hyperparams>
dumytitletes1=pd.get_dummies(test['Ticket1'])
 test=pd.concat([test,dumytitletes1], axis=1 )
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.03,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
lgb_params = {
'learning_rate': 0.03,
'metric': 'rmse',
'subsample': 0.9,
'colsample_bytree': 0.9,
'random_state': 1,
'num_leaves': 31
}
lgb_params_2 = {
'learning_rate': 0.03,
'metric': 'rmse',
'subsample': 0.9,
'colsample_bytree': 0.9,
'random_state': 1,
'num_leaves': 10
}<define_variables>
Sex_mapping={"male":0,"female":1}
train["Sex"]=train["Sex"].map(Sex_mapping)
 test["Sex"]=test["Sex"].map(Sex_mapping )
splits = get_folds(df = user_train.reset_index() , n_splits = n_splits)
 oof_preds = {'lgb': np.zeros(user_train.shape[0]),
'xgb': np.zeros(user_train.shape[0]),
'weighted': np.zeros(user_train.shape[0])}
sub_preds = {'lgb': np.zeros(user_test.shape[0]),
'xgb': np.zeros(user_test.shape[0])}
val_scores = {'lgb': [], 'xgb': [], 'weighted': []}
print(' fold | lgb | xgb | weighted ')
 print('---------------------------------------')
 for i in range(n_splits):
tr, val = splits[i]
train_X, train_y = np.hstack([user_train.iloc[tr], dates_train[tr], times_train.iloc[tr]]), user_sums[tr]
val_X, val_y = np.hstack([user_train.iloc[val], dates_train[val], times_train.iloc[val]]), user_sums[val]
models = {'lgb': LGBMRegressor(**lgb_params, n_estimators = 1500),
'xgb': XGBRegressor(**xgb_params, n_estimators = 1000)}
for name in ['xgb', 'lgb']:
models[name].fit(train_X, train_y, eval_set = [(val_X, val_y)],
early_stopping_rounds = 100, eval_metric = 'rmse', verbose = False)
 val_pred = models[name].predict(val_X)
 oof_preds[name][val] = val_pred
val_scores[name].append(np.sqrt(mean_squared_error(val_pred, val_y)))
 test_pred = models[name].predict(np.hstack([user_test, dates_test, times_test]))
test_pred[test_pred < 0] = 0
sub_preds[name] += test_pred / n_splits
val_pred = 0.7 * oof_preds['lgb'][val] + 0.3 * oof_preds['xgb'][val]
oof_preds['weighted'][val] = val_pred
val_scores['weighted'].append(np.sqrt(mean_squared_error(val_pred, val_y)))
 print(' {fold: 3d} | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\ 
.format(fold = i + 1, lgb = val_scores['lgb'][-1], xgb = val_scores['xgb'][-1], w = val_scores['weighted'][-1]))
print('---------------------------------------')
 cv_scores = {}
for name in ['lgb', 'xgb']:
oof_preds[name][oof_preds[name] < 0] = 0
cv_scores[name] = mean_squared_error(user_sums, oof_preds[name])**.5
cv_scores['weighted'] = np.sqrt(mean_squared_error(user_sums, 0.6 * oof_preds['lgb'] + 0.4 * oof_preds['xgb']))
print(' CV | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\ 
.format(lgb = cv_scores['lgb'], xgb = cv_scores['xgb'], w = cv_scores['weighted']))<save_to_csv>
test=test.drop(['Ticket', 'Ticket1', 'Cabin', 'Title'], axis=1 )
sub = pd.DataFrame()
sub['fullVisitorId'] = user_test.index
sub['PredictedLogRevenue'] = sub_preds['lgb'] * 0.6 + sub_preds['xgb'] * 0.4
sub.loc[(test.groupby('id', sort = False)['totals.bounces'].min() == 1 ).values, 'PredictedLogRevenue'] = 0.
 sub.to_csv("sub.csv", index = False )<set_options>
train=train.drop(['Ticket', 'Ticket1', 'Cabin', 'Title'], axis=1 )
warnings.simplefilter('error', SettingWithCopyWarning)
 warnings.simplefilter("ignore")
 gc.enable()
sns.set()
%matplotlib inline<load_from_csv>
X = train.drop(['Survived', 'PassengerId'], axis=1)
 y = train["Survived"]
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.20, random_state = 21 )
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, "visitId":str}, nrows=None)
 train.shape, test.shape<load_from_csv>
logreg = LogisticRegression()
logreg.fit(x_train, y_train)
 y_pred = logreg.predict(x_test)
 acc_logreg = round(accuracy_score(y_pred, y_test)* 100, 2)
 print(acc_logreg )
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'} )<data_type_conversions>
logmodel = LogisticRegression(random_state= 21, solver='lbfgs', max_iter=1000)
 logmodel.fit(x_train, y_train )
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(str )<merge>
logmodel.score(x_train, y_train )
train_exdata = pd.concat([train_store_1, train_store_2], sort=False)
 test_exdata = pd.concat([test_store_1, test_store_2], sort=False)
 for df in [train, test]:
df["visitId"] = df["visitId"].apply(lambda x: x.split('.', 1)[0] ).astype(str)
 train_new = train.merge(train_exdata, how="left", on="visitId")
 test_new = test.merge(test_exdata, how="left", on="visitId" )<data_type_conversions>
logmodel.intercept_
for df in [train_new, test_new]:
df.drop("Client Id", 1, inplace=True)
 for df in [train_new, test_new]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True)
 for df in [train_new, test_new]:
df["Sessions"] = df["Sessions"].fillna(0)
 df["Avg.Session Duration"] = df["Avg.Session Duration"].fillna(0)
 df["Bounce Rate"] = df["Bounce Rate"].fillna(0)
 df["Revenue"] = df["Revenue"].fillna(0)
 df["Transactions"] = df["Transactions"].fillna(0)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].fillna(0)
 df['trafficSource.adContent'].fillna('N/A', inplace=True)
 df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)
 df['trafficSource.referralPath'].fillna('N/A', inplace=True)
 df['trafficSource.keyword'].fillna('N/A', inplace=True)
 df['totals.bounces'].fillna(0.0, inplace=True)
 df['totals.newVisits'].fillna(0.0, inplace=True)
 df['totals.pageviews'].fillna(0.0, inplace=True )<drop_column>
y_pred = logmodel.predict(x_test )
del train
del test
train = train_new
test = test_new
del train_new
del test_new
gc.collect()<prepare_x_and_y>
from sklearn.metrics import classification_report
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
print(classification_report(y_test,y_pred))
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<categorify>
logmodel.score(x_train, y_train )
def browser_mapping(x):
browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']
if x in browsers:
return x.lower()
elif('android' in x)or('samsung' in x)or('mini' in x)or('iphone' in x)or('in-app' in x)or('playstation' in x):
return 'mobile browser'
elif('mozilla' in x)or('chrome' in x)or('blackberry' in x)or('nokia' in x)or('browser' in x)or('amazon' in x):
return 'mobile browser'
elif('lunascape' in x)or('netscape' in x)or('blackberry' in x)or('konqueror' in x)or('puffin' in x)or('amazon' in x):
return 'mobile browser'
elif '(not set)' in x:
return x
else:
return 'others'
def adcontents_mapping(x):
if('google' in x):
return 'google'
elif('placement' in x)|('placememnt' in x):
return 'placement'
elif '(not set)' in x or 'nan' in x:
return x
elif 'ad' in x:
return 'ad'
else:
return 'others'
def source_mapping(x):
if('google' in x):
return 'google'
elif('youtube' in x):
return 'youtube'
elif '(not set)' in x or 'nan' in x:
return x
elif 'yahoo' in x:
return 'yahoo'
elif 'facebook' in x:
return 'facebook'
elif 'reddit' in x:
return 'reddit'
elif 'bing' in x:
return 'bing'
elif 'quora' in x:
return 'quora'
elif 'outlook' in x:
return 'outlook'
elif 'linkedin' in x:
return 'linkedin'
elif 'pinterest' in x:
return 'pinterest'
elif 'ask' in x:
return 'ask'
elif 'siliconvalley' in x:
return 'siliconvalley'
elif 'lunametrics' in x:
return 'lunametrics'
elif 'amazon' in x:
return 'amazon'
elif 'mysearch' in x:
return 'mysearch'
elif 'qiita' in x:
return 'qiita'
elif 'messenger' in x:
return 'messenger'
elif 'twitter' in x:
return 'twitter'
elif 't.co' in x:
return 't.co'
elif 'vk.com' in x:
return 'vk.com'
elif 'search' in x:
return 'search'
elif 'edu' in x:
return 'edu'
elif 'mail' in x:
return 'mail'
elif 'ad' in x:
return 'ad'
elif 'golang' in x:
return 'golang'
elif 'direct' in x:
return 'direct'
elif 'dealspotr' in x:
return 'dealspotr'
elif 'sashihara' in x:
return 'sashihara'
elif 'phandroid' in x:
return 'phandroid'
elif 'baidu' in x:
return 'baidu'
elif 'mdn' in x:
return 'mdn'
elif 'duckduckgo' in x:
return 'duckduckgo'
elif 'seroundtable' in x:
return 'seroundtable'
elif 'metrics' in x:
return 'metrics'
elif 'sogou' in x:
return 'sogou'
elif 'businessinsider' in x:
return 'businessinsider'
elif 'github' in x:
return 'github'
elif 'gophergala' in x:
return 'gophergala'
elif 'yandex' in x:
return 'yandex'
elif 'msn' in x:
return 'msn'
elif 'dfa' in x:
return 'dfa'
elif '(not set)' in x:
return '(not set)'
elif 'feedly' in x:
return 'feedly'
elif 'arstechnica' in x:
return 'arstechnica'
elif 'squishable' in x:
return 'squishable'
elif 'flipboard' in x:
return 'flipboard'
elif 't-online.de' in x:
return 't-online.de'
elif 'sm.cn' in x:
return 'sm.cn'
elif 'wow' in x:
return 'wow'
elif 'baidu' in x:
return 'baidu'
elif 'partners' in x:
return 'partners'
else:
return 'others'
train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 train['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 train['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 test['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x ).lower())).astype('str')
 test['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x ).lower())).astype('str')
 test['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x ).lower())).astype('str')
 def process_device(data_df):
print("process device...")
 data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']
data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']
data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']
data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']
return data_df
train = process_device(train)
 test = process_device(test)
 def custom(data):
print('custom.. ')
 data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + "_" + data['channelGrouping']
data['channelGrouping_browser'] = data['device.browser'] + "_" + data['channelGrouping']
data['channelGrouping_OS'] = data['device.operatingSystem'] + "_" + data['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
data[i + "_" + j] = data[i] + "_" + data[j]
data['content.source'] = data['trafficSource.adContent'] + "_" + data['source.country']
data['medium.source'] = data['trafficSource.medium'] + "_" + data['source.country']
return data
train = custom(train)
 test = custom(test )<define_variables>
X_test = test.drop(['PassengerId'], axis=1 )
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
y_test_pred = logmodel.predict(X_test )
<init_hyperparams><EOS> 
output = pd.DataFrame({ 'PassengerId' : test['PassengerId'], 'Survived': y_test_pred })
 output.to_csv('submission.csv', index=False )